From eb8c806b5dd44e6207a38177954f7a49c288fa67 Mon Sep 17 00:00:00 2001 From: Einhornimmond Date: Mon, 3 Nov 2025 14:25:03 +0100 Subject: [PATCH] restructure, optimize logging --- dlt-connector/bun.lock | 3 + dlt-connector/package.json | 3 +- .../OrderedContainer.ts | 5 + .../blockchain.ts | 39 +++---- .../db-v2.7.0_to_blockchain-v3.5/convert.ts | 27 ++++- .../db-v2.7.0_to_blockchain-v3.5/database.ts | 14 ++- .../db-v2.7.0_to_blockchain-v3.5/index.ts | 109 ++++++++---------- 7 files changed, 108 insertions(+), 92 deletions(-) diff --git a/dlt-connector/bun.lock b/dlt-connector/bun.lock index 21caa0f07..1f27138de 100644 --- a/dlt-connector/bun.lock +++ b/dlt-connector/bun.lock @@ -25,6 +25,7 @@ "typescript": "^5.8.3", "uuid": "^8.3.2", "valibot": "1.1.0", + "yoctocolors-cjs": "^2.1.3", }, }, }, @@ -999,6 +1000,8 @@ "yargs-parser": ["yargs-parser@21.1.1", "", {}, "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw=="], + "yoctocolors-cjs": ["yoctocolors-cjs@2.1.3", "", {}, "sha512-U/PBtDf35ff0D8X8D0jfdzHYEPFxAI7jJlxZXwCSez5M3190m+QobIfh+sWDWSHMCWWJN2AWamkegn6vr6YBTw=="], + "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], "@babel/core/debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], diff --git a/dlt-connector/package.json b/dlt-connector/package.json index 78c3c1b96..fd33612a5 100644 --- a/dlt-connector/package.json +++ b/dlt-connector/package.json @@ -38,7 +38,8 @@ "log4js": "^6.9.1", "typescript": "^5.8.3", "uuid": "^8.3.2", - "valibot": "1.1.0" + "valibot": "1.1.0", + "yoctocolors-cjs": "^2.1.3" }, "engines": { "node": ">=18" diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/OrderedContainer.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/OrderedContainer.ts index 90dcc0e74..47595b3d0 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/OrderedContainer.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/OrderedContainer.ts @@ -7,6 +7,7 @@ export interface Orderable { ensureFilled(context: ContextType, batchSize: number): Promise pushToBlockchain(context: ContextType): Promise isEmpty(): boolean + get length(): number } export class OrderedContainer implements Orderable { @@ -48,6 +49,10 @@ export class OrderedContainer implements Orderable return item } + get length(): number { + return this.items.length + } + getDate(): Date { return this.getDateHandler(this.peek()) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts index 9d1e43409..12ba38017 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts @@ -1,38 +1,21 @@ -import { sql, SQL } from 'bun' import { InMemoryBlockchain, - InMemoryBlockchainProvider, GradidoTransactionBuilder, - KeyPairEd25519, - MemoryBlock, Timestamp, HieroTransactionId, HieroAccountId, InteractionSerialize, - loadCryptoKeys, - Filter, - SearchDirection_ASC } from 'gradido-blockchain-js' -import { Logger, getLogger } from 'log4js' -import { CONFIG } from '../../config' -import { amountSchema, HieroId, hieroIdSchema, memoSchema, uuidv4Schema, Uuidv4, gradidoAmountSchema } from '../../schemas/typeGuard.schema' -import { dateSchema } from '../../schemas/typeConverter.schema' -import * as v from 'valibot' +import { getLogger } from 'log4js' import { RegisterAddressTransactionRole } from '../../interactions/sendToHiero/RegisterAddressTransaction.role' -import { KeyPairCacheManager } from '../../cache/KeyPairCacheManager' -import { InputTransactionType } from '../../data/InputTransactionType.enum' -import { AccountType } from '../../data/AccountType.enum' import { CommunityRootTransactionRole } from '../../interactions/sendToHiero/CommunityRootTransaction.role' -import { UserKeyPairRole } from '../../interactions/resolveKeyPair/UserKeyPair.role' -import { KeyPairIdentifierLogic } from '../../data/KeyPairIdentifier.logic' -import { AccountKeyPairRole } from '../../interactions/resolveKeyPair/AccountKeyPair.role' -import { loadConfig } from '../../bootstrap/init' import { CreationTransactionRole } from '../../interactions/sendToHiero/CreationTransaction.role' -import { CommunityDb, loadCommunities, TransactionDb, TransactionTypeId, UserDb } from './database' import { LOG4JS_BASE_CATEGORY } from '../../config/const' import { Community, Transaction } from '../../schemas/transaction.schema' -import { communityDbToCommunity, userDbToTransaction } from './convert' import { TransferTransactionRole } from '../../interactions/sendToHiero/TransferTransaction.role' +import { DeferredTransferTransactionRole } from '../../interactions/sendToHiero/DeferredTransferTransaction.role' +import { RedeemDeferredTransferTransactionRole } from '../../interactions/sendToHiero/RedeemDeferredTransferTransaction.role' +import { InputTransactionType } from '../../data/InputTransactionType.enum' const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`) export const defaultHieroAccount = new HieroAccountId(0, 0, 2) @@ -83,5 +66,19 @@ export async function addTransaction( } else { throw new Error(`Transfer Transaction not added for user ${transaction.user.account!.userUuid}`) } + } else if (transaction.type === InputTransactionType.GRADIDO_DEFERRED_TRANSFER) { + const transferTransactionRole = new DeferredTransferTransactionRole(transaction) + if(addToBlockchain(await transferTransactionRole.getGradidoTransactionBuilder(), senderBlockchain, createdAtTimestamp)) { + logger.debug(`Deferred Transfer Transaction added for user ${transaction.user.account!.userUuid}`) + } else { + throw new Error(`Deferred Transfer Transaction not added for user ${transaction.user.account!.userUuid}`) + } + } else if (transaction.type === InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER) { + const redeemTransactionRole = new RedeemDeferredTransferTransactionRole(transaction) + if(addToBlockchain(await redeemTransactionRole.getGradidoTransactionBuilder(), senderBlockchain, createdAtTimestamp)) { + logger.debug(`Redeem Deferred Transfer Transaction added for user ${transaction.user.account!.userUuid}`) + } else { + throw new Error(`Redeem Deferred Transfer Transaction not added for user ${transaction.user.account!.userUuid}`) + } } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/convert.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/convert.ts index a455aa1e1..747e6c2ef 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/convert.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/convert.ts @@ -1,8 +1,8 @@ import { InputTransactionType } from '../../data/InputTransactionType.enum' -import { CommunityDb, TransactionDb, TransactionTypeId, CreatedUserDb } from './database' +import { CommunityDb, TransactionDb, TransactionTypeId, CreatedUserDb, TransactionLinkDb } from './database' import { Community, communitySchema, transactionSchema, Transaction, TransactionInput } from '../../schemas/transaction.schema' import { AccountType } from '../../data/AccountType.enum' -import { gradidoAmountSchema, HieroId, memoSchema } from '../../schemas/typeGuard.schema' +import { gradidoAmountSchema, HieroId, memoSchema, timeoutDurationSchema } from '../../schemas/typeGuard.schema' import * as v from 'valibot' export function getInputTransactionTypeFromTypeId(typeId: TransactionTypeId): InputTransactionType { @@ -53,11 +53,11 @@ export function transactionDbToTransaction( const user = { communityTopicId: communityTopicId, - account: { userUuid: transactionDb.user.gradidoId, accountNr: 0 }, + account: { userUuid: transactionDb.user.gradidoId }, } const linkedUser = { communityTopicId: recipientCommunityTopicId, - account: { userUuid: transactionDb.linkedUser.gradidoId, accountNr: 0 }, + account: { userUuid: transactionDb.linkedUser.gradidoId }, } const transaction: TransactionInput = { user, @@ -89,3 +89,22 @@ export function transactionDbToTransaction( } return v.parse(transactionSchema, transaction) } + +export function transactionLinkDbToTransaction(transactionLinkDb: TransactionLinkDb, communityTopicId: HieroId): Transaction { + return v.parse(transactionSchema, { + user: { + communityTopicId: communityTopicId, + account: { userUuid: transactionLinkDb.user.gradidoId }, + }, + linkedUser: { + communityTopicId: communityTopicId, + seed: transactionLinkDb.code, + }, + type: InputTransactionType.GRADIDO_DEFERRED_TRANSFER, + amount: v.parse(gradidoAmountSchema, transactionLinkDb.amount), + memo: v.parse(memoSchema, transactionLinkDb.memo), + createdAt: transactionLinkDb.createdAt, + timeoutDuration: v.parse(timeoutDurationSchema, Math.round((transactionLinkDb.validUntil.getTime() - transactionLinkDb.createdAt.getTime()) / 1000)), + }) +} + diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts index bdd377821..f63a7a002 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts @@ -1,5 +1,5 @@ import { SQL } from 'bun' -import { amountSchema, memoSchema, uuidv4Schema, identifierSeedSchema, gradidoAmountSchema } from '../../schemas/typeGuard.schema' +import { memoSchema, uuidv4Schema, identifierSeedSchema, gradidoAmountSchema } from '../../schemas/typeGuard.schema' import { dateSchema, booleanSchema } from '../../schemas/typeConverter.schema' import * as v from 'valibot' import { GradidoUnit } from 'gradido-blockchain-js' @@ -38,7 +38,7 @@ export const transactionDbSchema = v.object({ }) export const transactionLinkDbSchema = v.object({ - userUuid: uuidv4Schema, + user: userDbSchema, code: identifierSeedSchema, amount: gradidoAmountSchema, memo: memoSchema, @@ -157,14 +157,20 @@ export async function loadTransactions(db: SQL, offset: number, count: number): export async function loadTransactionLinks(db: SQL, offset: number, count: number): Promise { const result = await db` - SELECT u.gradido_id as userUuid, tl.code, tl.amount, tl.memo, tl.createdAt, tl.validUntil + SELECT u.gradido_id as userGradidoId, u.community_uuid as userCommunityUuid, tl.code, tl.amount, tl.memo, tl.createdAt, tl.validUntil FROM transaction_links tl LEFT JOIN users u ON tl.userId = u.id ORDER by createdAt ASC LIMIT ${offset}, ${count} ` return result.map((row: any) => { - return v.parse(transactionLinkDbSchema, row) + return v.parse(transactionLinkDbSchema, { + ...row, + user: { + gradidoId: row.userGradidoId, + communityUuid: row.userCommunityUuid + } + }) }) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts index 113f37aa1..b7edfb6ec 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts @@ -2,22 +2,19 @@ import { InMemoryBlockchain, Filter, SearchDirection_ASC, - HieroTransactionId, - Timestamp, - InteractionSerialize, Profiler } from 'gradido-blockchain-js' import { Logger } from 'log4js' import { CreatedUserDb, loadDeletedTransactionLinks, loadTransactionLinks, loadTransactions, loadUsers, TransactionDb, TransactionLinkDb } from './database' -import { addRegisterAddressTransaction, addTransaction, defaultHieroAccount } from './blockchain' +import { addRegisterAddressTransaction, addTransaction } from './blockchain' import { generateKeyPairUserAccount } from './keyPair' -import { transactionDbToTransaction, userDbToTransaction } from './convert' +import { transactionDbToTransaction, transactionLinkDbToTransaction, userDbToTransaction } from './convert' import { Orderable, OrderedContainer } from './OrderedContainer' import { Context } from './Context' import { bootstrap } from './bootstrap' -import { RegisterAddressTransactionRole } from '../../interactions/sendToHiero/RegisterAddressTransaction.role' import { heapStats } from 'bun:jsc' +import { onShutdown } from '../../../../shared/src/helper/onShutdown' const publicKeyUserIdMap = new Map() @@ -25,7 +22,13 @@ async function main() { const timeUsed = new Profiler() // prepare in memory blockchains const context = await bootstrap() - + onShutdown(async (reason, error) => { + context.logger.info(`shutdown reason: ${reason}`) + if(error) { + context.logger.error(error) + } + context.db.close() + }) // synchronize to blockchain const BATCH_SIZE = 100 @@ -40,10 +43,19 @@ async function main() { (context: Context, transaction: TransactionDb) => pushTransaction(context, transaction) ) - // const transactionLinks = new OrderedContainer(getNextTransactionLinks, (transactionLink) => transactionLink.createdAt) - // const deletedTransactionLinks = new OrderedContainer(getNextDeletedTransactionLinks, (transactionLink) => transactionLink.balanceDate) + const transactionLinks = new OrderedContainer( + getNextTransactionLinks, + (transactionLink: TransactionLinkDb) => transactionLink.createdAt, + (context: Context, transactionLink: TransactionLinkDb) => pushTransactionLink(context, transactionLink) + ) - await synchronizeToBlockchain(context, [users, transactions], BATCH_SIZE) + const deletedTransactionLinks = new OrderedContainer( + getNextDeletedTransactionLinks, + (transaction: TransactionDb) => transaction.balanceDate, + (context: Context, transaction: TransactionDb) => pushTransaction(context, transaction) + ) + + await synchronizeToBlockchain(context, [users, transactions, transactionLinks, deletedTransactionLinks], BATCH_SIZE) context.logger.info(`${timeUsed.string()} for synchronizing to blockchain`) timeUsed.reset() context.communities.forEach((communityContext) => { @@ -55,17 +67,12 @@ async function main() { // logBlogchain(context.logger, communityContext.blockchain) }) context.logger.info(`${timeUsed.string()} for logging blockchains`) - context.db.close() const runtimeStats = heapStats() - /* - heapSize: 24254530, - heapCapacity: 32191922, - extraMemorySize: 7003858 - */ context.logger.info( `Memory Statistics: heap size: ${bytesToMbyte(runtimeStats.heapSize)} MByte, heap capacity: ${bytesToMbyte(runtimeStats.heapCapacity)} MByte, extra memory: ${bytesToMbyte(runtimeStats.extraMemorySize)} MByte` ) - return Promise.resolve() + // needed because of shutdown handler (TODO: fix shutdown handler) + process.exit(0) } function bytesToMbyte(bytes: number): string { @@ -77,57 +84,26 @@ async function synchronizeToBlockchain( containers: Orderable[], batchSize: number ): Promise { - let rounds = 200 - while (rounds-- > 0) { + while (true) { + const timeUsed = new Profiler() await Promise.all(containers.map(c => c.ensureFilled(context, batchSize))) - // console.log(`filled containers, rounds left: ${rounds}`) + const itemCount = containers.reduce((acc, c) => acc + c.length, 0) + context.logger.info(`${timeUsed.string()} for ensuring filled containers, ${itemCount} items`) + // remove empty containers const available = containers.filter(c => !c.isEmpty()) if (available.length === 0) break - // console.log(`available containers: ${available.length}`) // find container with smallest date available.sort((a, b) => a.getDate().getTime() - b.getDate().getTime()) - // console.log(`smallest date: ${available[0].getDate()}`) - if(rounds >= 0) { - try { - await available[0].pushToBlockchain(context) - } catch (e) { - console.error(e) - logBlogchain(context.logger, context.communities.values().next().value!.blockchain) - // for(const [key, value] of publicKeyUserIdMap.entries()) { - // console.log(`${key}: ${value}`) - // } - throw e - } - } else { - const user = (available[0] as any as OrderedContainer).shift() - console.log(JSON.stringify(user, null, 2)) - console.log(`context: ${JSON.stringify(context, null, 2)}`) - const communityContext = context.getCommunityContextByUuid(user.communityUuid) - const transactionBase = userDbToTransaction(user, communityContext.topicId) - console.log(JSON.stringify(transactionBase, null, 2)) - const registerAddressRole = new RegisterAddressTransactionRole(transactionBase) - const builder = await registerAddressRole.getGradidoTransactionBuilder() - const transaction = builder.build() - console.log(transaction.toJson(true)) - const createdAtTimestamp = new Timestamp(user.createdAt) - console.log(`createdAtTimestamp: ${createdAtTimestamp.toJson()}`) - const transactionId = new HieroTransactionId(createdAtTimestamp, defaultHieroAccount) - const interactionSerialize = new InteractionSerialize(transactionId) - const serializedTransactionId = interactionSerialize.run() - if (serializedTransactionId) { - console.log(`serialized transaction id: ${serializedTransactionId.convertToHex()}`) - } - console.log(communityContext.blockchain) - try { - communityContext.blockchain.createAndAddConfirmedTransaction(transaction, serializedTransactionId, createdAtTimestamp) - } catch(e) { - console.log(e) - } + try { + await available[0].pushToBlockchain(context) + } catch (e) { + console.error(e) + logBlogchain(context.logger, context.communities.values().next().value!.blockchain) + throw e } - // console.log(`pushed to blockchain, rounds left: ${rounds}`) } } @@ -163,7 +139,9 @@ async function getNextTransactions(context: Context, offset: number, count: numb async function getNextTransactionLinks(context: Context, offset: number, count: number): Promise { const timeUsed = new Profiler() const transactionLinks = await loadTransactionLinks(context.db, offset, count) - context.logger.info(`${timeUsed.string()} for loading ${transactionLinks.length} transaction links from db`) + if(transactionLinks.length !== 0) { + context.logger.info(`${timeUsed.string()} for loading ${transactionLinks.length} transaction links from db`) + } return transactionLinks } @@ -171,7 +149,9 @@ async function getNextTransactionLinks(context: Context, offset: number, count: async function getNextDeletedTransactionLinks(context: Context, offset: number, count: number): Promise { const timeUsed = new Profiler() const deletedTransactionLinks = await loadDeletedTransactionLinks(context.db, offset, count) - context.logger.info(`${timeUsed.string()} for loading ${deletedTransactionLinks.length} deleted transaction links from db`) + if(deletedTransactionLinks.length !== 0) { + context.logger.info(`${timeUsed.string()} for loading ${deletedTransactionLinks.length} deleted transaction links from db`) + } return deletedTransactionLinks } @@ -183,7 +163,6 @@ async function pushRegisterAddressTransaction(context: Context, user: CreatedUse return await addRegisterAddressTransaction(communityContext.blockchain, transaction) } - async function pushTransaction(context: Context, transactionDb: TransactionDb): Promise { const senderCommunityContext = context.getCommunityContextByUuid(transactionDb.user.communityUuid) const recipientCommunityContext = context.getCommunityContextByUuid(transactionDb.linkedUser.communityUuid) @@ -193,6 +172,12 @@ async function pushTransaction(context: Context, transactionDb: TransactionDb): await addTransaction(senderCommunityContext.blockchain, recipientCommunityContext.blockchain, transaction) } +async function pushTransactionLink(context: Context, transactionLinkDb: TransactionLinkDb): Promise { + const communityContext = context.getCommunityContextByUuid(transactionLinkDb.user.communityUuid) + const transaction = transactionLinkDbToTransaction(transactionLinkDb, communityContext.topicId) + await addTransaction(communityContext.blockchain, communityContext.blockchain, transaction) +} + // ---------------- utils ---------------------------------------------------------------------- function logBlogchain(logger: Logger, blockchain: InMemoryBlockchain) {