restructure, optimize logging

dlt-connector/bun.lock

@@ -25,6 +25,7 @@
         "typescript": "^5.8.3",
         "uuid": "^8.3.2",
         "valibot": "1.1.0",
+        "yoctocolors-cjs": "^2.1.3",
       },
     },
   },
@@ -999,6 +1000,8 @@
 
     "yargs-parser": ["yargs-parser@21.1.1", "", {}, "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw=="],
 
+    "yoctocolors-cjs": ["yoctocolors-cjs@2.1.3", "", {}, "sha512-U/PBtDf35ff0D8X8D0jfdzHYEPFxAI7jJlxZXwCSez5M3190m+QobIfh+sWDWSHMCWWJN2AWamkegn6vr6YBTw=="],
+
     "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="],
 
     "@babel/core/debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],

dlt-connector/package.json

@@ -38,7 +38,8 @@
     "log4js": "^6.9.1",
     "typescript": "^5.8.3",
     "uuid": "^8.3.2",
-    "valibot": "1.1.0"
+    "valibot": "1.1.0",
+    "yoctocolors-cjs": "^2.1.3"
   },
   "engines": {
     "node": ">=18"

dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/OrderedContainer.ts

@@ -7,6 +7,7 @@ export interface Orderable<ContextType> {
   ensureFilled(context: ContextType, batchSize: number): Promise<number>
   pushToBlockchain(context: ContextType): Promise<void>
   isEmpty(): boolean
+  get length(): number
 }
 
 export class OrderedContainer<T, ContextType> implements Orderable<ContextType> {
@@ -48,6 +49,10 @@ export class OrderedContainer<T, ContextType> implements Orderable<ContextType>
     return item
   }
 
+  get length(): number {
+    return this.items.length
+  }
+
   getDate(): Date {
     return this.getDateHandler(this.peek())
   }

dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts

@@ -1,38 +1,21 @@
-import { sql, SQL } from 'bun'
 import { 
   InMemoryBlockchain, 
-  InMemoryBlockchainProvider, 
   GradidoTransactionBuilder, 
-  KeyPairEd25519, 
-  MemoryBlock, 
   Timestamp, 
   HieroTransactionId, 
   HieroAccountId, 
   InteractionSerialize, 
-  loadCryptoKeys, 
-  Filter,
-  SearchDirection_ASC
 } from 'gradido-blockchain-js'
-import { Logger, getLogger } from 'log4js'
-import { CONFIG } from '../../config'
-import { amountSchema, HieroId, hieroIdSchema, memoSchema, uuidv4Schema, Uuidv4, gradidoAmountSchema } from '../../schemas/typeGuard.schema'
-import { dateSchema } from '../../schemas/typeConverter.schema'
-import * as v from 'valibot'
+import { getLogger } from 'log4js'
 import { RegisterAddressTransactionRole } from '../../interactions/sendToHiero/RegisterAddressTransaction.role'
-import { KeyPairCacheManager } from '../../cache/KeyPairCacheManager'
-import { InputTransactionType } from '../../data/InputTransactionType.enum'
-import { AccountType } from '../../data/AccountType.enum'
 import { CommunityRootTransactionRole } from '../../interactions/sendToHiero/CommunityRootTransaction.role'
-import { UserKeyPairRole } from '../../interactions/resolveKeyPair/UserKeyPair.role'
-import { KeyPairIdentifierLogic } from '../../data/KeyPairIdentifier.logic'
-import { AccountKeyPairRole } from '../../interactions/resolveKeyPair/AccountKeyPair.role'
-import { loadConfig } from '../../bootstrap/init'
 import { CreationTransactionRole } from '../../interactions/sendToHiero/CreationTransaction.role'
-import { CommunityDb, loadCommunities, TransactionDb, TransactionTypeId, UserDb } from './database'
 import { LOG4JS_BASE_CATEGORY } from '../../config/const'
 import { Community, Transaction } from '../../schemas/transaction.schema'
-import { communityDbToCommunity, userDbToTransaction } from './convert'
 import { TransferTransactionRole } from '../../interactions/sendToHiero/TransferTransaction.role'
+import { DeferredTransferTransactionRole } from '../../interactions/sendToHiero/DeferredTransferTransaction.role'
+import { RedeemDeferredTransferTransactionRole } from '../../interactions/sendToHiero/RedeemDeferredTransferTransaction.role'
+import { InputTransactionType } from '../../data/InputTransactionType.enum'
 
 const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`)
 export const defaultHieroAccount = new HieroAccountId(0, 0, 2)
@@ -83,5 +66,19 @@ export async function addTransaction(
     } else {
       throw new Error(`Transfer Transaction not added for user ${transaction.user.account!.userUuid}`)
     }
+  } else if (transaction.type === InputTransactionType.GRADIDO_DEFERRED_TRANSFER) {
+    const transferTransactionRole = new DeferredTransferTransactionRole(transaction)
+    if(addToBlockchain(await transferTransactionRole.getGradidoTransactionBuilder(), senderBlockchain, createdAtTimestamp)) {
+      logger.debug(`Deferred Transfer Transaction added for user ${transaction.user.account!.userUuid}`)
+    } else {
+      throw new Error(`Deferred Transfer Transaction not added for user ${transaction.user.account!.userUuid}`)
+    }
+  } else if (transaction.type === InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER) {
+    const redeemTransactionRole = new RedeemDeferredTransferTransactionRole(transaction)
+    if(addToBlockchain(await redeemTransactionRole.getGradidoTransactionBuilder(), senderBlockchain, createdAtTimestamp)) {
+      logger.debug(`Redeem Deferred Transfer Transaction added for user ${transaction.user.account!.userUuid}`)
+    } else {
+      throw new Error(`Redeem Deferred Transfer Transaction not added for user ${transaction.user.account!.userUuid}`)
+    }
   }
 }

dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/convert.ts

@@ -1,8 +1,8 @@
 import { InputTransactionType } from '../../data/InputTransactionType.enum'
-import { CommunityDb, TransactionDb, TransactionTypeId, CreatedUserDb } from './database'
+import { CommunityDb, TransactionDb, TransactionTypeId, CreatedUserDb, TransactionLinkDb } from './database'
 import { Community, communitySchema, transactionSchema, Transaction, TransactionInput } from '../../schemas/transaction.schema'
 import { AccountType } from '../../data/AccountType.enum'
-import { gradidoAmountSchema, HieroId, memoSchema } from '../../schemas/typeGuard.schema'
+import { gradidoAmountSchema, HieroId, memoSchema, timeoutDurationSchema } from '../../schemas/typeGuard.schema'
 import * as v from 'valibot'
 
 export function getInputTransactionTypeFromTypeId(typeId: TransactionTypeId): InputTransactionType {
@@ -53,11 +53,11 @@ export function transactionDbToTransaction(
   
   const user = {
     communityTopicId: communityTopicId,
-    account: { userUuid: transactionDb.user.gradidoId, accountNr: 0 },
+    account: { userUuid: transactionDb.user.gradidoId },
   }
   const linkedUser = {
     communityTopicId: recipientCommunityTopicId,
-    account: { userUuid: transactionDb.linkedUser.gradidoId, accountNr: 0 },
+    account: { userUuid: transactionDb.linkedUser.gradidoId },
   }
   const transaction: TransactionInput = {
     user,
@@ -89,3 +89,22 @@ export function transactionDbToTransaction(
   }
   return v.parse(transactionSchema, transaction)
 }
+
+export function transactionLinkDbToTransaction(transactionLinkDb: TransactionLinkDb, communityTopicId: HieroId): Transaction {
+  return v.parse(transactionSchema, {
+    user: {
+      communityTopicId: communityTopicId,
+      account: { userUuid: transactionLinkDb.user.gradidoId },
+    },
+    linkedUser: {
+      communityTopicId: communityTopicId,
+      seed: transactionLinkDb.code,
+    },
+    type: InputTransactionType.GRADIDO_DEFERRED_TRANSFER,
+    amount: v.parse(gradidoAmountSchema, transactionLinkDb.amount),
+    memo: v.parse(memoSchema, transactionLinkDb.memo),
+    createdAt: transactionLinkDb.createdAt,
+    timeoutDuration: v.parse(timeoutDurationSchema, Math.round((transactionLinkDb.validUntil.getTime() - transactionLinkDb.createdAt.getTime()) / 1000)),
+  })
+}
+

dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts

@@ -1,5 +1,5 @@
 import { SQL } from 'bun'
-import { amountSchema, memoSchema, uuidv4Schema, identifierSeedSchema, gradidoAmountSchema } from '../../schemas/typeGuard.schema'
+import { memoSchema, uuidv4Schema, identifierSeedSchema, gradidoAmountSchema } from '../../schemas/typeGuard.schema'
 import { dateSchema, booleanSchema } from '../../schemas/typeConverter.schema'
 import * as v from 'valibot'
 import { GradidoUnit } from 'gradido-blockchain-js'
@@ -38,7 +38,7 @@ export const transactionDbSchema = v.object({
 })
 
 export const transactionLinkDbSchema = v.object({
-  userUuid: uuidv4Schema,
+  user: userDbSchema,
   code: identifierSeedSchema,
   amount: gradidoAmountSchema,
   memo: memoSchema,
@@ -157,14 +157,20 @@ export async function loadTransactions(db: SQL, offset: number, count: number):
 
 export async function loadTransactionLinks(db: SQL, offset: number, count: number): Promise<TransactionLinkDb[]> {
   const result = await db`
-    SELECT u.gradido_id as userUuid, tl.code, tl.amount, tl.memo, tl.createdAt, tl.validUntil
+    SELECT u.gradido_id as userGradidoId, u.community_uuid as userCommunityUuid, tl.code, tl.amount, tl.memo, tl.createdAt, tl.validUntil
     FROM transaction_links tl
     LEFT JOIN users u ON tl.userId = u.id
     ORDER by createdAt ASC
     LIMIT ${offset}, ${count}
   `
   return result.map((row: any) => {
-    return v.parse(transactionLinkDbSchema, row)
+    return v.parse(transactionLinkDbSchema, {
+      ...row,
+      user: {
+        gradidoId: row.userGradidoId,
+        communityUuid: row.userCommunityUuid
+      }
+    })
   })
 }
 

dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts

@@ -2,22 +2,19 @@ import {
   InMemoryBlockchain, 
   Filter,
   SearchDirection_ASC,
-  HieroTransactionId,
-  Timestamp,
-  InteractionSerialize,
   Profiler
 } from 'gradido-blockchain-js'
 import { Logger } from 'log4js'
 
 import { CreatedUserDb, loadDeletedTransactionLinks, loadTransactionLinks, loadTransactions, loadUsers, TransactionDb, TransactionLinkDb } from './database'
-import { addRegisterAddressTransaction, addTransaction, defaultHieroAccount } from './blockchain'
+import { addRegisterAddressTransaction, addTransaction } from './blockchain'
 import { generateKeyPairUserAccount } from './keyPair'
-import { transactionDbToTransaction, userDbToTransaction } from './convert'
+import { transactionDbToTransaction, transactionLinkDbToTransaction, userDbToTransaction } from './convert'
 import { Orderable, OrderedContainer } from './OrderedContainer'
 import { Context } from './Context'
 import { bootstrap } from './bootstrap'
-import { RegisterAddressTransactionRole } from '../../interactions/sendToHiero/RegisterAddressTransaction.role'
 import { heapStats } from 'bun:jsc'
+import { onShutdown } from '../../../../shared/src/helper/onShutdown'
 
 const publicKeyUserIdMap = new Map<string, string>()
 
@@ -25,7 +22,13 @@ async function main() {
   const timeUsed = new Profiler()
   // prepare in memory blockchains
   const context = await bootstrap()
-
+  onShutdown(async (reason, error) => {
+    context.logger.info(`shutdown reason: ${reason}`)
+    if(error) {
+      context.logger.error(error)
+    }
+    context.db.close()
+  })
   // synchronize to blockchain
   const BATCH_SIZE = 100
 
@@ -40,10 +43,19 @@ async function main() {
     (context: Context, transaction: TransactionDb) => pushTransaction(context, transaction)
   )
 
-  // const transactionLinks = new OrderedContainer(getNextTransactionLinks, (transactionLink) => transactionLink.createdAt)
-  // const deletedTransactionLinks = new OrderedContainer(getNextDeletedTransactionLinks, (transactionLink) => transactionLink.balanceDate)
+  const transactionLinks = new OrderedContainer(
+    getNextTransactionLinks, 
+    (transactionLink: TransactionLinkDb) => transactionLink.createdAt,
+    (context: Context, transactionLink: TransactionLinkDb) => pushTransactionLink(context, transactionLink)
+  )
 
-  await synchronizeToBlockchain(context, [users, transactions], BATCH_SIZE)
+  const deletedTransactionLinks = new OrderedContainer(
+    getNextDeletedTransactionLinks,
+    (transaction: TransactionDb) => transaction.balanceDate,
+    (context: Context, transaction: TransactionDb) => pushTransaction(context, transaction)
+  )
+
+  await synchronizeToBlockchain(context, [users, transactions, transactionLinks, deletedTransactionLinks], BATCH_SIZE)
   context.logger.info(`${timeUsed.string()} for synchronizing to blockchain`)
   timeUsed.reset()
   context.communities.forEach((communityContext) => {
@@ -55,17 +67,12 @@ async function main() {
     // logBlogchain(context.logger, communityContext.blockchain)
   })
   context.logger.info(`${timeUsed.string()} for logging blockchains`)
-  context.db.close()
   const runtimeStats = heapStats()
-  /*
-   heapSize: 24254530,
-  heapCapacity: 32191922,
-  extraMemorySize: 7003858
-  */
   context.logger.info(
     `Memory Statistics: heap size: ${bytesToMbyte(runtimeStats.heapSize)} MByte, heap capacity: ${bytesToMbyte(runtimeStats.heapCapacity)} MByte, extra memory: ${bytesToMbyte(runtimeStats.extraMemorySize)} MByte`
   )
-  return Promise.resolve()
+  // needed because of shutdown handler (TODO: fix shutdown handler)
+  process.exit(0)
 }
 
 function bytesToMbyte(bytes: number): string {
@@ -77,57 +84,26 @@ async function synchronizeToBlockchain(
   containers: Orderable<Context>[],
   batchSize: number
 ): Promise<void> {
-  let rounds = 200
-  while (rounds-- > 0) {
+  while (true) {
+    const timeUsed = new Profiler()
     await Promise.all(containers.map(c => c.ensureFilled(context, batchSize)))
-    // console.log(`filled containers, rounds left: ${rounds}`)
+    const itemCount = containers.reduce((acc, c) => acc + c.length, 0)
+    context.logger.info(`${timeUsed.string()} for ensuring filled containers, ${itemCount} items`)
+    
     // remove empty containers
     const available = containers.filter(c => !c.isEmpty())
     if (available.length === 0) break
-    // console.log(`available containers: ${available.length}`)
 
     // find container with smallest date
     available.sort((a, b) => a.getDate().getTime() - b.getDate().getTime())
-    // console.log(`smallest date: ${available[0].getDate()}`)
     
-    if(rounds >= 0) {
-      try {
-        await available[0].pushToBlockchain(context)
-      } catch (e) {
-        console.error(e)
-        logBlogchain(context.logger, context.communities.values().next().value!.blockchain)
-        // for(const [key, value] of publicKeyUserIdMap.entries()) {
-          // console.log(`${key}: ${value}`)
-        // }
-        throw e
-      }
-    } else {
-      const user = (available[0] as any as OrderedContainer<any, Context>).shift()
-      console.log(JSON.stringify(user, null, 2))
-      console.log(`context: ${JSON.stringify(context, null, 2)}`)
-      const communityContext = context.getCommunityContextByUuid(user.communityUuid)
-      const transactionBase = userDbToTransaction(user, communityContext.topicId)
-      console.log(JSON.stringify(transactionBase, null, 2))
-      const registerAddressRole = new RegisterAddressTransactionRole(transactionBase)
-      const builder = await registerAddressRole.getGradidoTransactionBuilder()
-      const transaction = builder.build()
-      console.log(transaction.toJson(true))
-      const createdAtTimestamp = new Timestamp(user.createdAt)
-      console.log(`createdAtTimestamp: ${createdAtTimestamp.toJson()}`)
-      const transactionId = new HieroTransactionId(createdAtTimestamp, defaultHieroAccount)
-      const interactionSerialize = new InteractionSerialize(transactionId)
-      const serializedTransactionId = interactionSerialize.run()
-      if (serializedTransactionId) {
-        console.log(`serialized transaction id: ${serializedTransactionId.convertToHex()}`)
-      }
-      console.log(communityContext.blockchain)
-      try {
-        communityContext.blockchain.createAndAddConfirmedTransaction(transaction, serializedTransactionId, createdAtTimestamp)
-      } catch(e) {
-        console.log(e)
-      }
+    try {
+      await available[0].pushToBlockchain(context)
+    } catch (e) {
+      console.error(e)
+      logBlogchain(context.logger, context.communities.values().next().value!.blockchain)
+      throw e
     }
-    // console.log(`pushed to blockchain, rounds left: ${rounds}`)
   }
 }
 
@@ -163,7 +139,9 @@ async function getNextTransactions(context: Context, offset: number, count: numb
 async function getNextTransactionLinks(context: Context, offset: number, count: number): Promise<TransactionLinkDb[]> {
   const timeUsed = new Profiler()
   const transactionLinks = await loadTransactionLinks(context.db, offset, count)
-  context.logger.info(`${timeUsed.string()} for loading ${transactionLinks.length} transaction links from db`)
+  if(transactionLinks.length !== 0) {
+    context.logger.info(`${timeUsed.string()} for loading ${transactionLinks.length} transaction links from db`)
+  }
   return transactionLinks
 }
 
@@ -171,7 +149,9 @@ async function getNextTransactionLinks(context: Context, offset: number, count:
 async function getNextDeletedTransactionLinks(context: Context, offset: number, count: number): Promise<TransactionDb[]> {
   const timeUsed = new Profiler()
   const deletedTransactionLinks = await loadDeletedTransactionLinks(context.db, offset, count)
-  context.logger.info(`${timeUsed.string()} for loading ${deletedTransactionLinks.length} deleted transaction links from db`)
+  if(deletedTransactionLinks.length !== 0) {
+    context.logger.info(`${timeUsed.string()} for loading ${deletedTransactionLinks.length} deleted transaction links from db`)
+  }
   return deletedTransactionLinks
 }
 
@@ -183,7 +163,6 @@ async function pushRegisterAddressTransaction(context: Context, user: CreatedUse
   return await addRegisterAddressTransaction(communityContext.blockchain, transaction)
 }
 
-
 async function pushTransaction(context: Context, transactionDb: TransactionDb): Promise<void> {
   const senderCommunityContext = context.getCommunityContextByUuid(transactionDb.user.communityUuid)
   const recipientCommunityContext = context.getCommunityContextByUuid(transactionDb.linkedUser.communityUuid)
@@ -193,6 +172,12 @@ async function pushTransaction(context: Context, transactionDb: TransactionDb):
   await addTransaction(senderCommunityContext.blockchain, recipientCommunityContext.blockchain, transaction)
 }
 
+async function pushTransactionLink(context: Context, transactionLinkDb: TransactionLinkDb): Promise<void> {
+  const communityContext = context.getCommunityContextByUuid(transactionLinkDb.user.communityUuid)
+  const transaction = transactionLinkDbToTransaction(transactionLinkDb, communityContext.topicId)
+  await addTransaction(communityContext.blockchain, communityContext.blockchain, transaction)
+}
+
 // ---------------- utils ----------------------------------------------------------------------
 
 function logBlogchain(logger: Logger, blockchain: InMemoryBlockchain) {