From be48f6aaa2bf75864a1c6e63ba505f915b8fc50a Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Mon, 9 Feb 2026 09:24:08 +0100 Subject: [PATCH] reduce new calls every tx --- .../binaryExport.ts | 6 +++++- .../db-v2.7.0_to_blockchain-v3.7/index.ts | 3 +++ .../syncDbWithBlockchain/AbstractSync.role.ts | 15 ++++++++++++++- .../CreationsSync.role.ts | 18 ++++++++++++------ .../DeletedTransactionLinksSync.role.ts | 16 +++++++++++----- .../LocalTransactionsSync.role.ts | 17 +++++++++++------ .../RedeemTransactionLinksSync.role.ts | 18 ++++++++++++------ .../RemoteTransactionsSync.role.ts | 19 ++++++++++++------- .../TransactionLinkFundingsSync.role.ts | 15 ++++++++++----- .../syncDbWithBlockchain/UsersSync.role.ts | 14 +++++++++----- .../syncDbWithBlockchain.context.ts | 10 +++++++--- 11 files changed, 106 insertions(+), 45 deletions(-) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts index 9d34f5588..8ec631201 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts @@ -27,6 +27,7 @@ export function exportCommunity( batchSize: number, ) { const timeUsed = new Profiler() + const timeSinceLastPrint = new Profiler() // write as binary file for GradidoNode const f = new Filter() f.pagination.size = batchSize @@ -64,7 +65,10 @@ export function exportCommunity( count++ } if (isDebug) { - printConsole() + if (timeSinceLastPrint.millis() > 100) { + printConsole() + timeSinceLastPrint.reset() + } } else { printCount++ if (printCount >= 100) { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts index b200e8769..cb8e99af3 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts @@ -2,10 +2,13 @@ import { onShutdown } from '../../../../shared/src/helper/onShutdown' import { exportAllCommunities } from './binaryExport' import { bootstrap } from './bootstrap' import { syncDbWithBlockchainContext } from './interaction/syncDbWithBlockchain/syncDbWithBlockchain.context' +// import { hello } from '../../../zig/hello.zig' const BATCH_SIZE = 1000 async function main() { + // hello() + // return // prepare in memory blockchains const context = await bootstrap() onShutdown(async (reason, error) => { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts index bcdc133b5..b2d532c44 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts @@ -1,4 +1,13 @@ -import { Filter, InMemoryBlockchain, KeyPairEd25519, MemoryBlockPtr, Profiler, SearchDirection_DESC } from 'gradido-blockchain-js' +import { + AccountBalances, + Filter, + InMemoryBlockchain, + KeyPairEd25519, + MemoryBlockPtr, + Profiler, + SearchDirection_DESC, + GradidoTransactionBuilder +} from 'gradido-blockchain-js' import { getLogger, Logger } from 'log4js' import { LOG4JS_BASE_CATEGORY } from '../../../../config/const' import { deriveFromKeyPairAndIndex, deriveFromKeyPairAndUuid } from '../../../../data/deriveKeyPair' @@ -18,11 +27,15 @@ export abstract class AbstractSyncRole { private items: ItemType[] = [] protected lastIndex: IndexType = { date: new Date(0), id: 0 } protected logger: Logger + protected transactionBuilder: GradidoTransactionBuilder + protected accountBalances: AccountBalances constructor(protected readonly context: Context) { this.logger = getLogger( `${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5.interaction.syncDbWithBlockchain`, ) + this.transactionBuilder = new GradidoTransactionBuilder() + this.accountBalances = new AccountBalances() } getAccountKeyPair(communityContext: CommunityContext, gradidoId: Uuidv4): KeyPairEd25519 { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts index 9546c07b9..589c65423 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts @@ -16,6 +16,7 @@ import { import * as v from 'valibot' import { addToBlockchain } from '../../blockchain' import { ContributionStatus } from '../../data/ContributionStatus' +import { Context } from '../../Context' import { contributionsTable, usersTable @@ -27,6 +28,11 @@ import { toMysqlDateTime } from '../../utils' export class CreationsSyncRole extends AbstractSyncRole { + constructor(context: Context) { + super(context) + this.accountBalances.reserve(3) + } + getDate(): Date { return this.peek().confirmedAt } @@ -85,7 +91,7 @@ export class CreationsSyncRole extends AbstractSyncRole { recipientKeyPair: KeyPairEd25519, signerKeyPair: KeyPairEd25519 ): GradidoTransactionBuilder { - return new GradidoTransactionBuilder() + return this.transactionBuilder .setCreatedAt(item.confirmedAt) .addMemo( new EncryptedMemo( @@ -107,7 +113,7 @@ export class CreationsSyncRole extends AbstractSyncRole { communityContext: CommunityContext, recipientPublicKey: MemoryBlockPtr ): AccountBalances { - const accountBalances = new AccountBalances() + this.accountBalances.clear() const balance = this.getLastBalanceForUser(recipientPublicKey, communityContext.blockchain, communityContext.communityId) // calculate decay since last balance with legacy calculation method @@ -115,10 +121,10 @@ export class CreationsSyncRole extends AbstractSyncRole { communityContext.aufBalance.updateLegacyDecay(item.amount, item.confirmedAt) communityContext.gmwBalance.updateLegacyDecay(item.amount, item.confirmedAt) - accountBalances.add(balance.getAccountBalance()) - accountBalances.add(communityContext.aufBalance.getAccountBalance()) - accountBalances.add(communityContext.gmwBalance.getAccountBalance()) - return accountBalances + this.accountBalances.add(balance.getAccountBalance()) + this.accountBalances.add(communityContext.aufBalance.getAccountBalance()) + this.accountBalances.add(communityContext.gmwBalance.getAccountBalance()) + return this.accountBalances } pushToBlockchain(item: CreationTransactionDb): void { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts index a1e6d046e..4eb338d37 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts @@ -21,8 +21,14 @@ import { addToBlockchain } from '../../blockchain' import { BlockchainError, DatabaseError } from '../../errors' import { Balance } from '../../data/Balance' import { toMysqlDateTime } from '../../utils' +import { Context } from '../../Context' export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { + constructor(context: Context) { + super(context) + this.accountBalances.reserve(2) + } + getDate(): Date { return this.peek().deletedAt } @@ -81,7 +87,7 @@ export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { - + constructor(context: Context) { + super(context) + this.accountBalances.reserve(2) + } + getDate(): Date { return this.peek().balanceDate } @@ -84,7 +89,7 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { senderKeyPair: KeyPairEd25519, recipientKeyPair: KeyPairEd25519, ): GradidoTransactionBuilder { - return new GradidoTransactionBuilder() + return this.transactionBuilder .setCreatedAt(item.balanceDate) .addMemo(new EncryptedMemo( item.memo, @@ -106,7 +111,7 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { senderPublicKey: MemoryBlockPtr, recipientPublicKey: MemoryBlockPtr, ): AccountBalances { - const accountBalances = new AccountBalances() + this.accountBalances.clear() const senderLastBalance = this.getLastBalanceForUser(senderPublicKey, communityContext.blockchain, communityContext.communityId) const recipientLastBalance = this.getLastBalanceForUser(recipientPublicKey, communityContext.blockchain, communityContext.communityId) @@ -121,9 +126,9 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { } recipientLastBalance.updateLegacyDecay(item.amount, item.balanceDate) - accountBalances.add(senderLastBalance.getAccountBalance()) - accountBalances.add(recipientLastBalance.getAccountBalance()) - return accountBalances + this.accountBalances.add(senderLastBalance.getAccountBalance()) + this.accountBalances.add(recipientLastBalance.getAccountBalance()) + return this.accountBalances } pushToBlockchain(item: TransactionDb): void { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts index 0f472a109..0134c7b6d 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts @@ -23,8 +23,14 @@ import { AbstractSyncRole, IndexType } from './AbstractSync.role' import { deriveFromCode } from '../../../../data/deriveKeyPair' import { alias } from 'drizzle-orm/mysql-core' import { toMysqlDateTime } from '../../utils' +import { Context } from '../../Context' export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { + constructor(context: Context) { + super(context) + this.accountBalances.reserve(3) + } + getDate(): Date { return this.peek().redeemedAt } @@ -88,7 +94,7 @@ export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { + constructor(context: Context) { + super(context) + this.accountBalances.reserve(1) + } + getDate(): Date { return this.peek().balanceDate } @@ -79,7 +85,7 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole senderCommunityId: string, recipientCommunityId: string, ): GradidoTransactionBuilder { - const builder = new GradidoTransactionBuilder() + return this.transactionBuilder .setCreatedAt(item.balanceDate) .addMemo(new EncryptedMemo( item.memo, @@ -94,7 +100,6 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole .setSenderCommunity(senderCommunityId) .setRecipientCommunity(recipientCommunityId) .sign(senderKeyPair) - return builder } calculateBalances( @@ -103,10 +108,10 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole amount: GradidoUnit, publicKey: MemoryBlockPtr, ): AccountBalances { - const accountBalances = new AccountBalances() + this.accountBalances.clear() if (communityContext.foreign) { - accountBalances.add(new AccountBalance(publicKey, GradidoUnit.zero(), communityContext.communityId)) - return accountBalances + this.accountBalances.add(new AccountBalance(publicKey, GradidoUnit.zero(), communityContext.communityId)) + return this.accountBalances } else { const lastBalance = this.getLastBalanceForUser(publicKey, communityContext.blockchain, communityContext.communityId) @@ -118,8 +123,8 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole throw e } } - accountBalances.add(lastBalance.getAccountBalance()) - return accountBalances + this.accountBalances.add(lastBalance.getAccountBalance()) + return this.accountBalances } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts index bc568df61..167de11ae 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts @@ -22,8 +22,13 @@ import { AbstractSyncRole, IndexType } from './AbstractSync.role' import { deriveFromCode } from '../../../../data/deriveKeyPair' import { reverseLegacyDecay, toMysqlDateTime } from '../../utils' import Decimal from 'decimal.js-light' +import { Context } from '../../Context' export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { + constructor(context: Context) { + super(context) + this.accountBalances.reserve(2) + } getDate(): Date { return this.peek().createdAt } @@ -73,7 +78,7 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { - + constructor(context: Context) { + super(context) + this.accountBalances.reserve(1) + } getDate(): Date { return this.peek().createdAt } @@ -66,7 +70,7 @@ export class UsersSyncRole extends AbstractSyncRole { accountKeyPair: KeyPairEd25519, userKeyPair: KeyPairEd25519 ): GradidoTransactionBuilder { - return new GradidoTransactionBuilder() + return this.transactionBuilder .setCreatedAt(item.createdAt) .setRegisterAddress( userKeyPair.getPublicKey(), @@ -81,9 +85,9 @@ export class UsersSyncRole extends AbstractSyncRole { } calculateAccountBalances(accountPublicKey: MemoryBlockPtr, communityContext: CommunityContext,): AccountBalances { - const accountBalances = new AccountBalances() - accountBalances.add(new AccountBalance(accountPublicKey, GradidoUnit.zero(), communityContext.communityId)) - return accountBalances + this.accountBalances.clear() + this.accountBalances.add(new AccountBalance(accountPublicKey, GradidoUnit.zero(), communityContext.communityId)) + return this.accountBalances } pushToBlockchain(item: UserDb): void { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts index 6446e2600..024836aa0 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts @@ -15,6 +15,7 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n const timeUsedDB = new Profiler() const timeUsedBlockchain = new Profiler() const timeUsedAll = new Profiler() + const timeBetweenPrints = new Profiler() const containers = [ new UsersSyncRole(context), new CreationsSyncRole(context), @@ -23,7 +24,7 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n new RedeemTransactionLinksSyncRole(context), new ContributionLinkTransactionSyncRole(context), new DeletedTransactionLinksSyncRole(context), - new RemoteTransactionsSyncRole(context), + // new RemoteTransactionsSyncRole(context), ] let transactionsCount = 0 let transactionsCountSinceLastLog = 0 @@ -54,8 +55,11 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n } available[0].toBlockchain() transactionsCount++ - if (isDebug) { - process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`) + if (isDebug) { + if (timeBetweenPrints.millis() > 100) { + process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`) + timeBetweenPrints.reset() + } transactionsCountSinceLastLog++ if (transactionsCountSinceLastLog >= batchSize) { context.logger.debug(`${transactionsCountSinceLastLog} transactions added to blockchain in ${timeUsedBlockchain.string()}`)