IT4Change/gradido
3
0
Fork 0
mirror of https://github.com/IT4Change/gradido.git synced 2026-04-06 01:25:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

reduce new calls every tx

Browse Source
This commit is contained in:
einhornimmond 2026-02-09 09:24:08 +01:00
parent 015481cee1
commit be48f6aaa2
11 changed files with 106 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts
View File

@ -27,6 +27,7 @@ export function exportCommunity(
batchSize: number,
) {
const timeUsed = new Profiler()
const timeSinceLastPrint = new Profiler()
// write as binary file for GradidoNode
const f = new Filter()
f.pagination.size = batchSize
@ -64,7 +65,10 @@ export function exportCommunity(
count++
}
if (isDebug) {
printConsole()
if (timeSinceLastPrint.millis() > 100) {
printConsole()
timeSinceLastPrint.reset()
}
} else {
printCount++
if (printCount >= 100) {

3
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts
View File

@ -2,10 +2,13 @@ import { onShutdown } from '../../../../shared/src/helper/onShutdown'
import { exportAllCommunities } from './binaryExport'
import { bootstrap } from './bootstrap'
import { syncDbWithBlockchainContext } from './interaction/syncDbWithBlockchain/syncDbWithBlockchain.context'
// import { hello } from '../../../zig/hello.zig'
const BATCH_SIZE = 1000
async function main() {
// hello()
// return
// prepare in memory blockchains
const context = await bootstrap()
onShutdown(async (reason, error) => {

15
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts
View File

@ -1,4 +1,13 @@
import { Filter, InMemoryBlockchain, KeyPairEd25519, MemoryBlockPtr, Profiler, SearchDirection_DESC } from 'gradido-blockchain-js'
import {
AccountBalances,
Filter,
InMemoryBlockchain,
KeyPairEd25519,
MemoryBlockPtr,
Profiler,
SearchDirection_DESC,
GradidoTransactionBuilder
} from 'gradido-blockchain-js'
import { getLogger, Logger } from 'log4js'
import { LOG4JS_BASE_CATEGORY } from '../../../../config/const'
import { deriveFromKeyPairAndIndex, deriveFromKeyPairAndUuid } from '../../../../data/deriveKeyPair'
@ -18,11 +27,15 @@ export abstract class AbstractSyncRole<ItemType> {
private items: ItemType[] = []
protected lastIndex: IndexType = { date: new Date(0), id: 0 }
protected logger: Logger
protected transactionBuilder: GradidoTransactionBuilder
protected accountBalances: AccountBalances
constructor(protected readonly context: Context) {
this.logger = getLogger(
`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5.interaction.syncDbWithBlockchain`,
)
this.transactionBuilder = new GradidoTransactionBuilder()
this.accountBalances = new AccountBalances()
}
getAccountKeyPair(communityContext: CommunityContext, gradidoId: Uuidv4): KeyPairEd25519 {

18
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts
View File

@ -16,6 +16,7 @@ import {
import * as v from 'valibot'
import { addToBlockchain } from '../../blockchain'
import { ContributionStatus } from '../../data/ContributionStatus'
import { Context } from '../../Context'
import {
contributionsTable,
usersTable
@ -27,6 +28,11 @@ import { toMysqlDateTime } from '../../utils'
export class CreationsSyncRole extends AbstractSyncRole<CreationTransactionDb> {
constructor(context: Context) {
super(context)
this.accountBalances.reserve(3)
}
getDate(): Date {
return this.peek().confirmedAt
}
@ -85,7 +91,7 @@ export class CreationsSyncRole extends AbstractSyncRole<CreationTransactionDb> {
recipientKeyPair: KeyPairEd25519,
signerKeyPair: KeyPairEd25519
): GradidoTransactionBuilder {
return new GradidoTransactionBuilder()
return this.transactionBuilder
.setCreatedAt(item.confirmedAt)
.addMemo(
new EncryptedMemo(
@ -107,7 +113,7 @@ export class CreationsSyncRole extends AbstractSyncRole<CreationTransactionDb> {
communityContext: CommunityContext,
recipientPublicKey: MemoryBlockPtr
): AccountBalances {
const accountBalances = new AccountBalances()
this.accountBalances.clear()
const balance = this.getLastBalanceForUser(recipientPublicKey, communityContext.blockchain, communityContext.communityId)
// calculate decay since last balance with legacy calculation method
@ -115,10 +121,10 @@ export class CreationsSyncRole extends AbstractSyncRole<CreationTransactionDb> {
communityContext.aufBalance.updateLegacyDecay(item.amount, item.confirmedAt)
communityContext.gmwBalance.updateLegacyDecay(item.amount, item.confirmedAt)
accountBalances.add(balance.getAccountBalance())
accountBalances.add(communityContext.aufBalance.getAccountBalance())
accountBalances.add(communityContext.gmwBalance.getAccountBalance())
return accountBalances
this.accountBalances.add(balance.getAccountBalance())
this.accountBalances.add(communityContext.aufBalance.getAccountBalance())
this.accountBalances.add(communityContext.gmwBalance.getAccountBalance())
return this.accountBalances
}
pushToBlockchain(item: CreationTransactionDb): void {

16
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts
View File

@ -21,8 +21,14 @@ import { addToBlockchain } from '../../blockchain'
import { BlockchainError, DatabaseError } from '../../errors'
import { Balance } from '../../data/Balance'
import { toMysqlDateTime } from '../../utils'
import { Context } from '../../Context'
export class DeletedTransactionLinksSyncRole extends AbstractSyncRole<DeletedTransactionLinkDb> {
constructor(context: Context) {
super(context)
this.accountBalances.reserve(2)
}
getDate(): Date {
return this.peek().deletedAt
}
@ -81,7 +87,7 @@ export class DeletedTransactionLinksSyncRole extends AbstractSyncRole<DeletedTra
senderKeyPair: KeyPairEd25519,
linkFundingPublicKey: MemoryBlockPtr,
): GradidoTransactionBuilder {
return new GradidoTransactionBuilder()
return this.transactionBuilder
.setCreatedAt(item.deletedAt)
.setRedeemDeferredTransfer(
linkFundingTransactionNr,
@ -101,7 +107,7 @@ export class DeletedTransactionLinksSyncRole extends AbstractSyncRole<DeletedTra
communityContext: CommunityContext,
senderPublicKey: MemoryBlockPtr,
): AccountBalances {
const accountBalances = new AccountBalances()
this.accountBalances.clear()
const fundingUserLastBalance = this.getLastBalanceForUser(
fundingTransaction.getSenderPublicKey()!,
@ -111,9 +117,9 @@ export class DeletedTransactionLinksSyncRole extends AbstractSyncRole<DeletedTra
fundingUserLastBalance.updateLegacyDecay(senderLastBalance.getBalance(), item.deletedAt)
// account of link is set to zero, gdd will be send back to initiator
accountBalances.add(new AccountBalance(senderPublicKey, GradidoUnit.zero(), communityContext.communityId))
accountBalances.add(fundingUserLastBalance.getAccountBalance())
return accountBalances
this.accountBalances.add(new AccountBalance(senderPublicKey, GradidoUnit.zero(), communityContext.communityId))
this.accountBalances.add(fundingUserLastBalance.getAccountBalance())
return this.accountBalances
}
pushToBlockchain(item: DeletedTransactionLinkDb): void {

17
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts
View File

@ -18,9 +18,14 @@ import { BlockchainError, DatabaseError, NegativeBalanceError, NotEnoughGradidoB
import { CommunityContext, TransactionDb, transactionDbSchema } from '../../valibot.schema'
import { AbstractSyncRole, IndexType } from './AbstractSync.role'
import { toMysqlDateTime } from '../../utils'
import { Context } from '../../Context'
export class LocalTransactionsSyncRole extends AbstractSyncRole<TransactionDb> {
constructor(context: Context) {
super(context)
this.accountBalances.reserve(2)
}
getDate(): Date {
return this.peek().balanceDate
}
@ -84,7 +89,7 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole<TransactionDb> {
senderKeyPair: KeyPairEd25519,
recipientKeyPair: KeyPairEd25519,
): GradidoTransactionBuilder {
return new GradidoTransactionBuilder()
return this.transactionBuilder
.setCreatedAt(item.balanceDate)
.addMemo(new EncryptedMemo(
item.memo,
@ -106,7 +111,7 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole<TransactionDb> {
senderPublicKey: MemoryBlockPtr,
recipientPublicKey: MemoryBlockPtr,
): AccountBalances {
const accountBalances = new AccountBalances()
this.accountBalances.clear()
const senderLastBalance = this.getLastBalanceForUser(senderPublicKey, communityContext.blockchain, communityContext.communityId)
const recipientLastBalance = this.getLastBalanceForUser(recipientPublicKey, communityContext.blockchain, communityContext.communityId)
@ -121,9 +126,9 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole<TransactionDb> {
}
recipientLastBalance.updateLegacyDecay(item.amount, item.balanceDate)
accountBalances.add(senderLastBalance.getAccountBalance())
accountBalances.add(recipientLastBalance.getAccountBalance())
return accountBalances
this.accountBalances.add(senderLastBalance.getAccountBalance())
this.accountBalances.add(recipientLastBalance.getAccountBalance())
return this.accountBalances
}
pushToBlockchain(item: TransactionDb): void {

18
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts
View File

@ -23,8 +23,14 @@ import { AbstractSyncRole, IndexType } from './AbstractSync.role'
import { deriveFromCode } from '../../../../data/deriveKeyPair'
import { alias } from 'drizzle-orm/mysql-core'
import { toMysqlDateTime } from '../../utils'
import { Context } from '../../Context'
export class RedeemTransactionLinksSyncRole extends AbstractSyncRole<RedeemedTransactionLinkDb> {
constructor(context: Context) {
super(context)
this.accountBalances.reserve(3)
}
getDate(): Date {
return this.peek().redeemedAt
}
@ -88,7 +94,7 @@ export class RedeemTransactionLinksSyncRole extends AbstractSyncRole<RedeemedTra
senderKeyPair: KeyPairEd25519,
recipientKeyPair: KeyPairEd25519,
): GradidoTransactionBuilder {
return new GradidoTransactionBuilder()
return this.transactionBuilder
.setCreatedAt(item.redeemedAt)
.addMemo(
new EncryptedMemo(
@ -115,7 +121,7 @@ export class RedeemTransactionLinksSyncRole extends AbstractSyncRole<RedeemedTra
senderPublicKey: MemoryBlockPtr,
recipientPublicKey: MemoryBlockPtr,
): AccountBalances {
const accountBalances = new AccountBalances()
this.accountBalances.clear()
const senderLastBalance = this.getLastBalanceForUser(
senderPublicKey,
@ -141,10 +147,10 @@ export class RedeemTransactionLinksSyncRole extends AbstractSyncRole<RedeemedTra
recipientLastBalance.updateLegacyDecay(item.amount, item.redeemedAt)
// account of link is set to zero, and change send back to link creator
accountBalances.add(new AccountBalance(senderPublicKey, GradidoUnit.zero(), communityContext.communityId))
accountBalances.add(recipientLastBalance.getAccountBalance())
accountBalances.add(fundingUserLastBalance.getAccountBalance())
return accountBalances
this.accountBalances.add(new AccountBalance(senderPublicKey, GradidoUnit.zero(), communityContext.communityId))
this.accountBalances.add(recipientLastBalance.getAccountBalance())
this.accountBalances.add(fundingUserLastBalance.getAccountBalance())
return this.accountBalances
}
pushToBlockchain(item: RedeemedTransactionLinkDb): void {

19
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts
View File

@ -12,8 +12,14 @@ import { BlockchainError } from '../../errors'
import { addToBlockchain } from '../../blockchain'
import { AccountBalance, AccountBalances, AuthenticatedEncryption, EncryptedMemo, GradidoTransactionBuilder, GradidoUnit, KeyPairEd25519, LedgerAnchor, MemoryBlockPtr, TransferAmount } from 'gradido-blockchain-js'
import { Decimal } from 'decimal.js'
import { Context } from '../../Context'
export class RemoteTransactionsSyncRole extends AbstractSyncRole<TransactionDb> {
constructor(context: Context) {
super(context)
this.accountBalances.reserve(1)
}
getDate(): Date {
return this.peek().balanceDate
}
@ -79,7 +85,7 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole<TransactionDb>
senderCommunityId: string,
recipientCommunityId: string,
): GradidoTransactionBuilder {
const builder = new GradidoTransactionBuilder()
return this.transactionBuilder
.setCreatedAt(item.balanceDate)
.addMemo(new EncryptedMemo(
item.memo,
@ -94,7 +100,6 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole<TransactionDb>
.setSenderCommunity(senderCommunityId)
.setRecipientCommunity(recipientCommunityId)
.sign(senderKeyPair)
return builder
}
calculateBalances(
@ -103,10 +108,10 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole<TransactionDb>
amount: GradidoUnit,
publicKey: MemoryBlockPtr,
): AccountBalances {
const accountBalances = new AccountBalances()
this.accountBalances.clear()
if (communityContext.foreign) {
accountBalances.add(new AccountBalance(publicKey, GradidoUnit.zero(), communityContext.communityId))
return accountBalances
this.accountBalances.add(new AccountBalance(publicKey, GradidoUnit.zero(), communityContext.communityId))
return this.accountBalances
} else {
const lastBalance = this.getLastBalanceForUser(publicKey, communityContext.blockchain, communityContext.communityId)
@ -118,8 +123,8 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole<TransactionDb>
throw e
}
}
accountBalances.add(lastBalance.getAccountBalance())
return accountBalances
this.accountBalances.add(lastBalance.getAccountBalance())
return this.accountBalances
}
}

15
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts
View File

@ -22,8 +22,13 @@ import { AbstractSyncRole, IndexType } from './AbstractSync.role'
import { deriveFromCode } from '../../../../data/deriveKeyPair'
import { reverseLegacyDecay, toMysqlDateTime } from '../../utils'
import Decimal from 'decimal.js-light'
import { Context } from '../../Context'
export class TransactionLinkFundingsSyncRole extends AbstractSyncRole<TransactionLinkDb> {
constructor(context: Context) {
super(context)
this.accountBalances.reserve(2)
}
getDate(): Date {
return this.peek().createdAt
}
@ -73,7 +78,7 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole<Transactio
senderKeyPair: KeyPairEd25519,
recipientKeyPair: KeyPairEd25519,
): GradidoTransactionBuilder {
return new GradidoTransactionBuilder()
return this.transactionBuilder
.setCreatedAt(item.createdAt)
.addMemo(
new EncryptedMemo(
@ -100,7 +105,7 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole<Transactio
senderPublicKey: MemoryBlockPtr,
recipientPublicKey: MemoryBlockPtr,
): AccountBalances {
const accountBalances = new AccountBalances()
this.accountBalances.clear()
let senderLastBalance = this.getLastBalanceForUser(senderPublicKey, communityContext.blockchain, communityContext.communityId)
try {
senderLastBalance.updateLegacyDecay(blockedAmount.negated(), item.createdAt)
@ -112,9 +117,9 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole<Transactio
}
}
accountBalances.add(senderLastBalance.getAccountBalance())
accountBalances.add(new AccountBalance(recipientPublicKey, blockedAmount, communityContext.communityId))
return accountBalances
this.accountBalances.add(senderLastBalance.getAccountBalance())
this.accountBalances.add(new AccountBalance(recipientPublicKey, blockedAmount, communityContext.communityId))
return this.accountBalances
}
pushToBlockchain(item: TransactionLinkDb): void {

14
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/UsersSync.role.ts
View File

@ -18,9 +18,13 @@ import { BlockchainError, DatabaseError } from '../../errors'
import { CommunityContext, UserDb, userDbSchema } from '../../valibot.schema'
import { AbstractSyncRole, IndexType } from './AbstractSync.role'
import { toMysqlDateTime } from '../../utils'
import { Context } from '../../Context'
export class UsersSyncRole extends AbstractSyncRole<UserDb> {
constructor(context: Context) {
super(context)
this.accountBalances.reserve(1)
}
getDate(): Date {
return this.peek().createdAt
}
@ -66,7 +70,7 @@ export class UsersSyncRole extends AbstractSyncRole<UserDb> {
accountKeyPair: KeyPairEd25519,
userKeyPair: KeyPairEd25519
): GradidoTransactionBuilder {
return new GradidoTransactionBuilder()
return this.transactionBuilder
.setCreatedAt(item.createdAt)
.setRegisterAddress(
userKeyPair.getPublicKey(),
@ -81,9 +85,9 @@ export class UsersSyncRole extends AbstractSyncRole<UserDb> {
}
calculateAccountBalances(accountPublicKey: MemoryBlockPtr, communityContext: CommunityContext,): AccountBalances {
const accountBalances = new AccountBalances()
accountBalances.add(new AccountBalance(accountPublicKey, GradidoUnit.zero(), communityContext.communityId))
return accountBalances
this.accountBalances.clear()
this.accountBalances.add(new AccountBalance(accountPublicKey, GradidoUnit.zero(), communityContext.communityId))
return this.accountBalances
}
pushToBlockchain(item: UserDb): void {

10
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts
View File

@ -15,6 +15,7 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n
const timeUsedDB = new Profiler()
const timeUsedBlockchain = new Profiler()
const timeUsedAll = new Profiler()
const timeBetweenPrints = new Profiler()
const containers = [
new UsersSyncRole(context),
new CreationsSyncRole(context),
@ -23,7 +24,7 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n
new RedeemTransactionLinksSyncRole(context),
new ContributionLinkTransactionSyncRole(context),
new DeletedTransactionLinksSyncRole(context),
new RemoteTransactionsSyncRole(context),
// new RemoteTransactionsSyncRole(context),
]
let transactionsCount = 0
let transactionsCountSinceLastLog = 0
@ -54,8 +55,11 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n
}
available[0].toBlockchain()
transactionsCount++
if (isDebug) {
process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`)
if (isDebug) {
if (timeBetweenPrints.millis() > 100) {
process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`)
timeBetweenPrints.reset()
}
transactionsCountSinceLastLog++
if (transactionsCountSinceLastLog >= batchSize) {
context.logger.debug(`${transactionsCountSinceLastLog} transactions added to blockchain in ${timeUsedBlockchain.string()}`)