diff --git a/dlt-connector/bun.lock b/dlt-connector/bun.lock index 1f27138de..7ad790bf3 100644 --- a/dlt-connector/bun.lock +++ b/dlt-connector/bun.lock @@ -160,11 +160,11 @@ "@grpc/proto-loader": ["@grpc/proto-loader@0.7.15", "", { "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", "protobufjs": "^7.2.5", "yargs": "^17.7.2" }, "bin": { "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" } }, "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ=="], - "@hashgraph/cryptography": ["@hashgraph/cryptography@1.13.0", "", { "dependencies": { "@noble/curves": "1.8.1", "ansi-regex": "6.2.2", "ansi-styles": "6.2.3", "asn1js": "3.0.6", "bignumber.js": "9.1.1", "bn.js": "5.2.1", "buffer": "6.0.3", "crypto-js": "4.2.0", "debug": "4.4.1", "forge-light": "1.1.4", "js-base64": "3.7.7", "react-native-get-random-values": "1.11.0", "spark-md5": "3.0.2", "strip-ansi": "7.1.2", "tweetnacl": "1.0.3", "utf8": "3.0.0" } }, "sha512-bttkU9cnbA2NgmE41V4IDYZ5IYMY9HtVTnlvg3fdj8m83+7T4KgTBPPSkqwFCgSeW2x/6MV2GDzvaj4Lx4IYfw=="], + "@hashgraph/cryptography": ["@hashgraph/cryptography@1.14.0", "", { "dependencies": { "@noble/curves": "1.8.1", "ansi-regex": "6.2.2", "ansi-styles": "6.2.3", "asn1js": "3.0.6", "bignumber.js": "9.1.1", "bn.js": "5.2.1", "buffer": "6.0.3", "crypto-js": "4.2.0", "debug": "4.4.1", "forge-light": "1.1.4", "js-base64": "3.7.7", "react-native-get-random-values": "1.11.0", "spark-md5": "3.0.2", "strip-ansi": "7.1.2", "tweetnacl": "1.0.3", "utf8": "3.0.0" } }, "sha512-vYeRpkdYHgO0y09BJJms4DQvjkSQuoOWnf1tCaOHfs4w2u1RttQUMgQbaZCM07+6YL3Pq8SPHbcEch5tbSdBNg=="], "@hashgraph/proto": ["@hashgraph/proto@2.24.0", "", { "dependencies": { "long": "5.3.1" }, "peerDependencies": { "ansi-regex": "6.2.2", "ansi-styles": "6.2.3", "debug": "4.4.1", "protobufjs": "7.5.4", "strip-ansi": "7.1.2" } }, "sha512-S1eE0CoQ17s40JuzKaKpMze8h0JoN13za5arCp3xaqbWVT4UEFq/O/zJud9cpZ31uYwpPe8gH65TPZ3pKd9ZWQ=="], - "@hashgraph/sdk": ["@hashgraph/sdk@2.75.0", "", { "dependencies": { "@ethersproject/abi": "5.8.0", "@ethersproject/bignumber": "5.8.0", "@ethersproject/bytes": "5.8.0", "@ethersproject/rlp": "5.8.0", "@grpc/grpc-js": "1.12.6", "@hashgraph/cryptography": "1.13.0", "@hashgraph/proto": "2.24.0", "ansi-regex": "6.2.2", "ansi-styles": "6.2.3", "bignumber.js": "9.1.1", "bn.js": "5.1.1", "crypto-js": "4.2.0", "debug": "4.4.1", "js-base64": "3.7.4", "long": "5.3.1", "pino": "9.6.0", "pino-pretty": "13.0.0", "protobufjs": "7.5.4", "rfc4648": "1.5.3", "strip-ansi": "7.1.2", "utf8": "3.0.0" } }, "sha512-Nkv57So2RbNlKxog1nsyqHgorgrStr3yzx0ZWse4R6yg1TztafiqA2+9m3YlmH3eNMr7vmghgtGqPkRflfVhZg=="], + "@hashgraph/sdk": ["@hashgraph/sdk@2.76.0", "", { "dependencies": { "@ethersproject/abi": "5.8.0", "@ethersproject/bignumber": "5.8.0", "@ethersproject/bytes": "5.8.0", "@ethersproject/rlp": "5.8.0", "@grpc/grpc-js": "1.12.6", "@hashgraph/cryptography": "1.14.0", "@hashgraph/proto": "2.24.0", "ansi-regex": "6.2.2", "ansi-styles": "6.2.3", "bignumber.js": "9.1.1", "bn.js": "5.1.1", "crypto-js": "4.2.0", "debug": "4.4.1", "js-base64": "3.7.4", "long": "5.3.1", "pino": "9.6.0", "pino-pretty": "13.0.0", "protobufjs": "7.5.4", "rfc4648": "1.5.3", "strip-ansi": "7.1.2", "utf8": "3.0.0" } }, "sha512-FrLDeiHCJak+ZBcRv5cFoX23LPHvn1xWvVIST+oheI0NGfMxOXrGLWICaP5FMWdOnLfkM1OklCQ93J+QBFP7+Q=="], "@isaacs/ttlcache": ["@isaacs/ttlcache@1.4.1", "", {}, "sha512-RQgQ4uQ+pLbqXfOmieB91ejmLwvSgv9nLx6sT6sD83s7umBypgg+OIBOBbEUiJXrfpnp9j0mRhYYdzp9uqq3lA=="], diff --git a/dlt-connector/package.json b/dlt-connector/package.json index fd33612a5..b314584c6 100644 --- a/dlt-connector/package.json +++ b/dlt-connector/package.json @@ -10,7 +10,7 @@ "start": "bun run src/index.ts", "build": "bun build src/index.ts --outdir=build --target=bun --external=gradido-blockchain-js --minify", "dev": "bun run --watch src/index.ts", - "migrate": "cross-env MIMALLOC_SHOW_STATS=1 bun src/migrations/db-v2.7.0_to_blockchain-v3.5", + "migrate": "bun src/migrations/db-v2.7.0_to_blockchain-v3.5", "test": "bun test", "test:debug": "bun test --inspect-brk", "typecheck": "tsc --noEmit", diff --git a/dlt-connector/src/interactions/sendToHiero/RedeemDeferredTransferTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/RedeemDeferredTransferTransaction.role.ts index 626712404..2f51f8f21 100644 --- a/dlt-connector/src/interactions/sendToHiero/RedeemDeferredTransferTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/RedeemDeferredTransferTransaction.role.ts @@ -1,6 +1,5 @@ -import { GradidoTransactionBuilder, GradidoTransfer, TransferAmount } from 'gradido-blockchain-js' +import { ConfirmedTransaction, GradidoTransactionBuilder, GradidoTransfer, TransferAmount } from 'gradido-blockchain-js' import * as v from 'valibot' -import { GradidoNodeClient } from '../../client/GradidoNode/GradidoNodeClient' import { KeyPairIdentifierLogic } from '../../data/KeyPairIdentifier.logic' import { RedeemDeferredTransferTransaction, @@ -15,12 +14,14 @@ import { AbstractTransactionRole } from './AbstractTransaction.role' export class RedeemDeferredTransferTransactionRole extends AbstractTransactionRole { private linkedUser: UserAccount private readonly redeemDeferredTransferTransaction: RedeemDeferredTransferTransaction - constructor(transaction: Transaction) { + private readonly parentDeferredTransaction: ConfirmedTransaction + constructor(transaction: Transaction, parentDeferredTransaction: ConfirmedTransaction) { super() this.redeemDeferredTransferTransaction = v.parse( redeemDeferredTransferTransactionSchema, transaction, ) + this.parentDeferredTransaction = parentDeferredTransaction this.linkedUser = this.redeemDeferredTransferTransaction.linkedUser } @@ -41,16 +42,7 @@ export class RedeemDeferredTransferTransactionRole extends AbstractTransactionRo if (!senderPublicKey) { throw new Error("redeem deferred transfer: couldn't calculate sender public key") } - // load deferred transfer transaction from gradido node - const transactions = await GradidoNodeClient.getInstance().getTransactionsForAccount( - { maxResultCount: 2, topic: this.getSenderCommunityTopicId() }, - senderPublicKey.convertToHex(), - ) - if (!transactions || transactions.length !== 1) { - throw new Error("redeem deferred transfer: couldn't find deferred transfer on Gradido Node") - } - const deferredTransfer = transactions[0] - const deferredTransferBody = deferredTransfer.getGradidoTransaction()?.getTransactionBody() + const deferredTransferBody = this.parentDeferredTransaction.getGradidoTransaction()?.getTransactionBody() if (!deferredTransferBody) { throw new Error( "redeem deferred transfer: couldn't deserialize deferred transfer from Gradido Node", @@ -61,7 +53,7 @@ export class RedeemDeferredTransferTransactionRole extends AbstractTransactionRo builder .setCreatedAt(this.redeemDeferredTransferTransaction.createdAt) .setRedeemDeferredTransfer( - deferredTransfer.getId(), + this.parentDeferredTransaction.getId(), new GradidoTransfer( new TransferAmount( senderKeyPair.getPublicKey(), diff --git a/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts b/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts index 5b5f1f629..5ef946ec9 100644 --- a/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts +++ b/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts @@ -6,6 +6,7 @@ import { ValidateType_SINGLE, } from 'gradido-blockchain-js' import { getLogger } from 'log4js' +import { GradidoNodeClient } from '../../client/GradidoNode/GradidoNodeClient' import * as v from 'valibot' import { ensureCommunitiesAvailable } from '../../client/GradidoNode/communities' import { HieroClient } from '../../client/hiero/HieroClient' @@ -21,6 +22,7 @@ import { HieroId, HieroTransactionIdString, hieroTransactionIdStringSchema, + identifierSeedSchema, } from '../../schemas/typeGuard.schema' import { isTopicStillOpen } from '../../utils/hiero' import { AbstractTransactionRole } from './AbstractTransaction.role' @@ -30,6 +32,7 @@ import { DeferredTransferTransactionRole } from './DeferredTransferTransaction.r import { RedeemDeferredTransferTransactionRole } from './RedeemDeferredTransferTransaction.role' import { RegisterAddressTransactionRole } from './RegisterAddressTransaction.role' import { TransferTransactionRole } from './TransferTransaction.role' +import { LinkedTransactionKeyPairRole } from '../resolveKeyPair/LinkedTransactionKeyPair.role' const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.interactions.sendToHiero.SendToHieroContext`) @@ -144,7 +147,20 @@ async function chooseCorrectRole( case InputTransactionType.GRADIDO_DEFERRED_TRANSFER: return new DeferredTransferTransactionRole(transaction) case InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER: - return new RedeemDeferredTransferTransactionRole(transaction) + // load deferred transfer transaction from gradido node + const seedKeyPairRole = new LinkedTransactionKeyPairRole(v.parse(identifierSeedSchema, transaction.user.seed)) + const seedPublicKey = seedKeyPairRole.generateKeyPair().getPublicKey() + if (!seedPublicKey) { + throw new Error("redeem deferred transfer: couldn't generate seed public key") + } + const transactions = await GradidoNodeClient.getInstance().getTransactionsForAccount( + { maxResultCount: 2, topic: transaction.user.communityTopicId }, + seedPublicKey.convertToHex(), + ) + if (!transactions || transactions.length !== 1) { + throw new Error("redeem deferred transfer: couldn't find exactly one deferred transfer on Gradido Node") + } + return new RedeemDeferredTransferTransactionRole(transaction, transactions[0]) default: throw new Error('not supported transaction type: ' + transaction.type) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts index 12ba38017..2d135b184 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts @@ -4,7 +4,8 @@ import { Timestamp, HieroTransactionId, HieroAccountId, - InteractionSerialize, + InteractionSerialize, + Filter, } from 'gradido-blockchain-js' import { getLogger } from 'log4js' import { RegisterAddressTransactionRole } from '../../interactions/sendToHiero/RegisterAddressTransaction.role' @@ -16,6 +17,9 @@ import { TransferTransactionRole } from '../../interactions/sendToHiero/Transfer import { DeferredTransferTransactionRole } from '../../interactions/sendToHiero/DeferredTransferTransaction.role' import { RedeemDeferredTransferTransactionRole } from '../../interactions/sendToHiero/RedeemDeferredTransferTransaction.role' import { InputTransactionType } from '../../data/InputTransactionType.enum' +import { LinkedTransactionKeyPairRole } from '../../interactions/resolveKeyPair/LinkedTransactionKeyPair.role' +import { identifierSeedSchema } from '../../schemas/typeGuard.schema' +import * as v from 'valibot' const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`) export const defaultHieroAccount = new HieroAccountId(0, 0, 2) @@ -24,7 +28,13 @@ function addToBlockchain(builder: GradidoTransactionBuilder, blockchain: InMemor const transaction = builder.build() const transactionId = new HieroTransactionId(createdAtTimestamp, defaultHieroAccount) const interactionSerialize = new InteractionSerialize(transactionId) - return blockchain.createAndAddConfirmedTransaction(transaction, interactionSerialize.run(), createdAtTimestamp) + try { + const result = blockchain.createAndAddConfirmedTransaction(transaction, interactionSerialize.run(), createdAtTimestamp) + return result + } catch (error) { + logger.error(`Transaction ${transaction.toJson(true)} not added: ${error}`) + return false + } } export async function addCommunityRootTransaction(blockchain: InMemoryBlockchain, community: Community): Promise { @@ -57,7 +67,7 @@ export async function addTransaction( logger.debug(`Creation Transaction added for user ${transaction.user.account!.userUuid}`) } else { throw new Error(`Creation Transaction not added for user ${transaction.user.account!.userUuid}`) - } + } } else if (transaction.type === InputTransactionType.GRADIDO_TRANSFER) { const transferTransactionRole = new TransferTransactionRole(transaction) // will crash with cross group transaction @@ -71,14 +81,27 @@ export async function addTransaction( if(addToBlockchain(await transferTransactionRole.getGradidoTransactionBuilder(), senderBlockchain, createdAtTimestamp)) { logger.debug(`Deferred Transfer Transaction added for user ${transaction.user.account!.userUuid}`) } else { + throw new Error(`Deferred Transfer Transaction not added for user ${transaction.user.account!.userUuid}`) } } else if (transaction.type === InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER) { - const redeemTransactionRole = new RedeemDeferredTransferTransactionRole(transaction) + const seedKeyPairRole = new LinkedTransactionKeyPairRole(v.parse(identifierSeedSchema, transaction.user.seed)) + const f = new Filter() + f.involvedPublicKey = seedKeyPairRole.generateKeyPair().getPublicKey() + const deferredTransaction = senderBlockchain.findOne(f) + if (!deferredTransaction) { + throw new Error("redeem deferred transfer: couldn't find parent deferred transfer on Gradido Node") + } + const confirmedDeferredTransaction = deferredTransaction.getConfirmedTransaction() + if (!confirmedDeferredTransaction) { + throw new Error("redeem deferred transfer: invalid TransactionEntry") + } + const redeemTransactionRole = new RedeemDeferredTransferTransactionRole(transaction, confirmedDeferredTransaction) + const involvedUser = transaction.user.account ? transaction.user.account.userUuid : transaction.linkedUser?.account?.userUuid if(addToBlockchain(await redeemTransactionRole.getGradidoTransactionBuilder(), senderBlockchain, createdAtTimestamp)) { - logger.debug(`Redeem Deferred Transfer Transaction added for user ${transaction.user.account!.userUuid}`) + logger.debug(`Redeem Deferred Transfer Transaction added for user ${involvedUser}`) } else { - throw new Error(`Redeem Deferred Transfer Transaction not added for user ${transaction.user.account!.userUuid}`) + throw new Error(`Redeem Deferred Transfer Transaction not added for user ${involvedUser}`) } } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts index b7edfb6ec..8d66db750 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts @@ -15,6 +15,7 @@ import { Context } from './Context' import { bootstrap } from './bootstrap' import { heapStats } from 'bun:jsc' import { onShutdown } from '../../../../shared/src/helper/onShutdown' +import { sleep } from 'bun' const publicKeyUserIdMap = new Map() @@ -29,6 +30,7 @@ async function main() { } context.db.close() }) + // synchronize to blockchain const BATCH_SIZE = 100 @@ -54,11 +56,15 @@ async function main() { (transaction: TransactionDb) => transaction.balanceDate, (context: Context, transaction: TransactionDb) => pushTransaction(context, transaction) ) - - await synchronizeToBlockchain(context, [users, transactions, transactionLinks, deletedTransactionLinks], BATCH_SIZE) + try { + await synchronizeToBlockchain(context, [users, transactions, transactionLinks, deletedTransactionLinks], BATCH_SIZE) + } catch (e) { + console.error(e) + throw e + } context.logger.info(`${timeUsed.string()} for synchronizing to blockchain`) - timeUsed.reset() - context.communities.forEach((communityContext) => { + // timeUsed.reset() + /*context.communities.forEach((communityContext) => { const f = new Filter() // hotfix for bug in gradido_blockchain for Filter::ALL_TRANSACTIONS f.pagination.size = 0 @@ -67,6 +73,7 @@ async function main() { // logBlogchain(context.logger, communityContext.blockchain) }) context.logger.info(`${timeUsed.string()} for logging blockchains`) + */ const runtimeStats = heapStats() context.logger.info( `Memory Statistics: heap size: ${bytesToMbyte(runtimeStats.heapSize)} MByte, heap capacity: ${bytesToMbyte(runtimeStats.heapCapacity)} MByte, extra memory: ${bytesToMbyte(runtimeStats.extraMemorySize)} MByte` @@ -84,24 +91,33 @@ async function synchronizeToBlockchain( containers: Orderable[], batchSize: number ): Promise { - while (true) { - const timeUsed = new Profiler() - await Promise.all(containers.map(c => c.ensureFilled(context, batchSize))) - const itemCount = containers.reduce((acc, c) => acc + c.length, 0) - context.logger.info(`${timeUsed.string()} for ensuring filled containers, ${itemCount} items`) + const timeUsed = new Profiler() + while (true) { + timeUsed.reset() + const results = await Promise.all(containers.map(c => c.ensureFilled(context, batchSize))) + const loadedItemsCount = results.reduce((acc, c) => acc + c, 0) + // log only, if at least one new item was loaded + if (loadedItemsCount && context.logger.isInfoEnabled()) { + context.logger.info(`${loadedItemsCount} new items loaded from db in ${timeUsed.string()}`) + } // remove empty containers const available = containers.filter(c => !c.isEmpty()) - if (available.length === 0) break + if (available.length === 0) { + break + } // find container with smallest date - available.sort((a, b) => a.getDate().getTime() - b.getDate().getTime()) + if (available.length > 0) { + available.sort((a, b) => a.getDate().getTime() - b.getDate().getTime()) + } try { await available[0].pushToBlockchain(context) + // await sleep(1) } catch (e) { - console.error(e) - logBlogchain(context.logger, context.communities.values().next().value!.blockchain) + context.logger.error(e) + // logBlogchain(context.logger, context.communities.values().next().value!.blockchain) throw e } } @@ -130,8 +146,8 @@ async function getNextTransactions(context: Context, offset: number, count: numb const timeUsed = new Profiler() const transactions = await loadTransactions(context.db, offset, count) if(transactions.length !== 0) { - context.logger.info(`${timeUsed.string()} for loading ${transactions.length} transactions from db`) - } + context.logger.debug(`${timeUsed.string()} for loading ${transactions.length} transactions from db`) + } return transactions } @@ -140,7 +156,7 @@ async function getNextTransactionLinks(context: Context, offset: number, count: const timeUsed = new Profiler() const transactionLinks = await loadTransactionLinks(context.db, offset, count) if(transactionLinks.length !== 0) { - context.logger.info(`${timeUsed.string()} for loading ${transactionLinks.length} transaction links from db`) + context.logger.debug(`${timeUsed.string()} for loading ${transactionLinks.length} transaction links from db`) } return transactionLinks } @@ -150,7 +166,7 @@ async function getNextDeletedTransactionLinks(context: Context, offset: number, const timeUsed = new Profiler() const deletedTransactionLinks = await loadDeletedTransactionLinks(context.db, offset, count) if(deletedTransactionLinks.length !== 0) { - context.logger.info(`${timeUsed.string()} for loading ${deletedTransactionLinks.length} deleted transaction links from db`) + context.logger.debug(`${timeUsed.string()} for loading ${deletedTransactionLinks.length} deleted transaction links from db`) } return deletedTransactionLinks } @@ -165,11 +181,15 @@ async function pushRegisterAddressTransaction(context: Context, user: CreatedUse async function pushTransaction(context: Context, transactionDb: TransactionDb): Promise { const senderCommunityContext = context.getCommunityContextByUuid(transactionDb.user.communityUuid) + // context.logger.info(`before adding non register address and non link transaction:`) + // logBlogchain(context.logger, senderCommunityContext.blockchain) const recipientCommunityContext = context.getCommunityContextByUuid(transactionDb.linkedUser.communityUuid) // CreationTransactionRole will check that community topic id belongs to home community context.cache.setHomeCommunityTopicId(senderCommunityContext.topicId) const transaction = transactionDbToTransaction(transactionDb, senderCommunityContext.topicId, recipientCommunityContext.topicId) await addTransaction(senderCommunityContext.blockchain, recipientCommunityContext.blockchain, transaction) + // const firstTransaction = senderCommunityContext.blockchain.findOne(Filter.FIRST_TRANSACTION) + // console.log(`first transaction: ${firstTransaction?.getConfirmedTransaction()?.toJson(true)}`) } async function pushTransactionLink(context: Context, transactionLinkDb: TransactionLinkDb): Promise {