From 6d781793919580fb7c6224a5a93807e1f1e11045 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Mon, 29 Dec 2025 19:54:17 +0100 Subject: [PATCH] refactor migration code, fix holdAvailable Amount from transaction links on the fly --- dlt-connector/bun.lock | 4 - dlt-connector/package.json | 1 - .../sendToHiero/SendToHiero.context.ts | 9 +- .../binaryExport.ts | 39 ++- .../blockchain.ts | 111 +----- .../db-v2.7.0_to_blockchain-v3.5/convert.ts | 136 -------- .../db-v2.7.0_to_blockchain-v3.5/database.ts | 324 +----------------- .../drizzle.schema.ts | 9 +- .../db-v2.7.0_to_blockchain-v3.5/errors.ts | 3 +- .../db-v2.7.0_to_blockchain-v3.5/index.ts | 4 +- .../accountBalances/AbstractBalances.role.ts | 38 -- .../accountBalances/CreationBalances.role.ts | 33 -- .../DeferredTransferBalances.role.ts | 33 -- .../RedeemDeferredTransferBalances.role.ts | 30 -- .../RegisterAddressBalances.role.ts | 17 - .../accountBalances/TransferBalances.role.ts | 30 -- .../accountBalances.context.ts | 32 -- .../syncDbWithBlockchain/AbstractSync.role.ts | 43 ++- .../ContributionLinkTransactionSync.role.ts | 14 +- .../CreationsSync.role.ts | 31 +- .../DeletedTransactionLinksSync.role.ts | 22 +- .../DoubleLinkedTransactions.role.ts | 27 -- ...InvalidContributionTransactionSync.role.ts | 27 -- .../LocalTransactionsSync.role.ts | 23 +- .../RedeemTransactionLinksSync.role.ts | 22 +- .../TransactionLinkFundingsSync.role.ts | 69 +++- .../syncDbWithBlockchain/UsersSync.role.ts | 20 +- .../syncDbWithBlockchain.context.ts | 39 ++- .../db-v2.7.0_to_blockchain-v3.5/utils.ts | 28 +- .../valibot.schema.ts | 2 + 30 files changed, 281 insertions(+), 939 deletions(-) delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/convert.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/AbstractBalances.role.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/CreationBalances.role.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/DeferredTransferBalances.role.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RedeemDeferredTransferBalances.role.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RegisterAddressBalances.role.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/TransferBalances.role.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/accountBalances.context.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DoubleLinkedTransactions.role.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/InvalidContributionTransactionSync.role.ts diff --git a/dlt-connector/bun.lock b/dlt-connector/bun.lock index 477af78c0..5337098a9 100644 --- a/dlt-connector/bun.lock +++ b/dlt-connector/bun.lock @@ -1,6 +1,5 @@ { "lockfileVersion": 1, - "configVersion": 0, "workspaces": { "": { "name": "dlt-connector", @@ -23,7 +22,6 @@ "decimal.js-light": "^2.5.1", "dotenv": "^10.0.0", "drizzle-orm": "^0.44.7", - "drizzle-valibot": "^0.4.2", "elysia": "1.3.8", "graphql-request": "^7.2.0", "jose": "^5.2.2", @@ -467,8 +465,6 @@ "drizzle-orm": ["drizzle-orm@0.44.7", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@prisma/client": "*", "@tidbcloud/serverless": "*", "@types/better-sqlite3": "*", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=7", "bun-types": "*", "expo-sqlite": ">=14.0.0", "gel": ">=2", "knex": "*", "kysely": "*", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@tidbcloud/serverless", "@types/better-sqlite3", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "gel", "knex", "kysely", "mysql2", "pg", "postgres", "sql.js", "sqlite3"] }, "sha512-quIpnYznjU9lHshEOAYLoZ9s3jweleHlZIAWR/jX9gAWNg/JhQ1wj0KGRf7/Zm+obRrYd9GjPVJg790QY9N5AQ=="], - "drizzle-valibot": ["drizzle-valibot@0.4.2", "", { "peerDependencies": { "drizzle-orm": ">=0.36.0", "valibot": ">=1.0.0-beta.7" } }, "sha512-tzjT7g0Di/HX7426marIy8IDtWODjPgrwvgscdevLQRUe5rzYzRhx6bDsYLdDFF9VI/eaYgnjNeF/fznWJoUjg=="], - "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="], diff --git a/dlt-connector/package.json b/dlt-connector/package.json index 1f4984b0b..e173d6ede 100644 --- a/dlt-connector/package.json +++ b/dlt-connector/package.json @@ -36,7 +36,6 @@ "decimal.js-light": "^2.5.1", "dotenv": "^10.0.0", "drizzle-orm": "^0.44.7", - "drizzle-valibot": "^0.4.2", "elysia": "1.3.8", "graphql-request": "^7.2.0", "jose": "^5.2.2", diff --git a/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts b/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts index 622addb28..f96bfe402 100644 --- a/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts +++ b/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts @@ -1,8 +1,8 @@ import { GradidoTransaction, HieroTransactionId, - InteractionSerialize, InteractionValidate, + LedgerAnchor, ValidateType_SINGLE, } from 'gradido-blockchain-js' import { getLogger } from 'log4js' @@ -61,11 +61,8 @@ export async function SendToHieroContext( role.getSenderCommunityTopicId(), ) - // serialize Hiero transaction ID and attach it to the builder for the inbound transaction - const transactionIdSerializer = new InteractionSerialize( - new HieroTransactionId(outboundHieroTransactionIdString), - ) - builder.setParentMessageId(transactionIdSerializer.run()) + // attach Hiero transaction ID to the builder for the inbound transaction + builder.setParentLedgerAnchor(new LedgerAnchor(new HieroTransactionId(outboundHieroTransactionIdString))) // build and validate inbound transaction const inboundTransaction = builder.buildInbound() diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/binaryExport.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/binaryExport.ts index 43459372b..9d34f5588 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/binaryExport.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/binaryExport.ts @@ -9,12 +9,13 @@ import { } from 'gradido-blockchain-js' import { CONFIG } from '../../config' import { Context } from './Context' -import { bytesToKbyte, calculateOneHashStep } from './utils' +import { bytesString, calculateOneHashStep } from './utils' import { CommunityContext } from './valibot.schema' export function exportAllCommunities(context: Context, batchSize: number) { const timeUsed = new Profiler() for (const communityContext of context.communities.values()) { + context.logger.info(`exporting community ${communityContext.communityId} to binary file`) exportCommunity(communityContext, context, batchSize) } context.logger.info(`time used for exporting communities to binary file: ${timeUsed.string()}`) @@ -25,36 +26,66 @@ export function exportCommunity( context: Context, batchSize: number, ) { + const timeUsed = new Profiler() // write as binary file for GradidoNode const f = new Filter() f.pagination.size = batchSize f.pagination.page = 1 f.searchDirection = SearchDirection_ASC const binFilePath = prepareFolder(communityContext) + let count = 0 + let printCount = 0 let lastTransactionCount = 0 + let triggeredTransactionsCount = 0 let hash = Buffer.alloc(32, 0) + const isDebug = context.logger.isDebugEnabled() + const printConsole = () => { + if (triggeredTransactionsCount > 0) { + process.stdout.write(`exported ${count} transactions + ${triggeredTransactionsCount} triggered from timeouted transaction links\r`) + } else { + process.stdout.write(`exported ${count} transactions\r`) + } + } do { const transactions = communityContext.blockchain.findAll(f) lastTransactionCount = transactions.size() for (let i = 0; i < lastTransactionCount; i++) { const confirmedTransaction = transactions.get(i)?.getConfirmedTransaction() - const transactionNr = f.pagination.page * batchSize + i + const transactionNr = (f.pagination.page - 2) * batchSize + i if (!confirmedTransaction) { throw new Error(`invalid TransactionEntry at index: ${transactionNr} `) } hash = exportTransaction(confirmedTransaction, hash, binFilePath) + if (confirmedTransaction?.getGradidoTransaction()?.getTransactionBody()?.isTimeoutDeferredTransfer()) { + triggeredTransactionsCount++ + } else { + count++ + } + if (isDebug) { + printConsole() + } else { + printCount++ + if (printCount >= 100) { + printConsole() + printCount = 0 + } + } } - f.pagination.page++ + f.pagination.page++ } while (lastTransactionCount === batchSize) + printConsole() + process.stdout.write(`\n`) fs.appendFileSync(binFilePath, hash!) context.logger.info( `binary file for community ${communityContext.communityId} written to ${binFilePath}`, ) + const sumTransactionsCount = ((f.pagination.page - 2) * batchSize) + lastTransactionCount + const fileSize = fs.statSync(binFilePath).size context.logger.info( - `transactions count: ${(f.pagination.page - 1) * batchSize + lastTransactionCount}, size: ${bytesToKbyte(fs.statSync(binFilePath).size)} KByte`, + `exported ${sumTransactionsCount} transactions (${bytesString(fileSize)}) in ${timeUsed.string()}`, ) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts index 1d473a1b2..d689f22e1 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts @@ -1,36 +1,20 @@ -import * as fs from 'node:fs' import { AccountBalances, Filter, GradidoTransactionBuilder, HieroAccountId, InMemoryBlockchain, - InteractionSerialize, - TransactionType_DEFERRED_TRANSFER, } from 'gradido-blockchain-js' import { getLogger } from 'log4js' -import * as v from 'valibot' import { LOG4JS_BASE_CATEGORY } from '../../config/const' -import { InputTransactionType } from '../../data/InputTransactionType.enum' -import { LinkedTransactionKeyPairRole } from '../../interactions/resolveKeyPair/LinkedTransactionKeyPair.role' -import { AbstractTransactionRole } from '../../interactions/sendToHiero/AbstractTransaction.role' import { CommunityRootTransactionRole } from '../../interactions/sendToHiero/CommunityRootTransaction.role' -import { CreationTransactionRole } from '../../interactions/sendToHiero/CreationTransaction.role' -import { DeferredTransferTransactionRole } from '../../interactions/sendToHiero/DeferredTransferTransaction.role' -import { RedeemDeferredTransferTransactionRole } from '../../interactions/sendToHiero/RedeemDeferredTransferTransaction.role' -import { RegisterAddressTransactionRole } from '../../interactions/sendToHiero/RegisterAddressTransaction.role' -import { TransferTransactionRole } from '../../interactions/sendToHiero/TransferTransaction.role' -import { Community, Transaction } from '../../schemas/transaction.schema' -import { identifierSeedSchema } from '../../schemas/typeGuard.schema' +import { Community } from '../../schemas/transaction.schema' import { NotEnoughGradidoBalanceError } from './errors' const logger = getLogger( `${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`, ) export const defaultHieroAccount = new HieroAccountId(0, 0, 2) -let transactionAddedToBlockchainSum = 0 -let addToBlockchainSum = 0 -const sizeBuffer = Buffer.alloc(2) export function addToBlockchain( builder: GradidoTransactionBuilder, @@ -39,30 +23,12 @@ export function addToBlockchain( accountBalances: AccountBalances, ): boolean { const transaction = builder.build() - const transactionSerializer = new InteractionSerialize(transaction) - const binTransaction = transactionSerializer.run() - if (!binTransaction) { - logger.error(`Failed to serialize transaction ${transaction.toJson(true)}`) - return false - } - const filePath = `${blockchain.getCommunityId()}.bin` - if (!addToBlockchainSum) { - // clear file - fs.writeFileSync(filePath, Buffer.alloc(0)) - } - sizeBuffer.writeUInt16LE(binTransaction.size(), 0) - fs.appendFileSync(filePath, sizeBuffer) - fs.appendFileSync(filePath, binTransaction.data()) - //*/ - try { const result = blockchain.createAndAddConfirmedTransactionExtern( transaction, transactionId, accountBalances, ) - // logger.info(`${transactionTypeToString(transaction.getTransactionBody()?.getTransactionType()!)} Transaction added in ${timeUsed.string()}`) - addToBlockchainSum++ return result } catch (error) { if (error instanceof Error) { @@ -98,78 +64,3 @@ export async function addCommunityRootTransaction( } } -export async function addTransaction( - senderBlockchain: InMemoryBlockchain, - _recipientBlockchain: InMemoryBlockchain, - transaction: Transaction, - transactionId: number, - accountBalances: AccountBalances, -): Promise { - - let debugTmpStr = '' - let role: AbstractTransactionRole - if (transaction.type === InputTransactionType.GRADIDO_CREATION) { - role = new CreationTransactionRole(transaction) - } else if (transaction.type === InputTransactionType.GRADIDO_TRANSFER) { - role = new TransferTransactionRole(transaction) - } else if (transaction.type === InputTransactionType.REGISTER_ADDRESS) { - role = new RegisterAddressTransactionRole(transaction) - } else if (transaction.type === InputTransactionType.GRADIDO_DEFERRED_TRANSFER) { - role = new DeferredTransferTransactionRole(transaction) - } else if (transaction.type === InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER) { - const seedKeyPairRole = new LinkedTransactionKeyPairRole( - v.parse(identifierSeedSchema, transaction.user.seed), - ) - const f = new Filter() - f.involvedPublicKey = seedKeyPairRole.generateKeyPair().getPublicKey() - f.transactionType = TransactionType_DEFERRED_TRANSFER - const deferredTransactions = senderBlockchain.findAll(f) - if (!deferredTransactions) { - throw new Error( - `redeem deferred transfer: couldn't find parent deferred transfer on Gradido Node for ${JSON.stringify(transaction, null, 2)} and public key from seed: ${f.involvedPublicKey?.convertToHex()}`, - ) - } - if (deferredTransactions.size() !== 1) { - logger.error( - `redeem deferred transfer: found ${deferredTransactions.size()} parent deferred transfer on Gradido Node for ${JSON.stringify(transaction, null, 2)} and public key from seed: ${f.involvedPublicKey?.convertToHex()}`, - ) - for(let i = 0; i < deferredTransactions.size(); i++) { - logger.error(`deferred transaction ${i}: ${deferredTransactions.get(i)?.getConfirmedTransaction()?.toJson(true)}`) - } - throw new Error( - `redeem deferred transfer: found ${deferredTransactions.size()} parent deferred transfer on Gradido Node for ${JSON.stringify(transaction, null, 2)} and public key from seed: ${f.involvedPublicKey?.convertToHex()}`, - ) - } - const deferredTransaction = deferredTransactions.get(0)! - const confirmedDeferredTransaction = deferredTransaction.getConfirmedTransaction() - if (!confirmedDeferredTransaction) { - throw new Error('redeem deferred transfer: invalid TransactionEntry') - } - debugTmpStr += `\nconfirmed deferred transaction: ${confirmedDeferredTransaction.toJson(true)} with filter: ${f.toJson(true)}` - role = new RedeemDeferredTransferTransactionRole( - transaction, - confirmedDeferredTransaction, - ) - } else { - throw new Error(`Transaction type ${transaction.type} not supported`) - } - const involvedUser = transaction.user.account - ? transaction.user.account.userUuid - : transaction.linkedUser?.account?.userUuid - try { - if (addToBlockchain(await role.getGradidoTransactionBuilder(), senderBlockchain, transactionId, accountBalances)) { - // logger.debug(`${transaction.type} Transaction added for user ${involvedUser}`) - transactionAddedToBlockchainSum++ - } else { - logger.error(debugTmpStr) - logger.error(`transaction: ${JSON.stringify(transaction, null, 2)}`) - throw new Error(`${transaction.type} Transaction not added for user ${involvedUser}, after ${transactionAddedToBlockchainSum} transactions`) - } - } catch(e) { - if (e instanceof NotEnoughGradidoBalanceError) { - throw e - } - logger.error(`error adding transaction: ${JSON.stringify(transaction, null, 2)}`) - throw e - } -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/convert.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/convert.ts deleted file mode 100644 index d347bd8ff..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/convert.ts +++ /dev/null @@ -1,136 +0,0 @@ -import * as v from 'valibot' -import { AccountType } from '../../data/AccountType.enum' -import { InputTransactionType } from '../../data/InputTransactionType.enum' -import { - Community, - communitySchema, - Transaction, - TransactionInput, - transactionSchema, -} from '../../schemas/transaction.schema' -import { - gradidoAmountSchema, - HieroId, - memoSchema, - timeoutDurationSchema, -} from '../../schemas/typeGuard.schema' -import { TransactionTypeId } from './data/TransactionTypeId' -import { CommunityDb, CreatedUserDb, TransactionDb, TransactionLinkDb } from './valibot.schema' - -export function getInputTransactionTypeFromTypeId(typeId: TransactionTypeId): InputTransactionType { - switch (typeId) { - case TransactionTypeId.CREATION: - return InputTransactionType.GRADIDO_CREATION - case TransactionTypeId.SEND: - return InputTransactionType.GRADIDO_TRANSFER - case TransactionTypeId.RECEIVE: - throw new Error('not used') - default: - throw new Error('not implemented') - } -} - -export function communityDbToCommunity( - topicId: HieroId, - communityDb: CommunityDb, - creationDate: Date, -): Community { - return v.parse(communitySchema, { - hieroTopicId: topicId, - uuid: communityDb.communityUuid, - foreign: communityDb.foreign, - creationDate, - }) -} - -export function userDbToTransaction(userDb: CreatedUserDb, communityTopicId: HieroId): Transaction { - return v.parse(transactionSchema, { - user: { - communityTopicId: communityTopicId, - account: { userUuid: userDb.gradidoId }, - }, - type: InputTransactionType.REGISTER_ADDRESS, - accountType: AccountType.COMMUNITY_HUMAN, - createdAt: userDb.createdAt, - }) -} - -export function transactionDbToTransaction( - transactionDb: TransactionDb, - communityTopicId: HieroId, - recipientCommunityTopicId: HieroId, -): Transaction { - if ( - transactionDb.typeId !== TransactionTypeId.CREATION && - transactionDb.typeId !== TransactionTypeId.SEND && - transactionDb.typeId !== TransactionTypeId.RECEIVE - ) { - throw new Error('not implemented') - } - - const user = { - communityTopicId: communityTopicId, - account: { userUuid: transactionDb.user.gradidoId }, - } - const linkedUser = { - communityTopicId: recipientCommunityTopicId, - account: { userUuid: transactionDb.linkedUser.gradidoId }, - } - const transaction: TransactionInput = { - user, - linkedUser, - amount: v.parse(gradidoAmountSchema, transactionDb.amount), - memo: v.parse(memoSchema, transactionDb.memo), - type: InputTransactionType.GRADIDO_TRANSFER, - createdAt: transactionDb.balanceDate, - } - if (transactionDb.typeId === TransactionTypeId.CREATION) { - if (!transactionDb.creationDate) { - throw new Error('contribution transaction without creation date') - } - transaction.targetDate = transactionDb.creationDate - transaction.type = InputTransactionType.GRADIDO_CREATION - } else if (transactionDb.typeId === TransactionTypeId.RECEIVE) { - transaction.user = linkedUser - transaction.linkedUser = user - } - if (transactionDb.transactionLinkCode) { - if (transactionDb.typeId !== TransactionTypeId.RECEIVE) { - throw new Error( - "linked transaction which isn't receive, send will taken care of on link creation", - ) - } - transaction.user = { - communityTopicId: recipientCommunityTopicId, - seed: transactionDb.transactionLinkCode, - } - transaction.type = InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER - } - return v.parse(transactionSchema, transaction) -} - -export function transactionLinkDbToTransaction( - transactionLinkDb: TransactionLinkDb, - communityTopicId: HieroId, -): Transaction { - return v.parse(transactionSchema, { - user: { - communityTopicId: communityTopicId, - account: { userUuid: transactionLinkDb.user.gradidoId }, - }, - linkedUser: { - communityTopicId: communityTopicId, - seed: transactionLinkDb.code, - }, - type: InputTransactionType.GRADIDO_DEFERRED_TRANSFER, - amount: v.parse(gradidoAmountSchema, transactionLinkDb.amount), - memo: v.parse(memoSchema, transactionLinkDb.memo), - createdAt: transactionLinkDb.createdAt, - timeoutDuration: v.parse( - timeoutDurationSchema, - Math.round( - (transactionLinkDb.validUntil.getTime() - transactionLinkDb.createdAt.getTime()) / 1000, - ), - ), - }) -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts index a4a81b826..47b7372a2 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts @@ -1,36 +1,19 @@ -import { and, asc, count, eq, gt, inArray, isNotNull, isNull, lt, sql } from 'drizzle-orm' -import { alias } from 'drizzle-orm/mysql-core' +import { asc, eq, isNotNull, sql } from 'drizzle-orm' import { MySql2Database } from 'drizzle-orm/mysql2' import { getLogger } from 'log4js' import * as v from 'valibot' import { LOG4JS_BASE_CATEGORY } from '../../config/const' -import { ContributionStatus } from './data/ContributionStatus' -import { TransactionTypeId } from './data/TransactionTypeId' import { communitiesTable, - contributionsTable, eventsTable, - TransactionSelect, - transactionLinksTable, - transactionSelectSchema, - transactionsTable, - UserSelect, userRolesTable, - userSelectSchema, usersTable } from './drizzle.schema' -import { DatabaseError } from './errors' import { CommunityDb, UserDb, - CreationTransactionDb, communityDbSchema, userDbSchema, - creationTransactionDbSchema, - TransactionDb, - TransactionLinkDb, - transactionDbSchema, - transactionLinkDbSchema, } from './valibot.schema' const logger = getLogger( @@ -93,21 +76,6 @@ export async function loadCommunities(db: MySql2Database): Promise { - const result = await db - .select() - .from(usersTable) - .orderBy(asc(usersTable.createdAt), asc(usersTable.id)) - .limit(count) - .offset(offset) - - return result.map((row: any) => v.parse(userDbSchema, row)) -} - export async function loadUserByGradidoId(db: MySql2Database, gradidoId: string): Promise { const result = await db .select() @@ -118,293 +86,3 @@ export async function loadUserByGradidoId(db: MySql2Database, gradidoId: string) return result.length ? v.parse(userDbSchema, result[0]) : null } -export async function loadLocalTransferTransactions( - db: MySql2Database, - offset: number, - count: number, -): Promise { - const linkedUsers = alias(usersTable, 'linkedUser') - const result = await db - .select({ - transaction: transactionsTable, - user: usersTable, - linkedUser: linkedUsers, - }) - .from(transactionsTable) - .where( - and( - eq(transactionsTable.typeId, TransactionTypeId.RECEIVE), - isNull(transactionsTable.transactionLinkId), - isNotNull(transactionsTable.linkedUserId), - eq(usersTable.foreign, 0), - eq(linkedUsers.foreign, 0), - ) - ) - .leftJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) - .leftJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id)) - .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) - .limit(count) - .offset(offset) - - return result.map((row: any) => { - const item = { - ...row.transaction, - user: row.user, - linkedUser: row.linkedUser, - } - try { - return v.parse(transactionDbSchema, item) - } catch (e) { - throw new DatabaseError('loadLocalTransferTransactions', item, e as Error) - } - }) -} - -export async function loadTransactions( - db: MySql2Database, - offset: number, - count: number, -): Promise { - const linkedUsers = alias(usersTable, 'linkedUser') - - const result = await db - .select({ - transaction: transactionsTable, - user: usersTable, - linkedUser: linkedUsers, - transactionLink: { - id: transactionLinksTable.id, - code: transactionLinksTable.code - }, - }) - .from(transactionsTable) - .where( - and( - inArray(transactionsTable.typeId, [TransactionTypeId.CREATION, TransactionTypeId.RECEIVE]), - isNotNull(transactionsTable.linkedUserId), - eq(usersTable.foreign, 0) - ) - ) - .leftJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) - .leftJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id)) - .leftJoin( - transactionLinksTable, - eq(transactionsTable.transactionLinkId, transactionLinksTable.id), - ) - .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) - .limit(count) - .offset(offset) - - return result.map((row: any) => { - // console.log(row) - try { - /*if (transactionIdSet.has(row.transaction.id)) { - throw new Error(`transaction ${row.transaction.id} already loaded`) - } - transactionIdSet.add(row.transaction.id) - */ - return v.parse(transactionDbSchema, { - ...row.transaction, - transactionLinkCode: row.transactionLink ? row.transactionLink.code : null, - user: row.user, - linkedUser: row.linkedUser, - }) - } catch (e) { - logger.error(`table row: ${JSON.stringify(row, null, 2)}`) - if (e instanceof v.ValiError) { - logger.error(v.flatten(e.issues)) - } - throw e - } - }) -} - -export async function loadCreations( - db: MySql2Database, - offset: number, - count: number, -): Promise { - const confirmedByUsers = alias(usersTable, 'confirmedByUser') - const result = await db - .select({ - contribution: contributionsTable, - user: usersTable, - confirmedByUser: confirmedByUsers, - }) - .from(contributionsTable) - .where(and( - isNull(contributionsTable.contributionLinkId), - eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED), - )) - .leftJoin(usersTable, eq(contributionsTable.userId, usersTable.id)) - .leftJoin(confirmedByUsers, eq(contributionsTable.confirmedBy, confirmedByUsers.id)) - .orderBy(asc(contributionsTable.confirmedAt), asc(contributionsTable.id)) - .limit(count) - .offset(offset) - - return result.map((row) => { - const creationTransactionDb = { - ...row.contribution, - user: row.user, - confirmedByUser: row.confirmedByUser, - } - try { - return v.parse(creationTransactionDbSchema, creationTransactionDb) - } catch (e) { - throw new DatabaseError('loadCreations', creationTransactionDb, e as Error) - } - }) -} - -export async function loadInvalidContributionTransactions( - db: MySql2Database, - offset: number, - count: number, -): Promise<{ id: number, balanceDate: Date }[]> { - const result = await db - .select({ - id: transactionsTable.id, - balanceDate: transactionsTable.balanceDate, - }) - .from(transactionsTable) - .where( - and( - eq(transactionsTable.typeId, TransactionTypeId.CREATION), - sql`NOT EXISTS (SELECT 1 FROM contributions WHERE contributions.transaction_id = transactions.id)`, - ) - ) - .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) - .limit(count) - .offset(offset) - - return result.map((row: any) => { - return { - id: row.id, - balanceDate: new Date(row.balanceDate), - } - }) -} - -export async function loadDoubleLinkedTransactions( - db: MySql2Database, - offset: number, - rowsCount: number, -): Promise<{ id: number, balanceDate: Date }[]> { - const result = await db - .select({ - id: transactionsTable.id, - balanceDate: transactionsTable.balanceDate, - transactionLinkId: transactionsTable.transactionLinkId, - cnt: count(), - }) - .from(transactionsTable) - .where( - and( - eq(transactionsTable.typeId, TransactionTypeId.RECEIVE), - isNotNull(transactionsTable.transactionLinkId), - ) - ) - .groupBy(transactionsTable.transactionLinkId) - .having(gt(count(), 1)) - .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) - .limit(rowsCount) - .offset(offset) - - // logger.info(`loadDoubleLinkedTransactions ${result.length}: ${timeUsed.string()}`) - - return result.map((row: any) => { - return { - id: row.transactionLinkId, - balanceDate: new Date(row.balanceDate), - } - }) -} - -export async function loadContributionLinkTransactions( - db: MySql2Database, - offset: number, - count: number, -): Promise<{ transaction: TransactionSelect, user: UserSelect, contributionLinkId: number }[]> { - const result = await db - .select({ - transaction: transactionsTable, - user: usersTable, - contributionLinkId: contributionsTable.contributionLinkId, - }) - .from(contributionsTable) - .where( - and( - isNotNull(contributionsTable.contributionLinkId), - isNull(transactionsTable.linkedUserId) - ) - ) - .leftJoin(transactionsTable, eq(contributionsTable.transactionId, transactionsTable.id)) - .leftJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) - .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) - .limit(count) - .offset(offset) - - return result.map((row: any) => { - if (transactionIdSet.has(row.transaction.id)) { - throw new Error(`transaction ${row.transaction.id} already loaded`) - } - transactionIdSet.add(row.transaction.id) - return { - transaction: v.parse(transactionSelectSchema, row.transaction), - user: v.parse(userSelectSchema, row.user), - contributionLinkId: row.contributionLinkId, - } - }) -} - -export async function loadTransactionLinks( - db: MySql2Database, - offset: number, - count: number, -): Promise { - const result = await db - .select() - .from(transactionLinksTable) - .leftJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id)) - .orderBy(asc(transactionLinksTable.createdAt), asc(transactionLinksTable.id)) - .limit(count) - .offset(offset) - - return result.map((row: any) => { - return v.parse(transactionLinkDbSchema, { - ...row.transaction_links, - user: row.users, - }) - }) -} - -export async function loadDeletedTransactionLinks( - db: MySql2Database, - offset: number, - count: number, -): Promise { - const result = await db - .select() - .from(transactionLinksTable) - .leftJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id)) - .where(and( - isNotNull(transactionLinksTable.deletedAt), - lt(transactionLinksTable.deletedAt, transactionLinksTable.validUntil) - )) - .orderBy(asc(transactionLinksTable.deletedAt), asc(transactionLinksTable.id)) - .limit(count) - .offset(offset) - - return result.map((row: any) => { - return v.parse(transactionDbSchema, { - id: row.transaction_links.id, - typeId: TransactionTypeId.RECEIVE, - amount: row.transaction_links.amount, - balanceDate: new Date(row.transaction_links.deletedAt), - memo: row.transaction_links.memo, - transactionLinkCode: row.transaction_links.code, - user: row.users, - linkedUser: row.users, - }) - }) -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts index 3b7b2ec45..ac708c7ed 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts @@ -10,8 +10,6 @@ import { unique, varchar, } from 'drizzle-orm/mysql-core' -import { createSelectSchema } from 'drizzle-valibot' -import * as v from 'valibot' // use only fields needed in the migration, after update the rest of the project, import database instead export const communitiesTable = mysqlTable( @@ -62,9 +60,6 @@ export const usersTable = mysqlTable( (table) => [unique('uuid_key').on(table.gradidoId, table.communityUuid)], ) -export const userSelectSchema = createSelectSchema(usersTable) -export type UserSelect = v.InferOutput - export const userRolesTable = mysqlTable('user_roles', { id: int().autoincrement().notNull(), userId: int('user_id').notNull(), @@ -94,13 +89,11 @@ export const transactionsTable = mysqlTable( (table) => [index('user_id').on(table.userId)], ) -export const transactionSelectSchema = createSelectSchema(transactionsTable) -export type TransactionSelect = v.InferOutput - export const transactionLinksTable = mysqlTable('transaction_links', { id: int().autoincrement().notNull(), userId: int().notNull(), amount: decimal({ precision: 40, scale: 20 }).notNull(), + holdAvailableAmount: decimal("hold_available_amount", { precision: 40, scale: 20 }).notNull(), memo: varchar({ length: 255 }).notNull(), code: varchar({ length: 24 }).notNull(), createdAt: datetime({ mode: 'string' }).notNull(), diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts index 64b3a28e7..0c18102f6 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts @@ -26,6 +26,7 @@ export class DatabaseError extends Error { super(parts.join('\n\n')) this.name = 'DatabaseError' + this.cause = originalError } } @@ -40,7 +41,7 @@ export class BlockchainError extends Error { super(parts.join('\n\n')) this.name = 'BlockchainError' - this.stack = originalError.stack + this.cause = originalError } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts index 0d7f69805..b200e8769 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts @@ -1,11 +1,9 @@ -import * as v from 'valibot' import { onShutdown } from '../../../../shared/src/helper/onShutdown' -import { uuidv4Schema } from '../../schemas/typeGuard.schema' import { exportAllCommunities } from './binaryExport' import { bootstrap } from './bootstrap' import { syncDbWithBlockchainContext } from './interaction/syncDbWithBlockchain/syncDbWithBlockchain.context' -const BATCH_SIZE = 500 +const BATCH_SIZE = 1000 async function main() { // prepare in memory blockchains diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/AbstractBalances.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/AbstractBalances.role.ts deleted file mode 100644 index b2433240e..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/AbstractBalances.role.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { AccountBalances, Filter, InMemoryBlockchain, SearchDirection_DESC } from 'gradido-blockchain-js' -import { KeyPairIdentifierLogic } from '../../../../data/KeyPairIdentifier.logic' -import { ResolveKeyPair } from '../../../../interactions/resolveKeyPair/ResolveKeyPair.context' -import { IdentifierAccount } from '../../../../schemas/account.schema' -import { Transaction } from '../../../../schemas/transaction.schema' -import { Context } from '../../Context' -import { Balance } from '../../data/Balance' - -export abstract class AbstractBalancesRole { - public constructor(protected transaction: Transaction) {} - - abstract getAccountBalances(context: Context): Promise - - async getLastBalanceForUser(identifierAccount: IdentifierAccount, blockchain: InMemoryBlockchain): Promise { - const userKeyPair = await ResolveKeyPair( - new KeyPairIdentifierLogic(identifierAccount), - ) - const f = new Filter() - f.involvedPublicKey = userKeyPair.getPublicKey() - f.pagination.size = 1 - f.searchDirection = SearchDirection_DESC - const lastSenderTransaction = blockchain.findOne(f) - if (!lastSenderTransaction) { - throw new Error(`no last transaction found for user: ${JSON.stringify(identifierAccount, null, 2)}`) - } - const lastConfirmedTransaction = lastSenderTransaction.getConfirmedTransaction() - if (!lastConfirmedTransaction) { - throw new Error(`invalid transaction, getConfirmedTransaction call failed for transaction nr: ${lastSenderTransaction.getTransactionNr()}`) - } - const senderLastAccountBalance = lastConfirmedTransaction.getAccountBalance(userKeyPair.getPublicKey(), '') - if (!senderLastAccountBalance) { - throw new Error( - `no sender account balance found for transaction nr: ${lastSenderTransaction.getTransactionNr()} and public key: ${userKeyPair.getPublicKey()?.convertToHex()}` - ) - } - return Balance.fromAccountBalance(senderLastAccountBalance, lastConfirmedTransaction.getConfirmedAt().getDate()) - } -} \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/CreationBalances.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/CreationBalances.role.ts deleted file mode 100644 index 45d8e2308..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/CreationBalances.role.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { AccountBalances } from 'gradido-blockchain-js' -import { Transaction } from '../../../../schemas/transaction.schema' -import { Context } from '../../Context' -import { TransactionDb } from '../../valibot.schema' -import { AbstractBalancesRole } from './AbstractBalances.role' - -export class CreationBalancesRole extends AbstractBalancesRole { - - constructor(transaction: Transaction, protected dbTransaction: TransactionDb) { - super(transaction) - } - - async getAccountBalances(context: Context): Promise { - if (this.dbTransaction.linkedUser.communityUuid !== this.dbTransaction.user.communityUuid) { - throw new Error('creation: both recipient and signer must belong to same community') - } - - const accountBalances = new AccountBalances() - const communityContext = context.getCommunityContextByUuid(this.dbTransaction.user.communityUuid) - const balance = await this.getLastBalanceForUser(this.transaction.user, communityContext.blockchain) - - // calculate decay since last balance with legacy calculation method - balance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate) - communityContext.aufBalance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate) - communityContext.gmwBalance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate) - - accountBalances.add(balance.getAccountBalance()) - accountBalances.add(communityContext.aufBalance.getAccountBalance()) - accountBalances.add(communityContext.gmwBalance.getAccountBalance()) - - return accountBalances - } -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/DeferredTransferBalances.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/DeferredTransferBalances.role.ts deleted file mode 100644 index 077c20ca7..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/DeferredTransferBalances.role.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { AccountBalance, AccountBalances, GradidoUnit } from 'gradido-blockchain-js' -import { KeyPairIdentifierLogic } from '../../../../data/KeyPairIdentifier.logic' -import { ResolveKeyPair } from '../../../../interactions/resolveKeyPair/ResolveKeyPair.context' -import { Transaction } from '../../../../schemas/transaction.schema' -import { Context } from '../../Context' -import { TransactionLinkDb } from '../../valibot.schema' -import { AbstractBalancesRole } from './AbstractBalances.role' - - -export class DeferredTransferBalancesRole extends AbstractBalancesRole { - constructor(transaction: Transaction, protected dbTransactionLink: TransactionLinkDb) { - super(transaction) - } - - async getAccountBalances(context: Context): Promise { - const senderCommunityContext = context.getCommunityContextByUuid(this.dbTransactionLink.user.communityUuid) - const accountBalances = new AccountBalances() - - const seededIdentifier = new KeyPairIdentifierLogic(this.transaction.linkedUser!) - if (!seededIdentifier.isSeedKeyPair()) { - throw new Error(`linked user is not a seed: ${JSON.stringify(this.transaction, null, 2)}`) - } - const seedKeyPair = await ResolveKeyPair(seededIdentifier) - const senderAccountBalance = await this.getLastBalanceForUser(this.transaction.user, senderCommunityContext.blockchain) - - let amount = GradidoUnit.fromString(this.dbTransactionLink.amount.toString()) - amount = amount.calculateCompoundInterest((this.dbTransactionLink.validUntil.getTime() - this.dbTransactionLink.createdAt.getTime()) / 60000) - senderAccountBalance.updateLegacyDecay(amount.negated(), this.dbTransactionLink.createdAt) - accountBalances.add(senderAccountBalance.getAccountBalance()) - accountBalances.add(new AccountBalance(seedKeyPair.getPublicKey(), amount, '')) - return accountBalances - } -} \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RedeemDeferredTransferBalances.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RedeemDeferredTransferBalances.role.ts deleted file mode 100644 index 4024c3dd3..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RedeemDeferredTransferBalances.role.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { AccountBalances } from 'gradido-blockchain-js' -import { Transaction } from '../../../../schemas/transaction.schema' -import { Context } from '../../Context' -import { TransactionDb } from '../../valibot.schema' -import { AbstractBalancesRole } from './AbstractBalances.role' - -export class RedeemDeferredTransferBalancesRole extends AbstractBalancesRole { - constructor(transaction: Transaction, protected dbTransaction: TransactionDb) { - super(transaction) - } - - async getAccountBalances(context: Context): Promise { - // I use the receiving part of transaction pair, so the user is the recipient and the linked user the sender and amount is positive - const senderCommunityContext = context.getCommunityContextByUuid(this.dbTransaction.linkedUser.communityUuid) - const recipientCommunityContext = context.getCommunityContextByUuid(this.dbTransaction.user.communityUuid) - const accountBalances = new AccountBalances() - - context.cache.setHomeCommunityTopicId(senderCommunityContext.topicId) - const senderLastBalance = await this.getLastBalanceForUser(this.transaction.linkedUser!, senderCommunityContext.blockchain) - context.cache.setHomeCommunityTopicId(recipientCommunityContext.topicId) - const recipientLastBalance = await this.getLastBalanceForUser(this.transaction.user, recipientCommunityContext.blockchain) - - senderLastBalance.updateLegacyDecay(this.dbTransaction.amount.negate(), this.dbTransaction.balanceDate) - recipientLastBalance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate) - - accountBalances.add(senderLastBalance.getAccountBalance()) - accountBalances.add(recipientLastBalance.getAccountBalance()) - return accountBalances - } -} \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RegisterAddressBalances.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RegisterAddressBalances.role.ts deleted file mode 100644 index 0c4c53454..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RegisterAddressBalances.role.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { AccountBalance, AccountBalances, GradidoUnit } from 'gradido-blockchain-js' -import { KeyPairIdentifierLogic } from '../../../../data/KeyPairIdentifier.logic' -import { ResolveKeyPair } from '../../../../interactions/resolveKeyPair/ResolveKeyPair.context' -import { Context } from '../../Context' -import { AbstractBalancesRole } from './AbstractBalances.role' - - -export class RegisterAddressBalancesRole extends AbstractBalancesRole { - async getAccountBalances(_context: Context): Promise { - const accountBalances = new AccountBalances() - const recipientKeyPair = await ResolveKeyPair( - new KeyPairIdentifierLogic(this.transaction.user), - ) - accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), GradidoUnit.zero(), '')) - return accountBalances - } -} \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/TransferBalances.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/TransferBalances.role.ts deleted file mode 100644 index c5182627d..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/TransferBalances.role.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { AccountBalances } from 'gradido-blockchain-js' -import { Transaction } from '../../../../schemas/transaction.schema' -import { Context } from '../../Context' -import { TransactionDb } from '../../valibot.schema' -import { AbstractBalancesRole } from './AbstractBalances.role' - -export class TransferBalancesRole extends AbstractBalancesRole { - constructor(transaction: Transaction, protected dbTransaction: TransactionDb) { - super(transaction) - } - - async getAccountBalances(context: Context): Promise { - // I use the receiving part of transaction pair, so the user is the recipient and the linked user the sender and amount is positive - const senderCommunityContext = context.getCommunityContextByUuid(this.dbTransaction.linkedUser.communityUuid) - const recipientCommunityContext = context.getCommunityContextByUuid(this.dbTransaction.user.communityUuid) - const accountBalances = new AccountBalances() - - context.cache.setHomeCommunityTopicId(senderCommunityContext.topicId) - const senderLastBalance = await this.getLastBalanceForUser(this.transaction.linkedUser!, senderCommunityContext.blockchain) - context.cache.setHomeCommunityTopicId(recipientCommunityContext.topicId) - const recipientLastBalance = await this.getLastBalanceForUser(this.transaction.user, recipientCommunityContext.blockchain) - - senderLastBalance.updateLegacyDecay(this.dbTransaction.amount.negate(), this.dbTransaction.balanceDate) - recipientLastBalance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate) - - accountBalances.add(senderLastBalance.getAccountBalance()) - accountBalances.add(recipientLastBalance.getAccountBalance()) - return accountBalances - } -} \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/accountBalances.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/accountBalances.context.ts deleted file mode 100644 index 6a2f63604..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/accountBalances.context.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { AccountBalances } from 'gradido-blockchain-js' -import * as v from 'valibot' -import { InputTransactionType } from '../../../../data/InputTransactionType.enum' -import { Transaction } from '../../../../schemas/transaction.schema' -import { Context } from '../../Context' -import { TransactionDb, TransactionLinkDb, transactionDbSchema, transactionLinkDbSchema } from '../../valibot.schema' -import { AbstractBalancesRole } from './AbstractBalances.role' -import { CreationBalancesRole } from './CreationBalances.role' -import { DeferredTransferBalancesRole } from './DeferredTransferBalances.role' -import { RedeemDeferredTransferBalancesRole } from './RedeemDeferredTransferBalances.role' -import { RegisterAddressBalancesRole } from './RegisterAddressBalances.role' -import { TransferBalancesRole } from './TransferBalances.role' - -export async function accountBalancesContext(transaction: Transaction, item: TransactionDb | TransactionLinkDb, context: Context): Promise { - let role: AbstractBalancesRole | null = null - if (InputTransactionType.GRADIDO_CREATION === transaction.type) { - role = new CreationBalancesRole(transaction, v.parse(transactionDbSchema, item)) - } else if (InputTransactionType.GRADIDO_TRANSFER === transaction.type) { - role = new TransferBalancesRole(transaction, v.parse(transactionDbSchema, item)) - } else if (InputTransactionType.GRADIDO_DEFERRED_TRANSFER === transaction.type) { - role = new DeferredTransferBalancesRole(transaction, v.parse(transactionLinkDbSchema, item)) - } else if (InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER === transaction.type) { - role = new RedeemDeferredTransferBalancesRole(transaction, v.parse(transactionDbSchema, item)) - } else if (InputTransactionType.REGISTER_ADDRESS === transaction.type) { - role = new RegisterAddressBalancesRole(transaction) - } - if (!role) { - throw new Error(`No role found for transaction type ${transaction.type}`) - } - return await role.getAccountBalances(context) -} - diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts index 307b2933c..adb84e0c9 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts @@ -7,9 +7,14 @@ import { Context } from '../../Context' import { Balance } from '../../data/Balance' import { CommunityContext } from '../../valibot.schema' -export abstract class AbstractSyncRole { - private items: T[] = [] - private offset = 0 +export type IndexType = { + date: Date + id: number +} + +export abstract class AbstractSyncRole { + private items: ItemType[] = [] + protected lastIndex: IndexType = { date: new Date(0), id: 0 } protected logger: Logger constructor(protected readonly context: Context) { @@ -54,13 +59,15 @@ export abstract class AbstractSyncRole { const lastTransactions = blockchain.findAll(f) for (let i = lastTransactions.size() - 1; i >= 0; i--) { const tx = lastTransactions.get(i) - this.context.logger.debug(`${tx?.getConfirmedTransaction()!.toJson(true)}`) + this.context.logger.debug(`${i}: ${tx?.getConfirmedTransaction()!.toJson(true)}`) } } abstract getDate(): Date - abstract loadFromDb(offset: number, count: number): Promise - abstract pushToBlockchain(item: T): void + // for using seek rather than offset pagination approach + abstract getLastIndex(): IndexType + abstract loadFromDb(lastIndex: IndexType, count: number): Promise + abstract pushToBlockchain(item: ItemType): void abstract itemTypeName(): string // return count of new loaded items @@ -70,12 +77,14 @@ export abstract class AbstractSyncRole { if (this.logger.isDebugEnabled()) { timeUsed = new Profiler() } - this.items = await this.loadFromDb(this.offset, batchSize) - this.offset += this.items.length - if (timeUsed && this.items.length) { - this.logger.debug( - `${timeUsed.string()} for loading ${this.items.length} ${this.itemTypeName()} from db`, - ) + this.items = await this.loadFromDb(this.lastIndex, batchSize) + if (this.length > 0) { + this.lastIndex = this.getLastIndex() + if (timeUsed) { + this.logger.debug( + `${timeUsed.string()} for loading ${this.items.length} ${this.itemTypeName()} from db`, + ) + } } return this.items.length } @@ -89,14 +98,20 @@ export abstract class AbstractSyncRole { this.pushToBlockchain(this.shift()) } - peek(): T { + peek(): ItemType { if (this.isEmpty()) { throw new Error(`[peek] No items, please call this only if isEmpty returns false`) } return this.items[0] } + peekLast(): ItemType { + if (this.isEmpty()) { + throw new Error(`[peekLast] No items, please call this only if isEmpty returns false`) + } + return this.items[this.items.length - 1] + } - shift(): T { + shift(): ItemType { const item = this.items.shift() if (!item) { throw new Error(`[shift] No items, shift return undefined`) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts index 41f488d4b..299f4cec4 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts @@ -1,4 +1,4 @@ -import { and, asc, eq, isNotNull } from 'drizzle-orm' +import { and, asc, eq, gt, isNotNull, or } from 'drizzle-orm' import * as v from 'valibot' import { Context } from '../../Context' import { contributionLinkModerators } from '../../database' @@ -7,6 +7,8 @@ import { CreationsSyncRole } from './CreationsSync.role' import { contributionsTable, usersTable } from '../../drizzle.schema' import { ContributionStatus } from '../../data/ContributionStatus' import { DatabaseError } from '../../errors' +import { IndexType } from './AbstractSync.role' +import { toMysqlDateTime } from '../../utils' export class ContributionLinkTransactionSyncRole extends CreationsSyncRole { constructor(readonly context: Context) { @@ -16,7 +18,7 @@ export class ContributionLinkTransactionSyncRole extends CreationsSyncRole { return 'contributionLinkTransaction' } - async loadFromDb(offset: number, count: number): Promise { + async loadFromDb(lastIndex: IndexType, count: number): Promise { const result = await this.context.db .select({ contribution: contributionsTable, @@ -26,11 +28,17 @@ export class ContributionLinkTransactionSyncRole extends CreationsSyncRole { .where(and( isNotNull(contributionsTable.contributionLinkId), eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED), + or( + gt(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), + and( + eq(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), + gt(contributionsTable.transactionId, lastIndex.id) + ) + ) )) .innerJoin(usersTable, eq(contributionsTable.userId, usersTable.id)) .orderBy(asc(contributionsTable.confirmedAt), asc(contributionsTable.transactionId)) .limit(count) - .offset(offset) const verifiedCreationTransactions: CreationTransactionDb[] = [] for(const row of result) { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/CreationsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/CreationsSync.role.ts index ff23e1fcb..17d1b3119 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/CreationsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/CreationsSync.role.ts @@ -1,12 +1,15 @@ -import { and, asc, eq, isNull } from 'drizzle-orm' +import { and, asc, eq, isNull, gt, or } from 'drizzle-orm' import { alias } from 'drizzle-orm/mysql-core' import { AccountBalances, AuthenticatedEncryption, EncryptedMemo, + Filter, GradidoTransactionBuilder, KeyPairEd25519, MemoryBlockPtr, + SearchDirection_DESC, + TransactionType_CREATION, TransferAmount } from 'gradido-blockchain-js' import * as v from 'valibot' @@ -18,7 +21,8 @@ import { } from '../../drizzle.schema' import { BlockchainError, DatabaseError } from '../../errors' import { CommunityContext, CreationTransactionDb, creationTransactionDbSchema } from '../../valibot.schema' -import { AbstractSyncRole } from './AbstractSync.role' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' +import { toMysqlDateTime } from '../../utils' export class CreationsSyncRole extends AbstractSyncRole { @@ -26,11 +30,16 @@ export class CreationsSyncRole extends AbstractSyncRole { return this.peek().confirmedAt } + getLastIndex(): IndexType { + const lastItem = this.peekLast() + return { date: lastItem.confirmedAt, id: lastItem.transactionId } + } + itemTypeName(): string { return 'creationTransactions' } - async loadFromDb(offset: number, count: number): Promise { + async loadFromDb(lastIndex: IndexType, count: number): Promise { const confirmedByUsers = alias(usersTable, 'confirmedByUser') const result = await this.context.db .select({ @@ -42,12 +51,18 @@ export class CreationsSyncRole extends AbstractSyncRole { .where(and( isNull(contributionsTable.contributionLinkId), eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED), + or( + gt(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), + and( + eq(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), + gt(contributionsTable.transactionId, lastIndex.id) + ) + ) )) .innerJoin(usersTable, eq(contributionsTable.userId, usersTable.id)) .innerJoin(confirmedByUsers, eq(contributionsTable.confirmedBy, confirmedByUsers.id)) .orderBy(asc(contributionsTable.confirmedAt), asc(contributionsTable.transactionId)) .limit(count) - .offset(offset) return result.map((row) => { const item = { @@ -126,6 +141,14 @@ export class CreationsSyncRole extends AbstractSyncRole { this.calculateAccountBalances(item, communityContext, recipientPublicKey), ) } catch(e) { + const f= new Filter() + f.transactionType = TransactionType_CREATION + f.searchDirection = SearchDirection_DESC + f.pagination.size = 1 + const lastContribution = blockchain.findOne(f) + if (lastContribution) { + this.context.logger.warn(`last contribution: ${lastContribution.getConfirmedTransaction()?.toJson(true)}`) + } throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts index c1c12e724..0111234b7 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts @@ -1,7 +1,7 @@ import { CommunityContext, DeletedTransactionLinkDb, deletedTransactionLinKDbSchema } from '../../valibot.schema' -import { AbstractSyncRole } from './AbstractSync.role' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' import { transactionLinksTable, usersTable } from '../../drizzle.schema' -import { and, lt, asc, isNotNull, eq } from 'drizzle-orm' +import { and, lt, asc, isNotNull, eq, or, gt } from 'drizzle-orm' import * as v from 'valibot' import { AccountBalance, @@ -19,17 +19,23 @@ import { deriveFromCode } from '../../../../data/deriveKeyPair' import { addToBlockchain } from '../../blockchain' import { BlockchainError, DatabaseError } from '../../errors' import { Balance } from '../../data/Balance' +import { toMysqlDateTime } from '../../utils' export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { getDate(): Date { return this.peek().deletedAt } + getLastIndex(): IndexType { + const lastItem = this.peekLast() + return { date: lastItem.deletedAt, id: lastItem.id } + } + itemTypeName(): string { return 'deletedTransactionLinks' } - async loadFromDb(offset: number, count: number): Promise { + async loadFromDb(lastIndex: IndexType, count: number): Promise { const result = await this.context.db .select({ transactionLink: transactionLinksTable, @@ -39,13 +45,19 @@ export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { const item = { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DoubleLinkedTransactions.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DoubleLinkedTransactions.role.ts deleted file mode 100644 index 136155a2c..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DoubleLinkedTransactions.role.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Context } from '../../Context' -import { loadDoubleLinkedTransactions } from '../../database' -import { AbstractSyncRole } from './AbstractSync.role' - -export class DoubleLinkedTransactionsSyncRole extends AbstractSyncRole<{ id: number, balanceDate: Date }> { - static allTransactionIds: number[] = [] - constructor(readonly context: Context) { - super(context) - } - itemTypeName(): string { - return 'doubleLinkedTransaction' - } - - async loadFromDb(offset: number, count: number): Promise<{ id: number, balanceDate: Date }[]> { - const result = await loadDoubleLinkedTransactions(this.context.db, offset, count) - DoubleLinkedTransactionsSyncRole.allTransactionIds.push(...result.map((r) => r.id)) - return result - } - - getDate(): Date { - return this.peek().balanceDate - } - - async pushToBlockchain(item: { id: number, balanceDate: Date }): Promise { - this.logger.warn(`Double transaction_links ${item.id} found.`) - } -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/InvalidContributionTransactionSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/InvalidContributionTransactionSync.role.ts deleted file mode 100644 index 616dd1fe4..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/InvalidContributionTransactionSync.role.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Context } from '../../Context' -import { loadInvalidContributionTransactions } from '../../database' -import { AbstractSyncRole } from './AbstractSync.role' - -export class InvalidContributionTransactionSyncRole extends AbstractSyncRole<{ id: number, balanceDate: Date }> { - static allTransactionIds: number[] = [] - constructor(readonly context: Context) { - super(context) - } - itemTypeName(): string { - return 'invalidContributionTransaction' - } - - async loadFromDb(offset: number, count: number): Promise<{ id: number, balanceDate: Date }[]> { - const result = await loadInvalidContributionTransactions(this.context.db, offset, count) - InvalidContributionTransactionSyncRole.allTransactionIds.push(...result.map((r) => r.id)) - return result - } - - getDate(): Date { - return this.peek().balanceDate - } - - async pushToBlockchain(item: { id: number, balanceDate: Date }): Promise { - this.logger.warn(`Invalid contribution transaction ${item.id} found.`) - } -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts index 6476d886b..62ee0c832 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts @@ -1,4 +1,4 @@ -import { and, asc, eq, isNotNull, isNull } from 'drizzle-orm' +import { and, asc, eq, isNotNull, isNull, gt, or } from 'drizzle-orm' import { alias } from 'drizzle-orm/mysql-core' import { AccountBalances, @@ -17,7 +17,8 @@ import { TransactionTypeId } from '../../data/TransactionTypeId' import { transactionsTable, usersTable } from '../../drizzle.schema' import { BlockchainError, DatabaseError, NegativeBalanceError, NotEnoughGradidoBalanceError } from '../../errors' import { CommunityContext, TransactionDb, transactionDbSchema } from '../../valibot.schema' -import { AbstractSyncRole } from './AbstractSync.role' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' +import { toMysqlDateTime } from '../../utils' export class LocalTransactionsSyncRole extends AbstractSyncRole { @@ -25,11 +26,16 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { return this.peek().balanceDate } + getLastIndex(): IndexType { + const lastItem = this.peekLast() + return { date: lastItem.balanceDate, id: lastItem.id } + } + itemTypeName(): string { return 'localTransactions' } - async loadFromDb(offset: number, count: number): Promise { + async loadFromDb(lastIndex: IndexType, count: number): Promise { const linkedUsers = alias(usersTable, 'linkedUser') const result = await this.context.db .select({ @@ -45,13 +51,19 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { isNotNull(transactionsTable.linkedUserId), eq(usersTable.foreign, 0), eq(linkedUsers.foreign, 0), + or( + gt(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)), + and( + eq(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)), + gt(transactionsTable.id, lastIndex.id) + ) + ) ) ) .innerJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) .innerJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id)) .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) .limit(count) - .offset(offset) return result.map((row) => { const item = { @@ -74,8 +86,7 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { ): GradidoTransactionBuilder { return new GradidoTransactionBuilder() .setCreatedAt(item.balanceDate) - .addMemo( - new EncryptedMemo( + .addMemo(new EncryptedMemo( item.memo, new AuthenticatedEncryption(senderKeyPair), new AuthenticatedEncryption(recipientKeyPair), diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts index b2d0fd707..810fed60d 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts @@ -1,4 +1,4 @@ -import { and, asc, eq, isNotNull, isNull } from 'drizzle-orm' +import { and, asc, eq, isNotNull, isNull, or, gt } from 'drizzle-orm' import { AccountBalance, AccountBalances, @@ -18,20 +18,26 @@ import { addToBlockchain } from '../../blockchain' import { transactionLinksTable, usersTable } from '../../drizzle.schema' import { BlockchainError, DatabaseError } from '../../errors' import { CommunityContext, RedeemedTransactionLinkDb, redeemedTransactionLinkDbSchema } from '../../valibot.schema' -import { AbstractSyncRole } from './AbstractSync.role' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' import { deriveFromCode } from '../../../../data/deriveKeyPair' import { alias } from 'drizzle-orm/mysql-core' +import { toMysqlDateTime } from '../../utils' export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { getDate(): Date { return this.peek().redeemedAt } + getLastIndex(): IndexType { + const lastItem = this.peekLast() + return { date: lastItem.redeemedAt, id: lastItem.id } + } + itemTypeName(): string { return 'redeemTransactionLinks' } - async loadFromDb(offset: number, count: number): Promise { + async loadFromDb(lastIndex: IndexType, count: number): Promise { const redeemedByUser = alias(usersTable, 'redeemedByUser') const result = await this.context.db .select({ @@ -45,14 +51,20 @@ export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { const item = { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts index 574c02235..6ee368252 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts @@ -1,17 +1,15 @@ -import { asc, eq } from 'drizzle-orm' +import { asc, eq, or, gt, and, isNull } from 'drizzle-orm' import { AccountBalance, AccountBalances, AuthenticatedEncryption, DurationSeconds, EncryptedMemo, - Filter, GradidoTransactionBuilder, GradidoTransfer, GradidoUnit, KeyPairEd25519, MemoryBlockPtr, - SearchDirection_DESC, TransferAmount } from 'gradido-blockchain-js' import * as v from 'valibot' @@ -19,9 +17,9 @@ import { addToBlockchain } from '../../blockchain' import { transactionLinksTable, usersTable } from '../../drizzle.schema' import { BlockchainError, DatabaseError, NegativeBalanceError } from '../../errors' import { CommunityContext, TransactionLinkDb, transactionLinkDbSchema } from '../../valibot.schema' -import { AbstractSyncRole } from './AbstractSync.role' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' import { deriveFromCode } from '../../../../data/deriveKeyPair' -import { legacyCalculateDecay } from '../../utils' +import { calculateEffectiveSeconds, reverseLegacyDecay, toMysqlDateTime } from '../../utils' import Decimal from 'decimal.js-light' export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { @@ -29,19 +27,30 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { + async loadFromDb(lastIndex: IndexType, count: number): Promise { const result = await this.context.db .select() .from(transactionLinksTable) .innerJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id)) + .where(or( + gt(transactionLinksTable.createdAt, toMysqlDateTime(lastIndex.date)), + and( + eq(transactionLinksTable.createdAt, toMysqlDateTime(lastIndex.date)), + gt(transactionLinksTable.id, lastIndex.id) + ) + )) .orderBy(asc(transactionLinksTable.createdAt), asc(transactionLinksTable.id)) .limit(count) - .offset(offset) - + return result.map((row) => { const item = { ...row.transaction_links, @@ -90,7 +99,15 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole item.validUntil.getTime()) { + endDateTime = item.validUntil.getTime() + } + let duration = new DurationSeconds((endDateTime - item.createdAt.getTime()) / 1000) + const hourInSeconds = 60 * 60 + if (duration.getSeconds() < hourInSeconds) { + duration = new DurationSeconds(hourInSeconds) + } + let blockedAmount = GradidoUnit.fromString(reverseLegacyDecay(new Decimal(item.amount.toString()), duration.getSeconds()).toString()) + blockedAmount = blockedAmount.add(GradidoUnit.fromGradidoCent(1)) + // let blockedAmount = decayedAmount.calculateCompoundInterest(duration.getSeconds()) let accountBalances: AccountBalances try { accountBalances = this.calculateBalances(item, blockedAmount, communityContext, senderPublicKey, recipientPublicKey) @@ -119,9 +159,14 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { @@ -23,17 +24,28 @@ export class UsersSyncRole extends AbstractSyncRole { return this.peek().createdAt } + getLastIndex(): IndexType { + const lastItem = this.peekLast() + return { date: lastItem.createdAt, id: lastItem.id } + } + itemTypeName(): string { return 'users' } - async loadFromDb(offset: number, count: number): Promise { + async loadFromDb(lastIndex: IndexType, count: number): Promise { const result = await this.context.db .select() .from(usersTable) + .where(or( + gt(usersTable.createdAt, toMysqlDateTime(lastIndex.date)), + and( + eq(usersTable.createdAt, toMysqlDateTime(lastIndex.date)), + gt(usersTable.id, lastIndex.id) + ) + )) .orderBy(asc(usersTable.createdAt), asc(usersTable.id)) .limit(count) - .offset(offset) return result.map((row) => { try { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts index e3b2d059f..37ab78373 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts @@ -1,7 +1,6 @@ import { Profiler } from 'gradido-blockchain-js' import { Context } from '../../Context' import { CreationsSyncRole } from './CreationsSync.role' -import { InvalidContributionTransactionSyncRole } from './InvalidContributionTransactionSync.role' import { LocalTransactionsSyncRole } from './LocalTransactionsSync.role' import { UsersSyncRole } from './UsersSync.role' import { TransactionLinkFundingsSyncRole } from './TransactionLinkFundingsSync.role' @@ -24,13 +23,15 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n ] let transactionsCount = 0 let transactionsCountSinceLastLog = 0 + let transactionsCountSinceLastPrint = 0 let available = containers + const isDebug = context.logger.isDebugEnabled() while (true) { timeUsedDB.reset() const results = await Promise.all(available.map((c) => c.ensureFilled(batchSize))) const loadedItemsCount = results.reduce((acc, c) => acc + c, 0) // log only, if at least one new item was loaded - if (loadedItemsCount && context.logger.isDebugEnabled()) { + if (loadedItemsCount && isDebug) { context.logger.debug(`${loadedItemsCount} new items loaded from db in ${timeUsedDB.string()}`) } @@ -46,24 +47,24 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n available.sort((a, b) => a.getDate().getTime() - b.getDate().getTime()) // context.logger.debug(`sorted ${available.length} containers in ${sortTime.string()}`) } - available[0].toBlockchain() - process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`) + available[0].toBlockchain() transactionsCount++ - transactionsCountSinceLastLog++ - if (transactionsCountSinceLastLog >= batchSize) { - context.logger.debug(`${transactionsCountSinceLastLog} transactions added to blockchain in ${timeUsedBlockchain.string()}`) - timeUsedBlockchain.reset() - transactionsCountSinceLastLog = 0 + if (isDebug) { + process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`) + transactionsCountSinceLastLog++ + if (transactionsCountSinceLastLog >= batchSize) { + context.logger.debug(`${transactionsCountSinceLastLog} transactions added to blockchain in ${timeUsedBlockchain.string()}`) + timeUsedBlockchain.reset() + transactionsCountSinceLastLog = 0 + } + } else { + transactionsCountSinceLastPrint++ + if (transactionsCountSinceLastPrint >= 100) { + process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`) + transactionsCountSinceLastPrint = 0 + } } } - process.stdout.write(`\n`) - context.logger.info(`Synced ${transactionsCount} transactions to blockchain in ${(timeUsedAll.seconds() / 60).toFixed(2)} minutes`) - context.logger.info(`Invalid contribution transactions: ${InvalidContributionTransactionSyncRole.allTransactionIds.length}`) - if (context.logger.isDebugEnabled()) { - context.logger.debug(InvalidContributionTransactionSyncRole.allTransactionIds.join(', ')) - } - /*context.logger.info(`Double linked transactions: ${TransactionsSyncRole.doubleTransactionLinkCodes.length}`) - if (context.logger.isDebugEnabled()) { - context.logger.debug(TransactionsSyncRole.doubleTransactionLinkCodes.join(', ')) - }*/ + process.stdout.write(`successfully added to blockchain: ${transactionsCount}\n`) + context.logger.info(`Synced ${transactionsCount} transactions to blockchain in ${timeUsedAll.string()}`) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts index dc44af15f..1e7d0f0b2 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts @@ -9,6 +9,19 @@ export function bytesToKbyte(bytes: number): string { return (bytes / 1024).toFixed(0) } +export function bytesString(bytes: number): string { + if (bytes > (1024 * 1024)) { + return `${bytesToMbyte(bytes)} MB` + } else if (bytes > 1024) { + return `${bytesToKbyte(bytes)} KB` + } + return `${bytes.toFixed(0)} Bytes` +} + +export function toMysqlDateTime(date: Date): string { + return date.toISOString().slice(0, 23).replace('T', ' ') +} + export function calculateOneHashStep(hash: Buffer, data: Buffer): Buffer { const outputHash = Buffer.alloc(crypto_generichash_KEYBYTES, 0) crypto_generichash_batch(outputHash, [hash, data]) @@ -17,15 +30,22 @@ export function calculateOneHashStep(hash: Buffer, data: Buffer): Buffer { if (value.user && value.confirmedByUser && value.user.gradidoId === value.confirmedByUser.gradidoId) { throw new Error(`expect user to be different from confirmedByUser: ${JSON.stringify(value, null, 2)}`) @@ -77,6 +78,7 @@ export const transactionLinkDbSchema = v.object({ code: identifierSeedSchema, createdAt: dateSchema, validUntil: dateSchema, + holdAvailableAmount: gradidoAmountSchema, redeemedAt: v.nullish(dateSchema), deletedAt: v.nullish(dateSchema), })