From b1214f8b6cff602d90b6c1c66b3ed940dca12bad Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 18 Dec 2025 07:50:27 +0100 Subject: [PATCH] use own account balances --- dlt-connector/bun.lock | 9 + dlt-connector/package.json | 2 + .../db-v2.7.0_to_blockchain-v3.5/Context.ts | 26 ++- .../blockchain.ts | 91 ++++---- .../db-v2.7.0_to_blockchain-v3.5/bootstrap.ts | 36 ++- .../db-v2.7.0_to_blockchain-v3.5/database.ts | 216 +++++++++++++----- .../drizzle.schema.ts | 12 +- .../db-v2.7.0_to_blockchain-v3.5/errors.ts | 6 + .../db-v2.7.0_to_blockchain-v3.5/index.ts | 2 +- .../syncDbWithBlockchain/AbstractSync.role.ts | 2 +- .../ContributionLinkTransactionSync.role.ts | 35 +++ .../DoubleLinkedTransactions.role.ts | 27 +++ ...InvalidContributionTransactionSync.role.ts | 27 +++ .../TransactionLinksSync.role.ts | 22 +- .../TransactionsSync.role.ts | 110 ++++++++- .../syncDbWithBlockchain/UsersSync.role.ts | 2 +- .../syncDbWithBlockchain.context.ts | 19 +- .../db-v2.7.0_to_blockchain-v3.5/utils.ts | 37 +++ .../valibot.schema.ts | 22 +- dlt-connector/src/schemas/typeGuard.schema.ts | 2 +- 20 files changed, 582 insertions(+), 123 deletions(-) create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DoubleLinkedTransactions.role.ts create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/InvalidContributionTransactionSync.role.ts diff --git a/dlt-connector/bun.lock b/dlt-connector/bun.lock index b6747359b..477af78c0 100644 --- a/dlt-connector/bun.lock +++ b/dlt-connector/bun.lock @@ -5,6 +5,7 @@ "": { "name": "dlt-connector", "dependencies": { + "cross-env": "^7.0.3", "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#f265dbb1780a912cf8b0418dfe3eaf5cdc5b51cf", }, "devDependencies": { @@ -19,8 +20,10 @@ "@types/uuid": "^8.3.4", "adm-zip": "^0.5.16", "async-mutex": "^0.5.0", + "decimal.js-light": "^2.5.1", "dotenv": "^10.0.0", "drizzle-orm": "^0.44.7", + "drizzle-valibot": "^0.4.2", "elysia": "1.3.8", "graphql-request": "^7.2.0", "jose": "^5.2.2", @@ -432,6 +435,8 @@ "cookie": ["cookie@1.0.2", "", {}, "sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA=="], + "cross-env": ["cross-env@7.0.3", "", { "dependencies": { "cross-spawn": "^7.0.1" }, "bin": { "cross-env": "src/bin/cross-env.js", "cross-env-shell": "src/bin/cross-env-shell.js" } }, "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw=="], + "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], "crypto-js": ["crypto-js@4.2.0", "", {}, "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q=="], @@ -444,6 +449,8 @@ "debug": ["debug@4.4.1", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ=="], + "decimal.js-light": ["decimal.js-light@2.5.1", "", {}, "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg=="], + "deep-extend": ["deep-extend@0.6.0", "", {}, "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="], "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="], @@ -460,6 +467,8 @@ "drizzle-orm": ["drizzle-orm@0.44.7", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@prisma/client": "*", "@tidbcloud/serverless": "*", "@types/better-sqlite3": "*", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=7", "bun-types": "*", "expo-sqlite": ">=14.0.0", "gel": ">=2", "knex": "*", "kysely": "*", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@tidbcloud/serverless", "@types/better-sqlite3", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "gel", "knex", "kysely", "mysql2", "pg", "postgres", "sql.js", "sqlite3"] }, "sha512-quIpnYznjU9lHshEOAYLoZ9s3jweleHlZIAWR/jX9gAWNg/JhQ1wj0KGRf7/Zm+obRrYd9GjPVJg790QY9N5AQ=="], + "drizzle-valibot": ["drizzle-valibot@0.4.2", "", { "peerDependencies": { "drizzle-orm": ">=0.36.0", "valibot": ">=1.0.0-beta.7" } }, "sha512-tzjT7g0Di/HX7426marIy8IDtWODjPgrwvgscdevLQRUe5rzYzRhx6bDsYLdDFF9VI/eaYgnjNeF/fznWJoUjg=="], + "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="], diff --git a/dlt-connector/package.json b/dlt-connector/package.json index 87566cbd2..1f4984b0b 100644 --- a/dlt-connector/package.json +++ b/dlt-connector/package.json @@ -33,8 +33,10 @@ "@types/uuid": "^8.3.4", "adm-zip": "^0.5.16", "async-mutex": "^0.5.0", + "decimal.js-light": "^2.5.1", "dotenv": "^10.0.0", "drizzle-orm": "^0.44.7", + "drizzle-valibot": "^0.4.2", "elysia": "1.3.8", "graphql-request": "^7.2.0", "jose": "^5.2.2", diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts index 58400a615..40931ef1e 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts @@ -1,4 +1,5 @@ import { heapStats } from 'bun:jsc' +import dotenv from 'dotenv' import { drizzle, MySql2Database } from 'drizzle-orm/mysql2' import { Filter, Profiler, SearchDirection_ASC } from 'gradido-blockchain-js' import { getLogger, Logger } from 'log4js' @@ -8,22 +9,27 @@ import { KeyPairCacheManager } from '../../cache/KeyPairCacheManager' import { CONFIG } from '../../config' import { LOG4JS_BASE_CATEGORY } from '../../config/const' import { Uuidv4 } from '../../schemas/typeGuard.schema' +import { loadUserByGradidoId } from './database' import { bytesToMbyte } from './utils' -import { CommunityContext } from './valibot.schema' +import { CommunityContext, CreatedUserDb } from './valibot.schema' + +dotenv.config() export class Context { public logger: Logger public db: MySql2Database public communities: Map public cache: KeyPairCacheManager + public balanceFixGradidoUser: CreatedUserDb | null private timeUsed: Profiler - constructor(logger: Logger, db: MySql2Database, cache: KeyPairCacheManager) { + constructor(logger: Logger, db: MySql2Database, cache: KeyPairCacheManager, balanceFixGradidoUser: CreatedUserDb | null) { this.logger = logger this.db = db this.cache = cache this.communities = new Map() this.timeUsed = new Profiler() + this.balanceFixGradidoUser = balanceFixGradidoUser } static async create(): Promise { @@ -36,10 +42,22 @@ export class Context { database: CONFIG.MYSQL_DATABASE, port: CONFIG.MYSQL_PORT, }) + const db = drizzle({ client: connection }) + const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5`) + let balanceFixGradidoUser: CreatedUserDb | null = null + if (process.env.MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID) { + balanceFixGradidoUser = await loadUserByGradidoId(db, process.env.MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID) + if (!balanceFixGradidoUser) { + logger.error(`MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID was set to ${process.env.MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID} but user not found`) + } + } else { + logger.debug(`MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID was not set`) + } return new Context( - getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5`), - drizzle({ client: connection }), + logger, + db, KeyPairCacheManager.getInstance(), + balanceFixGradidoUser, ) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts index 128418cea..e8ad337bc 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts @@ -1,20 +1,19 @@ +import * as fs from 'node:fs' import { + AccountBalances, Filter, GradidoTransactionBuilder, HieroAccountId, - HieroTransactionId, InMemoryBlockchain, InteractionSerialize, - Pagination, - Profiler, - SearchDirection_DESC, - Timestamp, TransactionType_DEFERRED_TRANSFER, } from 'gradido-blockchain-js' import { getLogger } from 'log4js' +import * as v from 'valibot' import { LOG4JS_BASE_CATEGORY } from '../../config/const' import { InputTransactionType } from '../../data/InputTransactionType.enum' import { LinkedTransactionKeyPairRole } from '../../interactions/resolveKeyPair/LinkedTransactionKeyPair.role' +import { AbstractTransactionRole } from '../../interactions/sendToHiero/AbstractTransaction.role' import { CommunityRootTransactionRole } from '../../interactions/sendToHiero/CommunityRootTransaction.role' import { CreationTransactionRole } from '../../interactions/sendToHiero/CreationTransaction.role' import { DeferredTransferTransactionRole } from '../../interactions/sendToHiero/DeferredTransferTransaction.role' @@ -23,61 +22,74 @@ import { RegisterAddressTransactionRole } from '../../interactions/sendToHiero/R import { TransferTransactionRole } from '../../interactions/sendToHiero/TransferTransaction.role' import { Community, Transaction } from '../../schemas/transaction.schema' import { identifierSeedSchema } from '../../schemas/typeGuard.schema' -import { AbstractTransactionRole } from '../../interactions/sendToHiero/AbstractTransaction.role' -import * as v from 'valibot' -import * as fs from 'node:fs' +import { NotEnoughGradidoBalanceError } from './errors' const logger = getLogger( `${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`, ) export const defaultHieroAccount = new HieroAccountId(0, 0, 2) let transactionAddedToBlockchainSum = 0 +let addToBlockchainSum = 0 const sizeBuffer = Buffer.alloc(2) function addToBlockchain( builder: GradidoTransactionBuilder, blockchain: InMemoryBlockchain, - createdAtTimestamp: Timestamp, + transactionId: number, + accountBalances: AccountBalances, ): boolean { const transaction = builder.build() - /* const transactionSerializer = new InteractionSerialize(transaction) + const transactionSerializer = new InteractionSerialize(transaction) const binTransaction = transactionSerializer.run() if (!binTransaction) { logger.error(`Failed to serialize transaction ${transaction.toJson(true)}`) return false } const filePath = `${blockchain.getCommunityId()}.bin` + if (!addToBlockchainSum) { + // clear file + fs.writeFileSync(filePath, Buffer.alloc(0)) + } sizeBuffer.writeUInt16LE(binTransaction.size(), 0) fs.appendFileSync(filePath, sizeBuffer) fs.appendFileSync(filePath, binTransaction.data()) - */ - // TODO: use actual transaction id if exist in dlt_transactions table - const transactionId = new HieroTransactionId(createdAtTimestamp, defaultHieroAccount) - const interactionSerialize = new InteractionSerialize(transactionId) - + // + try { - const result = blockchain.createAndAddConfirmedTransaction( + const result = blockchain.createAndAddConfirmedTransactionExtern( transaction, - interactionSerialize.run(), - createdAtTimestamp, + transactionId, + accountBalances, ) + // logger.info(`${transactionTypeToString(transaction.getTransactionBody()?.getTransactionType()!)} Transaction added in ${timeUsed.string()}`) + addToBlockchainSum++ return result } catch (error) { - logger.error(`Transaction ${transaction.toJson(true)} not added: ${error}`) - return true + if (error instanceof Error) { + const matches = error.message.match(/not enough Gradido Balance for (send coins|operation), needed: -?(\d+\.\d+), exist: (\d+\.\d+)/) + if (matches) { + const needed = parseFloat(matches[2]) + const exist = parseFloat(matches[3]) + throw new NotEnoughGradidoBalanceError(needed, exist) + } + } + const lastTransaction = blockchain.findOne(Filter.LAST_TRANSACTION) + throw new Error(`Transaction ${transaction.toJson(true)} not added: ${error}, last transaction was: ${lastTransaction?.getConfirmedTransaction()?.toJson(true)}`) } } export async function addCommunityRootTransaction( blockchain: InMemoryBlockchain, community: Community, + accountBalances: AccountBalances ): Promise { const communityRootTransactionRole = new CommunityRootTransactionRole(community) if ( addToBlockchain( await communityRootTransactionRole.getGradidoTransactionBuilder(), blockchain, - new Timestamp(community.creationDate), + 0, + accountBalances, ) ) { logger.info(`Community Root Transaction added`) @@ -90,17 +102,17 @@ export async function addTransaction( senderBlockchain: InMemoryBlockchain, _recipientBlockchain: InMemoryBlockchain, transaction: Transaction, + transactionId: number, + accountBalances: AccountBalances, ): Promise { let debugTmpStr = '' - - const createdAtTimestamp = new Timestamp(transaction.createdAt) let role: AbstractTransactionRole if (transaction.type === InputTransactionType.GRADIDO_CREATION) { role = new CreationTransactionRole(transaction) } else if (transaction.type === InputTransactionType.GRADIDO_TRANSFER) { role = new TransferTransactionRole(transaction) - } else if (transaction.type == InputTransactionType.REGISTER_ADDRESS) { + } else if (transaction.type === InputTransactionType.REGISTER_ADDRESS) { role = new RegisterAddressTransactionRole(transaction) } else if (transaction.type === InputTransactionType.GRADIDO_DEFERRED_TRANSFER) { role = new DeferredTransferTransactionRole(transaction) @@ -117,7 +129,7 @@ export async function addTransaction( `redeem deferred transfer: couldn't find parent deferred transfer on Gradido Node for ${JSON.stringify(transaction, null, 2)} and public key from seed: ${f.involvedPublicKey?.convertToHex()}`, ) } - if (deferredTransactions.size() != 1) { + if (deferredTransactions.size() !== 1) { logger.error( `redeem deferred transfer: found ${deferredTransactions.size()} parent deferred transfer on Gradido Node for ${JSON.stringify(transaction, null, 2)} and public key from seed: ${f.involvedPublicKey?.convertToHex()}`, ) @@ -144,19 +156,20 @@ export async function addTransaction( const involvedUser = transaction.user.account ? transaction.user.account.userUuid : transaction.linkedUser?.account?.userUuid - if (addToBlockchain(await role.getGradidoTransactionBuilder(), senderBlockchain, createdAtTimestamp)) { - logger.debug(`${transaction.type} Transaction added for user ${involvedUser}`) - transactionAddedToBlockchainSum++ - } else { - logger.error(debugTmpStr) - /*const f = new Filter() - f.searchDirection = SearchDirection_DESC - f.pagination = new Pagination(15) - const transactions = senderBlockchain.findAll(f) - for(let i = transactions.size() - 1; i >= 0; i--) { - logger.error(`transaction ${i}: ${transactions.get(i)?.getConfirmedTransaction()?.toJson(true)}`) - }*/ - logger.error(`transaction: ${JSON.stringify(transaction, null, 2)}`) - throw new Error(`${transaction.type} Transaction not added for user ${involvedUser}, after ${transactionAddedToBlockchainSum} transactions`) + try { + if (addToBlockchain(await role.getGradidoTransactionBuilder(), senderBlockchain, transactionId, accountBalances)) { + // logger.debug(`${transaction.type} Transaction added for user ${involvedUser}`) + transactionAddedToBlockchainSum++ + } else { + logger.error(debugTmpStr) + logger.error(`transaction: ${JSON.stringify(transaction, null, 2)}`) + throw new Error(`${transaction.type} Transaction not added for user ${involvedUser}, after ${transactionAddedToBlockchainSum} transactions`) + } + } catch(e) { + if (e instanceof NotEnoughGradidoBalanceError) { + throw e + } + logger.error(`error adding transaction: ${JSON.stringify(transaction, null, 2)}`) + throw e } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts index 0f45d2197..a51f87f5d 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts @@ -1,17 +1,24 @@ -import { InMemoryBlockchainProvider } from 'gradido-blockchain-js' +import { AccountBalance, AccountBalances, GradidoUnit, InMemoryBlockchainProvider } from 'gradido-blockchain-js' import * as v from 'valibot' +import { KeyPairIdentifierLogic } from '../../data/KeyPairIdentifier.logic' +import { GradidoBlockchainCryptoError } from '../../errors' +import { ResolveKeyPair } from '../../interactions/resolveKeyPair/ResolveKeyPair.context' import { HieroId, hieroIdSchema } from '../../schemas/typeGuard.schema' +import { AUF_ACCOUNT_DERIVATION_INDEX, GMW_ACCOUNT_DERIVATION_INDEX, hardenDerivationIndex } from '../../utils/derivationHelper' import { addCommunityRootTransaction } from './blockchain' import { Context } from './Context' import { communityDbToCommunity } from './convert' -import { loadCommunities, loadContributionLinkModeratorCache } from './database' +import { loadAdminUsersCache, loadCommunities, loadContributionLinkModeratorCache } from './database' import { generateKeyPairCommunity } from './keyPair' import { CommunityContext } from './valibot.schema' export async function bootstrap(): Promise { const context = await Context.create() context.communities = await bootstrapCommunities(context) - await loadContributionLinkModeratorCache(context.db) + await Promise.all([ + loadContributionLinkModeratorCache(context.db), + loadAdminUsersCache(context.db) + ]) return context } @@ -50,7 +57,28 @@ async function bootstrapCommunities(context: Context): Promise() +export const contributionLinkModerators = new Map() +export const adminUsers = new Map() +const transactionIdSet = new Set() export async function loadContributionLinkModeratorCache(db: MySql2Database): Promise { const result = await db @@ -48,6 +53,20 @@ export async function loadContributionLinkModeratorCache(db: MySql2Database): Pr }) } +export async function loadAdminUsersCache(db: MySql2Database): Promise { + const result = await db + .select({ + user: usersTable, + }) + .from(userRolesTable) + .where(eq(userRolesTable.role, 'ADMIN')) + .leftJoin(usersTable, eq(userRolesTable.userId, usersTable.id)) + + result.map((row: any) => { + adminUsers.set(row.gradidoId, v.parse(createdUserDbSchema, row.user)) + }) +} + // queries export async function loadCommunities(db: MySql2Database): Promise { const result = await db @@ -96,23 +115,42 @@ export async function loadUsers( }) } +export async function loadUserByGradidoId(db: MySql2Database, gradidoId: string): Promise { + const result = await db + .select() + .from(usersTable) + .where(eq(usersTable.gradidoId, gradidoId)) + .limit(1) + + return result.length ? v.parse(createdUserDbSchema, result[0]) : null +} + export async function loadTransactions( db: MySql2Database, offset: number, count: number, ): Promise { const linkedUsers = alias(usersTable, 'linkedUser') + const linkedTransactions = alias(transactionsTable, 'linkedTransaction') const result = await db .select({ transaction: transactionsTable, user: usersTable, linkedUser: linkedUsers, - transactionLink: transactionLinksTable, + transactionLink: { + id: transactionLinksTable.id, + code: transactionLinksTable.code + }, + linkedUserBalance: linkedTransactions.balance, }) .from(transactionsTable) .where( - inArray(transactionsTable.typeId, [TransactionTypeId.CREATION, TransactionTypeId.RECEIVE]), + and( + inArray(transactionsTable.typeId, [TransactionTypeId.CREATION, TransactionTypeId.RECEIVE]), + isNotNull(transactionsTable.linkedUserId), + eq(usersTable.foreign, 0) + ) ) .leftJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) .leftJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id)) @@ -120,67 +158,25 @@ export async function loadTransactions( transactionLinksTable, eq(transactionsTable.transactionLinkId, transactionLinksTable.id), ) + .leftJoin(linkedTransactions, eq(transactionsTable.linkedTransactionId, linkedTransactions.id)) .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) .limit(count) .offset(offset) - return await Promise.all(result.map(async (row: any) => { + return result.map((row: any) => { // console.log(row) try { - const user = v.parse(createdUserDbSchema, row.user) - let linkedUser: CreatedUserDb | null | undefined = null - if (!row.linkedUser) { - const contribution = await db - .select({contributionLinkId: contributionsTable.contributionLinkId}) - .from(contributionsTable) - .where(eq(contributionsTable.transactionId, row.transaction.id)) - .limit(1) - if (contribution && contribution.length > 0 && contribution[0].contributionLinkId) { - linkedUser = contributionLinkModerators.get(contribution[0].contributionLinkId) - if (linkedUser?.gradidoId === user.gradidoId) { - const adminUser = await db - .select({ - user: usersTable - }) - .from(usersTable) - .leftJoin(userRolesTable, and(eq(usersTable.id, userRolesTable.userId), eq(userRolesTable.role, 'admin'))) - .orderBy(asc(userRolesTable.id)) - .where(ne(userRolesTable.userId, row.user.id)) - .limit(1) - if (!adminUser || !adminUser.length) { - throw new Error(`cannot find replace admin for contribution link`) - } - linkedUser = v.parse(createdUserDbSchema, adminUser[0].user) - } - } - } else { - linkedUser = v.parse(createdUserDbSchema, row.linkedUser) - } - if (!linkedUser) { - throw new Error(`linked user not found for transaction ${row.transaction.id}`) - } - - // check for consistent data beforehand - const balanceDate = new Date(row.transaction.balanceDate) - if ( - user.createdAt.getTime() > balanceDate.getTime() || - linkedUser?.createdAt.getTime() > balanceDate.getTime() - ) { - logger.error(`table row: `, row) - throw new Error( - 'at least one user was created after transaction balance date, logic error!', - ) - } - - let amount = GradidoUnit.fromString(row.transaction.amount) - if (row.transaction.typeId === TransactionTypeId.SEND) { - amount = amount.mul(new GradidoUnit(-1)) + /*if (transactionIdSet.has(row.transaction.id)) { + throw new Error(`transaction ${row.transaction.id} already loaded`) } + transactionIdSet.add(row.transaction.id) + */ return v.parse(transactionDbSchema, { ...row.transaction, transactionLinkCode: row.transactionLink ? row.transactionLink.code : null, - user, - linkedUser, + user: row.user, + linkedUser: row.linkedUser, + linkedUserBalance: row.linkedUserBalance, }) } catch (e) { logger.error(`table row: ${JSON.stringify(row, null, 2)}`) @@ -189,7 +185,108 @@ export async function loadTransactions( } throw e } - })) + }) +} + +export async function loadInvalidContributionTransactions( + db: MySql2Database, + offset: number, + count: number, +): Promise<{ id: number, balanceDate: Date }[]> { + const result = await db + .select({ + id: transactionsTable.id, + balanceDate: transactionsTable.balanceDate, + }) + .from(transactionsTable) + .where( + and( + eq(transactionsTable.typeId, TransactionTypeId.CREATION), + sql`NOT EXISTS (SELECT 1 FROM contributions WHERE contributions.transaction_id = transactions.id)`, + ) + ) + .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) + .limit(count) + .offset(offset) + + return result.map((row: any) => { + return { + id: row.id, + balanceDate: new Date(row.balanceDate), + } + }) +} + +export async function loadDoubleLinkedTransactions( + db: MySql2Database, + offset: number, + rowsCount: number, +): Promise<{ id: number, balanceDate: Date }[]> { + const result = await db + .select({ + id: transactionsTable.id, + balanceDate: transactionsTable.balanceDate, + transactionLinkId: transactionsTable.transactionLinkId, + cnt: count(), + }) + .from(transactionsTable) + .where( + and( + eq(transactionsTable.typeId, TransactionTypeId.RECEIVE), + isNotNull(transactionsTable.transactionLinkId), + ) + ) + .groupBy(transactionsTable.transactionLinkId) + .having(gt(count(), 1)) + .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) + .limit(rowsCount) + .offset(offset) + + // logger.info(`loadDoubleLinkedTransactions ${result.length}: ${timeUsed.string()}`) + + return result.map((row: any) => { + return { + id: row.transactionLinkId, + balanceDate: new Date(row.balanceDate), + } + }) +} + +export async function loadContributionLinkTransactions( + db: MySql2Database, + offset: number, + count: number, +): Promise<{ transaction: TransactionSelect, user: UserSelect, contributionLinkId: number }[]> { + const result = await db + .select({ + transaction: transactionsTable, + user: usersTable, + contributionLinkId: contributionsTable.contributionLinkId, + }) + .from(contributionsTable) + .where( + and( + isNotNull(contributionsTable.contributionLinkId), + isNull(transactionsTable.linkedUserId) + ) + ) + .leftJoin(transactionsTable, eq(contributionsTable.transactionId, transactionsTable.id)) + .leftJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) + .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) + .limit(count) + .offset(offset) + + return result.map((row: any) => { + if (transactionIdSet.has(row.transaction.id)) { + throw new Error(`transaction ${row.transaction.id} already loaded`) + } + transactionIdSet.add(row.transaction.id) + return { + transaction: v.parse(transactionSelectSchema, row.transaction), + user: v.parse(userSelectSchema, row.user), + contributionLinkId: row.contributionLinkId, + } + }) } export async function loadTransactionLinks( @@ -232,6 +329,7 @@ export async function loadDeletedTransactionLinks( return result.map((row: any) => { return v.parse(transactionDbSchema, { + id: row.transaction_links.id, typeId: TransactionTypeId.RECEIVE, amount: row.transaction_links.amount, balanceDate: new Date(row.transaction_links.deletedAt), diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts index c907c6c51..cb99275e9 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts @@ -1,6 +1,5 @@ import { sql } from 'drizzle-orm' import { - bigint, char, datetime, decimal, @@ -11,6 +10,8 @@ import { unique, varchar, } from 'drizzle-orm/mysql-core' +import { createSelectSchema } from 'drizzle-valibot' +import * as v from 'valibot' // use only fields needed in the migration, after update the rest of the project, import database instead export const communitiesTable = mysqlTable( @@ -45,6 +46,7 @@ export const usersTable = mysqlTable( 'users', { id: int().autoincrement().notNull(), + foreign: tinyint().default(0).notNull(), gradidoId: char('gradido_id', { length: 36 }).notNull(), communityUuid: varchar('community_uuid', { length: 36 }).default(sql`NULL`), createdAt: datetime('created_at', { mode: 'string', fsp: 3 }) @@ -54,6 +56,9 @@ export const usersTable = mysqlTable( (table) => [unique('uuid_key').on(table.gradidoId, table.communityUuid)], ) +export const userSelectSchema = createSelectSchema(usersTable) +export type UserSelect = v.InferOutput + export const userRolesTable = mysqlTable('user_roles', { id: int().autoincrement().notNull(), userId: int('user_id').notNull(), @@ -70,6 +75,7 @@ export const transactionsTable = mysqlTable( typeId: int('type_id').default(sql`NULL`), transactionLinkId: int('transaction_link_id').default(sql`NULL`), amount: decimal({ precision: 40, scale: 20 }).default(sql`NULL`), + balance: decimal({ precision: 40, scale: 20 }).default(sql`NULL`), balanceDate: datetime('balance_date', { mode: 'string', fsp: 3 }) .default(sql`current_timestamp(3)`) .notNull(), @@ -77,10 +83,14 @@ export const transactionsTable = mysqlTable( creationDate: datetime('creation_date', { mode: 'string', fsp: 3 }).default(sql`NULL`), userId: int('user_id').notNull(), linkedUserId: int('linked_user_id').default(sql`NULL`), + linkedTransactionId: int('linked_transaction_id').default(sql`NULL`), }, (table) => [index('user_id').on(table.userId)], ) +export const transactionSelectSchema = createSelectSchema(transactionsTable) +export type TransactionSelect = v.InferOutput + export const transactionLinksTable = mysqlTable('transaction_links', { id: int().autoincrement().notNull(), userId: int().notNull(), diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts new file mode 100644 index 000000000..85c5be50a --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts @@ -0,0 +1,6 @@ +export class NotEnoughGradidoBalanceError extends Error { + constructor(public needed: number, public exist: number) { + super(`Not enough Gradido Balance for send coins, needed: ${needed} Gradido, exist: ${exist} Gradido`) + } +} + \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts index 09da8abf8..9803429e2 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts @@ -4,7 +4,7 @@ import { exportAllCommunities } from './binaryExport' import { bootstrap } from './bootstrap' import { syncDbWithBlockchainContext } from './interaction/syncDbWithBlockchain/syncDbWithBlockchain.context' -const BATCH_SIZE = 250 +const BATCH_SIZE = 1000 async function main() { // prepare in memory blockchains diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts index 90523fa1b..9038e0e49 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts @@ -36,7 +36,7 @@ export abstract class AbstractSyncRole { return this.items.length } return 0 - } + } async toBlockchain(): Promise { if (this.isEmpty()) { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts new file mode 100644 index 000000000..27908fa19 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts @@ -0,0 +1,35 @@ +import * as v from 'valibot' +import { Context } from '../../Context' +import { adminUsers, contributionLinkModerators, loadContributionLinkTransactions } from '../../database' +import { CreatedUserDb, TransactionDb, transactionDbSchema } from '../../valibot.schema' +import { TransactionsSyncRole } from './TransactionsSync.role' + +export class ContributionLinkTransactionSyncRole extends TransactionsSyncRole { + constructor(readonly context: Context) { + super(context) + } + itemTypeName(): string { + return 'contributionLinkTransaction' + } + + async loadFromDb(offset: number, count: number): Promise { + const transactionUsers = await loadContributionLinkTransactions(this.context.db, offset, count) + return transactionUsers.map((transactionUser) => { + let linkedUser: CreatedUserDb | null | undefined = null + linkedUser = contributionLinkModerators.get(transactionUser.contributionLinkId) + if (linkedUser?.gradidoId === transactionUser.user.gradidoId) { + for (const adminUser of adminUsers.values()) { + if (adminUser.gradidoId !== transactionUser.user.gradidoId) { + linkedUser = adminUser + break + } + } + } + return v.parse(transactionDbSchema, { + ...transactionUser.transaction, + user: transactionUser.user, + linkedUser, + }) + }) + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DoubleLinkedTransactions.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DoubleLinkedTransactions.role.ts new file mode 100644 index 000000000..136155a2c --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DoubleLinkedTransactions.role.ts @@ -0,0 +1,27 @@ +import { Context } from '../../Context' +import { loadDoubleLinkedTransactions } from '../../database' +import { AbstractSyncRole } from './AbstractSync.role' + +export class DoubleLinkedTransactionsSyncRole extends AbstractSyncRole<{ id: number, balanceDate: Date }> { + static allTransactionIds: number[] = [] + constructor(readonly context: Context) { + super(context) + } + itemTypeName(): string { + return 'doubleLinkedTransaction' + } + + async loadFromDb(offset: number, count: number): Promise<{ id: number, balanceDate: Date }[]> { + const result = await loadDoubleLinkedTransactions(this.context.db, offset, count) + DoubleLinkedTransactionsSyncRole.allTransactionIds.push(...result.map((r) => r.id)) + return result + } + + getDate(): Date { + return this.peek().balanceDate + } + + async pushToBlockchain(item: { id: number, balanceDate: Date }): Promise { + this.logger.warn(`Double transaction_links ${item.id} found.`) + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/InvalidContributionTransactionSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/InvalidContributionTransactionSync.role.ts new file mode 100644 index 000000000..616dd1fe4 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/InvalidContributionTransactionSync.role.ts @@ -0,0 +1,27 @@ +import { Context } from '../../Context' +import { loadInvalidContributionTransactions } from '../../database' +import { AbstractSyncRole } from './AbstractSync.role' + +export class InvalidContributionTransactionSyncRole extends AbstractSyncRole<{ id: number, balanceDate: Date }> { + static allTransactionIds: number[] = [] + constructor(readonly context: Context) { + super(context) + } + itemTypeName(): string { + return 'invalidContributionTransaction' + } + + async loadFromDb(offset: number, count: number): Promise<{ id: number, balanceDate: Date }[]> { + const result = await loadInvalidContributionTransactions(this.context.db, offset, count) + InvalidContributionTransactionSyncRole.allTransactionIds.push(...result.map((r) => r.id)) + return result + } + + getDate(): Date { + return this.peek().balanceDate + } + + async pushToBlockchain(item: { id: number, balanceDate: Date }): Promise { + this.logger.warn(`Invalid contribution transaction ${item.id} found.`) + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinksSync.role.ts index 847be89ba..6920732cc 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinksSync.role.ts @@ -1,3 +1,6 @@ +import { AccountBalance, AccountBalances, Filter, MemoryBlockPtr, SearchDirection_DESC } from 'gradido-blockchain-js' +import { KeyPairIdentifierLogic } from '../../../../data/KeyPairIdentifier.logic' +import { ResolveKeyPair } from '../../../../interactions/resolveKeyPair/ResolveKeyPair.context' import { addTransaction } from '../../blockchain' import { transactionLinkDbToTransaction } from '../../convert' import { loadTransactionLinks } from '../../database' @@ -20,6 +23,23 @@ export class TransactionLinksSyncRole extends AbstractSyncRole { const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) const transaction = transactionLinkDbToTransaction(item, communityContext.topicId) - await addTransaction(communityContext.blockchain, communityContext.blockchain, transaction) + // I use the receiving part of transaction pair, so the user is the recipient and the linked user the sender + const accountBalances = new AccountBalances() + const senderKeyPair = await ResolveKeyPair( + new KeyPairIdentifierLogic(transaction.user), + ) + const recipientKeyPair = await ResolveKeyPair( + new KeyPairIdentifierLogic(transaction.linkedUser!), + ) + const f = new Filter() + f.involvedPublicKey = senderKeyPair.getPublicKey() + f.pagination.size = 1 + f.searchDirection = SearchDirection_DESC + communityContext.blockchain.findOne(f) + accountBalances.add(new AccountBalance(senderKeyPair.getPublicKey(), item.linkedUserBalance, '')) + accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), item.amount, '')) + + + await addTransaction(communityContext.blockchain, communityContext.blockchain, transaction, item.id, accountBalances) } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionsSync.role.ts index df8709ac8..43e6c2cd7 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionsSync.role.ts @@ -1,10 +1,28 @@ +import Decimal from 'decimal.js-light' +import { AccountBalance, AccountBalances, GradidoUnit } from 'gradido-blockchain-js' +import { InputTransactionType } from '../../../../data/InputTransactionType.enum' +import { KeyPairIdentifierLogic } from '../../../../data/KeyPairIdentifier.logic' +import { GradidoBlockchainCryptoError } from '../../../../errors' +import { ResolveKeyPair } from '../../../../interactions/resolveKeyPair/ResolveKeyPair.context' +import { AUF_ACCOUNT_DERIVATION_INDEX, GMW_ACCOUNT_DERIVATION_INDEX, hardenDerivationIndex } from '../../../../utils/derivationHelper' import { addTransaction } from '../../blockchain' import { transactionDbToTransaction } from '../../convert' import { loadTransactions } from '../../database' +import { legacyCalculateDecay } from '../../utils' import { TransactionDb } from '../../valibot.schema' import { AbstractSyncRole } from './AbstractSync.role' +type BalanceDate = { + balance: Decimal + date: Date +} + export class TransactionsSyncRole extends AbstractSyncRole { + private static transactionLinkCodes = new Set() + static doubleTransactionLinkCodes: string[] = [] + static gmwBalance: BalanceDate | undefined = undefined + static aufBalance: BalanceDate | undefined = undefined + getDate(): Date { return this.peek().balanceDate } @@ -14,7 +32,34 @@ export class TransactionsSyncRole extends AbstractSyncRole { } async loadFromDb(offset: number, count: number): Promise { - return await loadTransactions(this.context.db, offset, count) + const result = await loadTransactions(this.context.db, offset, count) + return result.filter((item) => { + if (item.transactionLinkCode) { + if (TransactionsSyncRole.transactionLinkCodes.has(item.transactionLinkCode)) { + TransactionsSyncRole.doubleTransactionLinkCodes.push(item.transactionLinkCode) + return false + } + TransactionsSyncRole.transactionLinkCodes.add(item.transactionLinkCode) + } + return true + }) + } + + updateGmwAuf(amount: Decimal, date: Date) { + if(!TransactionsSyncRole.gmwBalance) { + TransactionsSyncRole.gmwBalance = { balance: amount, date } + } else { + const oldGmwBalanceDate = TransactionsSyncRole.gmwBalance + const newBalance = legacyCalculateDecay(oldGmwBalanceDate.balance, oldGmwBalanceDate.date, date ) + TransactionsSyncRole.gmwBalance = { balance: newBalance, date } + } + if(!TransactionsSyncRole.aufBalance) { + TransactionsSyncRole.aufBalance = { balance: amount, date } + } else { + const oldAufBalanceDate = TransactionsSyncRole.aufBalance + const newBalance = legacyCalculateDecay(oldAufBalanceDate.balance, oldAufBalanceDate.date, date ) + TransactionsSyncRole.aufBalance = { balance: newBalance, date } + } } async pushToBlockchain(item: TransactionDb): Promise { @@ -28,10 +73,63 @@ export class TransactionsSyncRole extends AbstractSyncRole { senderCommunityContext.topicId, recipientCommunityContext.topicId, ) - await addTransaction( - senderCommunityContext.blockchain, - recipientCommunityContext.blockchain, - transaction, - ) + const accountBalances = new AccountBalances() + if (InputTransactionType.GRADIDO_CREATION === transaction.type) { + const recipientKeyPair = await ResolveKeyPair( + new KeyPairIdentifierLogic(transaction.linkedUser!), + ) + accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), item.balance, '')) + // update gmw and auf + this.updateGmwAuf(new Decimal(item.amount.toString(4)), item.balanceDate) + const communityKeyPair = await ResolveKeyPair(new KeyPairIdentifierLogic({ communityTopicId: senderCommunityContext.topicId })) + const gmwKeyPair = communityKeyPair.deriveChild( + hardenDerivationIndex(GMW_ACCOUNT_DERIVATION_INDEX), + ) + if (!gmwKeyPair) { + throw new GradidoBlockchainCryptoError( + `KeyPairEd25519 child derivation failed, has private key: ${communityKeyPair.hasPrivateKey()} for community: ${senderCommunityContext.communityId}`, + ) + } + const aufKeyPair = communityKeyPair.deriveChild( + hardenDerivationIndex(AUF_ACCOUNT_DERIVATION_INDEX), + ) + if (!aufKeyPair) { + throw new GradidoBlockchainCryptoError( + `KeyPairEd25519 child derivation failed, has private key: ${communityKeyPair.hasPrivateKey()} for community: ${senderCommunityContext.communityId}`, + ) + } + accountBalances.add(new AccountBalance(gmwKeyPair.getPublicKey(), GradidoUnit.fromString( + TransactionsSyncRole.gmwBalance!.balance.toString()), '')) + accountBalances.add(new AccountBalance(aufKeyPair.getPublicKey(), GradidoUnit.fromString( + TransactionsSyncRole.aufBalance!.balance.toString()), '')) + } else if (InputTransactionType.REGISTER_ADDRESS === transaction.type) { + const recipientKeyPair = await ResolveKeyPair( + new KeyPairIdentifierLogic(transaction.user), + ) + accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), GradidoUnit.zero(), '')) + } else { + // I use the receiving part of transaction pair, so the user is the recipient and the linked user the sender + const senderKeyPair = await ResolveKeyPair( + new KeyPairIdentifierLogic(transaction.linkedUser!), + ) + const recipientKeyPair = await ResolveKeyPair( + new KeyPairIdentifierLogic(transaction.user), + ) + accountBalances.add(new AccountBalance(senderKeyPair.getPublicKey(), item.linkedUserBalance, '')) + accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), item.balance, '')) + } + + try { + await addTransaction( + senderCommunityContext.blockchain, + recipientCommunityContext.blockchain, + transaction, + item.id, + accountBalances, + ) + } catch(e) { + this.context.logger.error(`error adding transaction: ${JSON.stringify(transaction, null, 2)}`) + throw e + } } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts index 4c1b9ca2c..0d6c79b38 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts @@ -26,6 +26,6 @@ export class UsersSyncRole extends AbstractSyncRole { async pushToBlockchain(item: CreatedUserDb): Promise { const communityContext = this.context.getCommunityContextByUuid(item.communityUuid) const transaction = userDbToTransaction(item, communityContext.topicId) - return await addTransaction(communityContext.blockchain, communityContext.blockchain, transaction) + return await addTransaction(communityContext.blockchain, communityContext.blockchain, transaction, item.id) } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts index c62187181..53e0fcf39 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts @@ -1,6 +1,8 @@ import { Profiler } from 'gradido-blockchain-js' import { Context } from '../../Context' +import { ContributionLinkTransactionSyncRole } from './ContributionLinkTransactionSync.role' import { DeletedTransactionLinksSyncRole } from './DeletedTransactionLinksSync.role' +import { InvalidContributionTransactionSyncRole } from './InvalidContributionTransactionSync.role' import { TransactionLinksSyncRole } from './TransactionLinksSync.role' import { TransactionsSyncRole } from './TransactionsSync.role' import { UsersSyncRole } from './UsersSync.role' @@ -8,17 +10,21 @@ import { UsersSyncRole } from './UsersSync.role' export async function syncDbWithBlockchainContext(context: Context, batchSize: number) { const timeUsedDB = new Profiler() const timeUsedBlockchain = new Profiler() + const timeUsedAll = new Profiler() const containers = [ new UsersSyncRole(context), new TransactionsSyncRole(context), new DeletedTransactionLinksSyncRole(context), new TransactionLinksSyncRole(context), + new InvalidContributionTransactionSyncRole(context), + new ContributionLinkTransactionSyncRole(context), ] let transactionsCount = 0 let transactionsCountSinceLastLog = 0 + let available = containers while (true) { timeUsedDB.reset() - const results = await Promise.all(containers.map((c) => c.ensureFilled(batchSize))) + const results = await Promise.all(available.map((c) => c.ensureFilled(batchSize))) const loadedItemsCount = results.reduce((acc, c) => acc + c, 0) // log only, if at least one new item was loaded if (loadedItemsCount && context.logger.isDebugEnabled()) { @@ -26,7 +32,7 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n } // remove empty containers - const available = containers.filter((c) => !c.isEmpty()) + available = available.filter((c) => !c.isEmpty()) if (available.length === 0) { break } @@ -46,4 +52,13 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n } } process.stdout.write(`\n`) + context.logger.info(`Synced ${transactionsCount} transactions to blockchain in ${(timeUsedAll.seconds() / 60).toFixed(2)} minutes`) + context.logger.info(`Invalid contribution transactions: ${InvalidContributionTransactionSyncRole.allTransactionIds.length}`) + if (context.logger.isDebugEnabled()) { + context.logger.debug(InvalidContributionTransactionSyncRole.allTransactionIds.join(', ')) + } + context.logger.info(`Double linked transactions: ${TransactionsSyncRole.doubleTransactionLinkCodes.length}`) + if (context.logger.isDebugEnabled()) { + context.logger.debug(TransactionsSyncRole.doubleTransactionLinkCodes.join(', ')) + } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts index c9b4eccb9..dc44af15f 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts @@ -1,3 +1,4 @@ +import Decimal from 'decimal.js-light' import { crypto_generichash_batch, crypto_generichash_KEYBYTES } from 'sodium-native' export function bytesToMbyte(bytes: number): string { @@ -13,3 +14,39 @@ export function calculateOneHashStep(hash: Buffer, data: Buffer): Buffer toMs) { + return amount + } + // decay started before start date; decay for full duration + let duration = (toMs - fromMs) / 1000 + + // decay started between start and end date; decay from decay start till end date + if (startBlockMs >= fromMs) { + duration = (toMs - startBlockMs) / 1000 + } + return legacyDecayFormula(amount, duration) +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts index 1fc99ec3b..f7d66b6ff 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts @@ -11,6 +11,7 @@ import { import { TransactionTypeId } from './TransactionTypeId' export const createdUserDbSchema = v.object({ + id: v.pipe(v.number(), v.minValue(1)), gradidoId: uuidv4Schema, communityUuid: uuidv4Schema, createdAt: dateSchema, @@ -22,22 +23,37 @@ export const userDbSchema = v.object({ }) export const transactionDbSchema = v.pipe(v.object({ + id: v.pipe(v.number(), v.minValue(1)), typeId: v.enum(TransactionTypeId), amount: gradidoAmountSchema, balanceDate: dateSchema, + balance: gradidoAmountSchema, + linkedUserBalance: gradidoAmountSchema, memo: memoSchema, creationDate: v.nullish(dateSchema), - user: userDbSchema, - linkedUser: userDbSchema, + user: createdUserDbSchema, + linkedUser: createdUserDbSchema, transactionLinkCode: v.nullish(identifierSeedSchema), }), v.custom((value: any) => { if (value.user && value.linkedUser && !value.transactionLinkCode && value.user.gradidoId === value.linkedUser.gradidoId) { throw new Error(`expect user to be different from linkedUser: ${JSON.stringify(value, null, 2)}`) } + // check that user and linked user exist before transaction balance date + const balanceDate = new Date(value.balanceDate) + if ( + value.user.createdAt.getTime() >= balanceDate.getTime() || + value.linkedUser?.createdAt.getTime() >= balanceDate.getTime() + ) { + throw new Error( + `at least one user was created after transaction balance date, logic error! ${JSON.stringify(value, null, 2)}`, + ) + } + return value })) export const transactionLinkDbSchema = v.object({ + id: v.pipe(v.number(), v.minValue(1)), user: userDbSchema, code: identifierSeedSchema, amount: gradidoAmountSchema, @@ -62,7 +78,7 @@ export const communityContextSchema = v.object({ folder: v.pipe( v.string(), v.minLength(1, 'expect string length >= 1'), - v.maxLength(255, 'expect string length <= 255'), + v.maxLength(512, 'expect string length <= 512'), v.regex(/^[a-zA-Z0-9-_]+$/, 'expect string to be a valid (alphanumeric, _, -) folder name'), ), }) diff --git a/dlt-connector/src/schemas/typeGuard.schema.ts b/dlt-connector/src/schemas/typeGuard.schema.ts index 3a93ac62f..1f9f12eaf 100644 --- a/dlt-connector/src/schemas/typeGuard.schema.ts +++ b/dlt-connector/src/schemas/typeGuard.schema.ts @@ -163,7 +163,7 @@ export type HieroTransactionIdInput = v.InferInput