From 51196f3941fe2a2ee396e082fc218056445b2189 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 6 Dec 2025 14:13:14 +0100 Subject: [PATCH 01/50] migrate production data to make them blockchain compatible --- ...097-fix_production_data_for_blockchain2.ts | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 database/migration/migrations/0097-fix_production_data_for_blockchain2.ts diff --git a/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts b/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts new file mode 100644 index 000000000..8e62b0dc6 --- /dev/null +++ b/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts @@ -0,0 +1,172 @@ +export async function upgrade(queryFn: (query: string, values?: any[]) => Promise>) { + /** + * Migration: Correct historical inconsistencies in transactions and users. + * + * Background: + * Early Gradido production data contains several inconsistencies that violate + * stricter blockchain validation rules. These inconsistencies include: + * - Contribution transactions confirmed by users who did not exist at the + * time of the transaction. + * - Users confirming their own contributions (self-signed transactions). + * - Users whose `created_at` timestamp is after or equal to their first + * transaction. + * - Transaction memos shorter than the required minimum length (5 characters). + * + * Purpose: + * This migration performs the following corrections to ensure historical + * consistency and full compatibility with blockchain validation rules: + * 1. Fix self-signed contributions by assigning the actual moderator. + * 2. Replace invalid moderators with the earliest ADMIN or MODERATOR where + * the linked user was created after the transaction. + * 3. Update user creation dates to be before their first transaction. + * 4. Ensure all transaction memos meet the minimum length requirement. + * + * Outcome: + * After this migration, all contribution transactions reference a valid + * moderator existing at the time of the transaction, user creation dates are + * logically consistent, and memos meet the minimum formatting rules. + */ + + /** + * Fix 1: Remove self-signed contributions. + * + * Background: + * A core rule in the system states that *no user may confirm their own + * contribution* — a moderator must always be someone else. + * + * However, early production data contains transactions where the `linked_user_id` + * matches the `user_id`, meaning the contributor confirmed their own contribution. + * + * This query corrects those records by replacing the `linked_user` with the + * moderator stored in `contributions.moderator_id`. + * + * Only transactions where: + * - the type is a contribution (type_id = 1), + * - the linked user equals the contributor (`t.user_id = t.linked_user_id`), + * - the moderator existed before the time of the transaction, + * - and the moderator is not the same person, + * are updated. + */ + await queryFn(` + UPDATE transactions t + JOIN contributions c ON(t.id = c.transaction_id) + JOIN users u ON(c.moderator_id = u.id) + SET t.linked_user_id = u.id, + t.linked_user_community_uuid = u.community_uuid, + t.linked_user_gradido_id = u.gradido_id, + t.linked_user_name = CONCAT(u.first_name, ' ', u.last_name) + WHERE t.type_id = 1 + AND t.user_id = t.linked_user_id + AND u.created_at < t.balance_date + AND t.user_id <> u.id + ;`) + + /** + * Fix 2: Replace invalid moderators with the earliest ADMIN. + * + * Background: + * Early production records contain contribution transactions where the assigned + * moderator ("linked_user") was created *after* the contribution itself. This + * is invalid in the blockchain verification process, which requires that the + * moderator account must have existed *before* the time of the transaction. + * + * This migration: + * 1. Identifies the earliest ADMIN or MODERATOR user in the system. + * 2. Reassigns them as moderator for all affected transactions where: + * - the type is a contribution (type_id = 1), + * - the linked user was created after or at the transaction date, + * - the transaction occurred after the ADMIN’s or MODERATOR's creation, + * - and the contributor is not the ADMIN or MODERATOR. + * + * Using the earliest ADMIN or MODERATOR ensures: + * - historical consistency, + * - minimal intrusion, + * - and compatibility with blockchain validation rules. + */ + await queryFn(` + UPDATE transactions t + JOIN ( + SELECT u.created_at, u.id, u.community_uuid, u.gradido_id, CONCAT(u.first_name, ' ', u.last_name) AS linked_user_name + FROM users u + JOIN user_roles r ON u.id = r.user_id + WHERE r.role IN ('ADMIN', 'MODERATOR') + ORDER BY r.created_at ASC + LIMIT 1 + ) mod ON 1=1 + LEFT JOIN users u on(t.linked_user_id = u.id) + SET t.linked_user_id = mod.id, + t.linked_user_community_uuid = mod.community_uuid, + t.linked_user_gradido_id = mod.gradido_id, + t.linked_user_name = mod.linked_user_name + WHERE t.type_id = 1 + AND t.balance_date >= u.created_at + AND t.balance_date < mod.created_at + AND t.user_id <> mod.id + ;`) + + /** + * Fix 3: Update user creation dates to ensure historical consistency. + * + * Background: + * In early production data, some users have a `created_at` timestamp that is + * **after or equal** to their first recorded transaction (`balance_date`). + * This violates logical consistency, because a user cannot exist *after* their + * own transaction. + * + * What this query does: + * - For each user, it finds the earliest transaction date (`first_date`) from + * the `transactions` table. + * - It updates the user's `created_at` timestamp to **1 second before** their + * first transaction. + * + * Notes: + * - Only users where `created_at >= first transaction date` are affected. + * - This is a historical data fix to ensure all transactions reference a user + * that already exists at the time of the transaction, which is required for + * blockchain validation and logical consistency in the system. + */ + await queryFn(` + UPDATE users u + LEFT JOIN ( + SELECT user_id, MIN(balance_date) AS first_date + FROM transactions + GROUP BY user_id + ) t ON t.user_id = u.id + SET u.created_at = DATE_SUB(t.first_date, INTERVAL 1 SECOND) + WHERE u.created_at >= t.first_date; + ;`) + + /** + * Ensure all transaction memos meet the minimum length requirement. + * + * Background: + * In early Gradido production data, some transactions have a `memo` field + * shorter than the current rule of 5 characters. This can cause issues in + * reporting, display, or blockchain validation processes that expect + * a minimum memo length. + * + * What this query does: + * - For memos with 0 characters, sets the value to 'empty empty'. + * - For memos with 1-4 characters, pads the memo on the left with spaces + * until it reaches 5 characters. + * - Memos that are already 5 characters or longer are left unchanged. + * + * Notes: + * - This ensures all memos are at least 5 characters long. + * - The padding uses spaces. + * - Only memos shorter than 5 characters are affected. + */ + await queryFn(` + UPDATE transactions t + SET t.memo = CASE + WHEN CHAR_LENGTH(t.memo) = 0 THEN 'empty empty' + WHEN CHAR_LENGTH(t.memo) < 5 THEN LPAD(t.memo, 5, ' ') + ELSE t.memo + END + WHERE CHAR_LENGTH(t.memo) < 5 + ;`) +} + +export async function downgrade(queryFn: (query: string, values?: any[]) => Promise>) { + // downgrade not possible +} From 55ef47f42c6e265dbd5d5c929c51545abeae9ed8 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 7 Dec 2025 09:17:37 +0100 Subject: [PATCH 02/50] update dlt-connector bun.lock --- dlt-connector/bun.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/dlt-connector/bun.lock b/dlt-connector/bun.lock index 1d12dcaf3..b6747359b 100644 --- a/dlt-connector/bun.lock +++ b/dlt-connector/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "dlt-connector", From 0936e87d86009d0ea9b4a7a52fd2ed8f82a52c92 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 7 Dec 2025 09:26:32 +0100 Subject: [PATCH 03/50] replace reserved mysql word mod with moderator --- .../0097-fix_production_data_for_blockchain2.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts b/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts index 8e62b0dc6..f34c755ec 100644 --- a/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts +++ b/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts @@ -92,16 +92,16 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis WHERE r.role IN ('ADMIN', 'MODERATOR') ORDER BY r.created_at ASC LIMIT 1 - ) mod ON 1=1 + ) moderator ON 1=1 LEFT JOIN users u on(t.linked_user_id = u.id) - SET t.linked_user_id = mod.id, - t.linked_user_community_uuid = mod.community_uuid, - t.linked_user_gradido_id = mod.gradido_id, - t.linked_user_name = mod.linked_user_name + SET t.linked_user_id = moderator.id, + t.linked_user_community_uuid = moderator.community_uuid, + t.linked_user_gradido_id = moderator.gradido_id, + t.linked_user_name = moderator.linked_user_name WHERE t.type_id = 1 AND t.balance_date >= u.created_at - AND t.balance_date < mod.created_at - AND t.user_id <> mod.id + AND t.balance_date < moderator.created_at + AND t.user_id <> moderator.id ;`) /** From 6739fa8cfe226d3b1b3725f84cf17da8ca263c5e Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 7 Dec 2025 10:49:37 +0100 Subject: [PATCH 04/50] fix migration mysql query --- ...097-fix_production_data_for_blockchain2.ts | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts b/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts index f34c755ec..34a45bf89 100644 --- a/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts +++ b/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts @@ -66,7 +66,7 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis * * Background: * Early production records contain contribution transactions where the assigned - * moderator ("linked_user") was created *after* the contribution itself. This + * moderator ("linked_user" or "contribution.moderator_id") was created *after* the contribution itself. This * is invalid in the blockchain verification process, which requires that the * moderator account must have existed *before* the time of the transaction. * @@ -86,21 +86,22 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis await queryFn(` UPDATE transactions t JOIN ( - SELECT u.created_at, u.id, u.community_uuid, u.gradido_id, CONCAT(u.first_name, ' ', u.last_name) AS linked_user_name - FROM users u - JOIN user_roles r ON u.id = r.user_id - WHERE r.role IN ('ADMIN', 'MODERATOR') - ORDER BY r.created_at ASC - LIMIT 1 - ) moderator ON 1=1 + SELECT t_sub.id as sub_t_id, u_sub.created_at, u_sub.id, u_sub.community_uuid, u_sub.gradido_id, CONCAT(u_sub.first_name, ' ', u_sub.last_name) AS linked_user_name + FROM transactions t_sub + JOIN users u_sub on(t_sub.user_id <> u_sub.id) + JOIN user_roles r_sub ON u_sub.id = r_sub.user_id + WHERE r_sub.role IN ('ADMIN', 'MODERATOR') + GROUP BY t_sub.id + ORDER BY r_sub.created_at ASC + ) moderator ON (t.id = moderator.sub_t_id) LEFT JOIN users u on(t.linked_user_id = u.id) SET t.linked_user_id = moderator.id, t.linked_user_community_uuid = moderator.community_uuid, t.linked_user_gradido_id = moderator.gradido_id, t.linked_user_name = moderator.linked_user_name WHERE t.type_id = 1 - AND t.balance_date >= u.created_at - AND t.balance_date < moderator.created_at + AND t.balance_date <= u.created_at + AND t.balance_date > moderator.created_at AND t.user_id <> moderator.id ;`) From 9c82c1469c01255d9fa2fdff696b92441b983a13 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 7 Dec 2025 19:14:53 +0100 Subject: [PATCH 05/50] add ADMIN_CONTRIBUTION_LINK_CREATE events for contribution_links --- ...097-fix_production_data_for_blockchain2.ts | 45 ++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts b/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts index 34a45bf89..004f5ccd3 100644 --- a/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts +++ b/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts @@ -1,6 +1,6 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promise>) { /** - * Migration: Correct historical inconsistencies in transactions and users. + * Migration: Correct historical inconsistencies in transactions, users, and contribution_links. * * Background: * Early Gradido production data contains several inconsistencies that violate @@ -11,6 +11,8 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis * - Users whose `created_at` timestamp is after or equal to their first * transaction. * - Transaction memos shorter than the required minimum length (5 characters). + * - Existing contribution_links without an associated 'ADMIN_CONTRIBUTION_LINK_CREATE' event, + * which is used to find someone who confirmed the contribution. * * Purpose: * This migration performs the following corrections to ensure historical @@ -20,11 +22,16 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis * the linked user was created after the transaction. * 3. Update user creation dates to be before their first transaction. * 4. Ensure all transaction memos meet the minimum length requirement. + * 5. Insert missing 'ADMIN_CONTRIBUTION_LINK_CREATE' events for contribution_links + * that do not yet have such events, using the first Admin as acting_user. * * Outcome: - * After this migration, all contribution transactions reference a valid - * moderator existing at the time of the transaction, user creation dates are - * logically consistent, and memos meet the minimum formatting rules. + * After this migration: + * - All contribution transactions reference a valid moderator existing at the time of the transaction. + * - User creation dates are logically consistent with their transactions. + * - Transaction memos meet the minimum formatting rules. + * - Every contribution_link has a corresponding 'ADMIN_CONTRIBUTION_LINK_CREATE' event, + * ensuring blockchain consistency for contributions. */ /** @@ -138,7 +145,7 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis ;`) /** - * Ensure all transaction memos meet the minimum length requirement. + * Fix 4: Ensure all transaction memos meet the minimum length requirement. * * Background: * In early Gradido production data, some transactions have a `memo` field @@ -166,6 +173,34 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis END WHERE CHAR_LENGTH(t.memo) < 5 ;`) + + /** + * Fix 5: Insert missing 'ADMIN_CONTRIBUTION_LINK_CREATE' events for contribution_links. + * + * Background: + * Each contribution in the blockchain requires a confirmation by a user. + * In the current DB version, there is no information about who confirmed contributions based on contribution_links. + * Recently, functionality was added to create an 'ADMIN_CONTRIBUTION_LINK_CREATE' event + * for newly created contribution_links, but existing contribution_links were not updated. + * + * This query inserts an 'ADMIN_CONTRIBUTION_LINK_CREATE' event for every contribution_link + * that does not already have such an event. + * The acting_user_id is set to the first Admin, and affected_user_id is set to 0. + */ + await queryFn(` + INSERT INTO \`events\`(acting_user_id, affected_user_id, \`type\`, involved_contribution_link_id) + SELECT ( + SELECT u.id + FROM users u + JOIN user_roles r ON r.user_id = u.id + WHERE r.role = 'ADMIN' + ORDER BY r.id ASC + LIMIT 1 + ) AS acting_user_id, 0 as affected_user_id, 'ADMIN_CONTRIBUTION_LINK_CREATE' AS \`type\`, c.id AS involved_contribution_link_id + FROM contribution_links c + LEFT JOIN \`events\` e ON e.involved_contribution_link_id = c.id AND e.type = 'ADMIN_CONTRIBUTION_LINK_CREATE' + WHERE e.id IS NULL + ;`) } export async function downgrade(queryFn: (query: string, values?: any[]) => Promise>) { From 7835553200897d4812b2afb43686fd4c1b1a1919 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 7 Dec 2025 19:15:24 +0100 Subject: [PATCH 06/50] fix timezone --- dlt-connector/package.json | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/dlt-connector/package.json b/dlt-connector/package.json index 4c90e7b5b..87566cbd2 100644 --- a/dlt-connector/package.json +++ b/dlt-connector/package.json @@ -7,17 +7,18 @@ "license": "Apache-2.0", "private": true, "scripts": { - "start": "bun run src/index.ts", + "start": "cross-env TZ=UTC bun run src/index.ts", "build": "bun build src/index.ts --outdir=build --target=bun --external=gradido-blockchain-js --minify", - "dev": "bun run --watch src/index.ts", - "migrate": "bun src/migrations/db-v2.7.0_to_blockchain-v3.5", - "test": "bun test", - "test:debug": "bun test --inspect-brk", + "dev": "cross-env TZ=UTC bun run --watch src/index.ts", + "migrate": "cross-env TZ=UTC bun src/migrations/db-v2.7.0_to_blockchain-v3.5", + "test": "cross-env TZ=UTC bun test", + "test:debug": "cross-env TZ=UTC bun test --inspect-brk", "typecheck": "tsc --noEmit", "lint": "biome check --error-on-warnings .", "lint:fix": "biome check --error-on-warnings . --write" }, "dependencies": { + "cross-env": "^7.0.3", "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#f265dbb1780a912cf8b0418dfe3eaf5cdc5b51cf" }, "devDependencies": { From 0ac776fc1493d79e481f2a186a7783f8224b217e Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 7 Dec 2025 19:16:11 +0100 Subject: [PATCH 07/50] handle more edge cases, print finished transaction count with carriage-return --- .../blockchain.ts | 140 +++++++----------- .../db-v2.7.0_to_blockchain-v3.5/bootstrap.ts | 3 +- .../db-v2.7.0_to_blockchain-v3.5/database.ts | 59 ++++++-- .../drizzle.schema.ts | 17 +++ .../db-v2.7.0_to_blockchain-v3.5/index.ts | 2 +- .../syncDbWithBlockchain/UsersSync.role.ts | 4 +- .../syncDbWithBlockchain.context.ts | 21 ++- .../valibot.schema.ts | 9 +- 8 files changed, 148 insertions(+), 107 deletions(-) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts index fd89d5ea0..fceecf541 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts @@ -5,10 +5,13 @@ import { HieroTransactionId, InMemoryBlockchain, InteractionSerialize, + Pagination, + Profiler, + SearchDirection_DESC, Timestamp, + TransactionType_DEFERRED_TRANSFER, } from 'gradido-blockchain-js' import { getLogger } from 'log4js' -import * as v from 'valibot' import { LOG4JS_BASE_CATEGORY } from '../../config/const' import { InputTransactionType } from '../../data/InputTransactionType.enum' import { LinkedTransactionKeyPairRole } from '../../interactions/resolveKeyPair/LinkedTransactionKeyPair.role' @@ -20,11 +23,14 @@ import { RegisterAddressTransactionRole } from '../../interactions/sendToHiero/R import { TransferTransactionRole } from '../../interactions/sendToHiero/TransferTransaction.role' import { Community, Transaction } from '../../schemas/transaction.schema' import { identifierSeedSchema } from '../../schemas/typeGuard.schema' +import { AbstractTransactionRole } from '../../interactions/sendToHiero/AbstractTransaction.role' +import * as v from 'valibot' const logger = getLogger( `${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`, ) export const defaultHieroAccount = new HieroAccountId(0, 0, 2) +let transactionAddedToBlockchainSum = 0 function addToBlockchain( builder: GradidoTransactionBuilder, @@ -36,7 +42,7 @@ function addToBlockchain( const transactionId = new HieroTransactionId(createdAtTimestamp, defaultHieroAccount) const interactionSerialize = new InteractionSerialize(transactionId) - try { + try { const result = blockchain.createAndAddConfirmedTransaction( transaction, interactionSerialize.run(), @@ -67,115 +73,77 @@ export async function addCommunityRootTransaction( } } -export async function addRegisterAddressTransaction( - blockchain: InMemoryBlockchain, - transaction: Transaction, -): Promise { - const registerAddressRole = new RegisterAddressTransactionRole(transaction) - if ( - addToBlockchain( - await registerAddressRole.getGradidoTransactionBuilder(), - blockchain, - new Timestamp(transaction.createdAt), - ) - ) { - logger.debug( - `Register Address Transaction added for user ${transaction.user.account!.userUuid}`, - ) - } else { - throw new Error( - `Register Address Transaction not added for user ${transaction.user.account!.userUuid}`, - ) - } -} - export async function addTransaction( senderBlockchain: InMemoryBlockchain, _recipientBlockchain: InMemoryBlockchain, transaction: Transaction, ): Promise { + + let debugTmpStr = '' + const createdAtTimestamp = new Timestamp(transaction.createdAt) + let role: AbstractTransactionRole if (transaction.type === InputTransactionType.GRADIDO_CREATION) { - const creationTransactionRole = new CreationTransactionRole(transaction) - if ( - addToBlockchain( - await creationTransactionRole.getGradidoTransactionBuilder(), - senderBlockchain, - createdAtTimestamp, - ) - ) { - logger.debug(`Creation Transaction added for user ${transaction.user.account!.userUuid}`) - } else { - throw new Error( - `Creation Transaction not added for user ${transaction.user.account!.userUuid}`, - ) - } + role = new CreationTransactionRole(transaction) } else if (transaction.type === InputTransactionType.GRADIDO_TRANSFER) { - const transferTransactionRole = new TransferTransactionRole(transaction) - // will crash with cross group transaction - if ( - addToBlockchain( - await transferTransactionRole.getGradidoTransactionBuilder(), - senderBlockchain, - createdAtTimestamp, - ) - ) { - logger.debug(`Transfer Transaction added for user ${transaction.user.account!.userUuid}`) - } else { - throw new Error( - `Transfer Transaction not added for user ${transaction.user.account!.userUuid}`, - ) - } + role = new TransferTransactionRole(transaction) + } else if (transaction.type == InputTransactionType.REGISTER_ADDRESS) { + role = new RegisterAddressTransactionRole(transaction) } else if (transaction.type === InputTransactionType.GRADIDO_DEFERRED_TRANSFER) { - const transferTransactionRole = new DeferredTransferTransactionRole(transaction) - if ( - addToBlockchain( - await transferTransactionRole.getGradidoTransactionBuilder(), - senderBlockchain, - createdAtTimestamp, - ) - ) { - logger.debug( - `Deferred Transfer Transaction added for user ${transaction.user.account!.userUuid}`, - ) - } else { - throw new Error( - `Deferred Transfer Transaction not added for user ${transaction.user.account!.userUuid}`, - ) - } + role = new DeferredTransferTransactionRole(transaction) } else if (transaction.type === InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER) { const seedKeyPairRole = new LinkedTransactionKeyPairRole( v.parse(identifierSeedSchema, transaction.user.seed), ) const f = new Filter() f.involvedPublicKey = seedKeyPairRole.generateKeyPair().getPublicKey() - const deferredTransaction = senderBlockchain.findOne(f) - if (!deferredTransaction) { + f.transactionType = TransactionType_DEFERRED_TRANSFER + const deferredTransactions = senderBlockchain.findAll(f) + if (!deferredTransactions) { throw new Error( `redeem deferred transfer: couldn't find parent deferred transfer on Gradido Node for ${JSON.stringify(transaction, null, 2)} and public key from seed: ${f.involvedPublicKey?.convertToHex()}`, ) } + if (deferredTransactions.size() != 1) { + logger.error( + `redeem deferred transfer: found ${deferredTransactions.size()} parent deferred transfer on Gradido Node for ${JSON.stringify(transaction, null, 2)} and public key from seed: ${f.involvedPublicKey?.convertToHex()}`, + ) + for(let i = 0; i < deferredTransactions.size(); i++) { + logger.error(`deferred transaction ${i}: ${deferredTransactions.get(i)?.getConfirmedTransaction()?.toJson(true)}`) + } + throw new Error( + `redeem deferred transfer: found ${deferredTransactions.size()} parent deferred transfer on Gradido Node for ${JSON.stringify(transaction, null, 2)} and public key from seed: ${f.involvedPublicKey?.convertToHex()}`, + ) + } + const deferredTransaction = deferredTransactions.get(0)! const confirmedDeferredTransaction = deferredTransaction.getConfirmedTransaction() if (!confirmedDeferredTransaction) { throw new Error('redeem deferred transfer: invalid TransactionEntry') } - const redeemTransactionRole = new RedeemDeferredTransferTransactionRole( + debugTmpStr += `\nconfirmed deferred transaction: ${confirmedDeferredTransaction.toJson(true)} with filter: ${f.toJson(true)}` + role = new RedeemDeferredTransferTransactionRole( transaction, confirmedDeferredTransaction, ) - const involvedUser = transaction.user.account - ? transaction.user.account.userUuid - : transaction.linkedUser?.account?.userUuid - if ( - addToBlockchain( - await redeemTransactionRole.getGradidoTransactionBuilder(), - senderBlockchain, - createdAtTimestamp, - ) - ) { - logger.debug(`Redeem Deferred Transfer Transaction added for user ${involvedUser}`) - } else { - throw new Error(`Redeem Deferred Transfer Transaction not added for user ${involvedUser}`) - } + } else { + throw new Error(`Transaction type ${transaction.type} not supported`) + } + const involvedUser = transaction.user.account + ? transaction.user.account.userUuid + : transaction.linkedUser?.account?.userUuid + if (addToBlockchain(await role.getGradidoTransactionBuilder(), senderBlockchain, createdAtTimestamp)) { + logger.debug(`${transaction.type} Transaction added for user ${involvedUser}`) + transactionAddedToBlockchainSum++ + } else { + logger.error(debugTmpStr) + /*const f = new Filter() + f.searchDirection = SearchDirection_DESC + f.pagination = new Pagination(15) + const transactions = senderBlockchain.findAll(f) + for(let i = transactions.size() - 1; i >= 0; i--) { + logger.error(`transaction ${i}: ${transactions.get(i)?.getConfirmedTransaction()?.toJson(true)}`) + }*/ + logger.error(`transaction: ${JSON.stringify(transaction, null, 2)}`) + throw new Error(`${transaction.type} Transaction not added for user ${involvedUser}, after ${transactionAddedToBlockchainSum} transactions`) } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts index 45eff5ffd..0f45d2197 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts @@ -4,13 +4,14 @@ import { HieroId, hieroIdSchema } from '../../schemas/typeGuard.schema' import { addCommunityRootTransaction } from './blockchain' import { Context } from './Context' import { communityDbToCommunity } from './convert' -import { loadCommunities } from './database' +import { loadCommunities, loadContributionLinkModeratorCache } from './database' import { generateKeyPairCommunity } from './keyPair' import { CommunityContext } from './valibot.schema' export async function bootstrap(): Promise { const context = await Context.create() context.communities = await bootstrapCommunities(context) + await loadContributionLinkModeratorCache(context.db) return context } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts index 87cb3eb4a..bfdcfe70e 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts @@ -1,4 +1,4 @@ -import { asc, eq, inArray, isNotNull, sql } from 'drizzle-orm' +import { and, asc, eq, inArray, isNotNull, lt, sql } from 'drizzle-orm' import { alias } from 'drizzle-orm/mysql-core' import { MySql2Database } from 'drizzle-orm/mysql2' import { GradidoUnit } from 'gradido-blockchain-js' @@ -7,6 +7,8 @@ import * as v from 'valibot' import { LOG4JS_BASE_CATEGORY } from '../../config/const' import { communitiesTable, + contributionsTable, + eventsTable, transactionLinksTable, transactionsTable, usersTable, @@ -27,6 +29,24 @@ const logger = getLogger( `${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`, ) +const contributionLinkModerators = new Map() + +export async function loadContributionLinkModeratorCache(db: MySql2Database): Promise { + const result = await db + .select({ + event: eventsTable, + user: usersTable, + }) + .from(eventsTable) + .leftJoin(usersTable, eq(eventsTable.actingUserId, usersTable.id)) + .where(eq(eventsTable.type, 'ADMIN_CONTRIBUTION_LINK_CREATE')) + .orderBy(asc(eventsTable.id)) + + result.map((row: any) => { + contributionLinkModerators.set(row.event.involvedContributionLinkId, v.parse(createdUserDbSchema, row.user)) + }) +} + // queries export async function loadCommunities(db: MySql2Database): Promise { const result = await db @@ -103,16 +123,32 @@ export async function loadTransactions( .limit(count) .offset(offset) - return result.map((row: any) => { + return await Promise.all(result.map(async (row: any) => { // console.log(row) try { + const user = v.parse(createdUserDbSchema, row.user) + let linkedUser: CreatedUserDb | null | undefined = null + if (!row.linkedUser) { + const contribution = await db + .select({contributionLinkId: contributionsTable.contributionLinkId}) + .from(contributionsTable) + .where(eq(contributionsTable.transactionId, row.transaction.id)) + .limit(1) + if (contribution && contribution.length > 0 && contribution[0].contributionLinkId) { + linkedUser = contributionLinkModerators.get(contribution[0].contributionLinkId) + } + } else { + linkedUser = v.parse(createdUserDbSchema, row.linkedUser) + } + if (!linkedUser) { + throw new Error(`linked user not found for transaction ${row.transaction.id}`) + } + // check for consistent data beforehand - const userCreatedAt = new Date(row.user.createdAt) - const linkedUserCreatedAd = new Date(row.linkedUser.createdAt) const balanceDate = new Date(row.transaction.balanceDate) if ( - userCreatedAt.getTime() > balanceDate.getTime() || - linkedUserCreatedAd.getTime() > balanceDate.getTime() + user.createdAt.getTime() > balanceDate.getTime() || + linkedUser?.createdAt.getTime() > balanceDate.getTime() ) { logger.error(`table row: `, row) throw new Error( @@ -127,8 +163,8 @@ export async function loadTransactions( return v.parse(transactionDbSchema, { ...row.transaction, transactionLinkCode: row.transactionLink ? row.transactionLink.code : null, - user: row.user, - linkedUser: row.linkedUser, + user, + linkedUser, }) } catch (e) { logger.error(`table row: ${JSON.stringify(row, null, 2)}`) @@ -137,7 +173,7 @@ export async function loadTransactions( } throw e } - }) + })) } export async function loadTransactionLinks( @@ -170,7 +206,10 @@ export async function loadDeletedTransactionLinks( .select() .from(transactionLinksTable) .leftJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id)) - .where(isNotNull(transactionLinksTable.deletedAt)) + .where(and( + isNotNull(transactionLinksTable.deletedAt), + lt(transactionLinksTable.deletedAt, transactionLinksTable.validUntil) + )) .orderBy(asc(transactionLinksTable.deletedAt), asc(transactionLinksTable.id)) .limit(count) .offset(offset) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts index e08231f4d..c984bb24e 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts @@ -1,5 +1,6 @@ import { sql } from 'drizzle-orm' import { + bigint, char, datetime, decimal, @@ -24,6 +25,22 @@ export const communitiesTable = mysqlTable( (table) => [unique('uuid_key').on(table.communityUuid)], ) +export const contributionsTable = mysqlTable("contributions", { + id: int().autoincrement().notNull(), + contributionLinkId: int("contribution_link_id").default(sql`NULL`), + confirmedBy: int("confirmed_by").default(sql`NULL`), + confirmedAt: datetime("confirmed_at", { mode: 'string'}).default(sql`NULL`), + deletedAt: datetime("deleted_at", { mode: 'string'}).default(sql`NULL`), + transactionId: int("transaction_id").default(sql`NULL`), +}) + +export const eventsTable = mysqlTable("events", { + id: int().autoincrement().notNull(), + type: varchar({ length: 100 }).notNull(), + actingUserId: int("acting_user_id").notNull(), + involvedContributionLinkId: int("involved_contribution_link_id").default(sql`NULL`), +}) + export const usersTable = mysqlTable( 'users', { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts index 875c59eba..09da8abf8 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts @@ -4,7 +4,7 @@ import { exportAllCommunities } from './binaryExport' import { bootstrap } from './bootstrap' import { syncDbWithBlockchainContext } from './interaction/syncDbWithBlockchain/syncDbWithBlockchain.context' -const BATCH_SIZE = 100 +const BATCH_SIZE = 250 async function main() { // prepare in memory blockchains diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts index d6b40938f..4c1b9ca2c 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts @@ -1,4 +1,4 @@ -import { addRegisterAddressTransaction } from '../../blockchain' +import { addTransaction } from '../../blockchain' import { userDbToTransaction } from '../../convert' import { loadUsers } from '../../database' import { generateKeyPairUserAccount } from '../../keyPair' @@ -26,6 +26,6 @@ export class UsersSyncRole extends AbstractSyncRole { async pushToBlockchain(item: CreatedUserDb): Promise { const communityContext = this.context.getCommunityContextByUuid(item.communityUuid) const transaction = userDbToTransaction(item, communityContext.topicId) - return await addRegisterAddressTransaction(communityContext.blockchain, transaction) + return await addTransaction(communityContext.blockchain, communityContext.blockchain, transaction) } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts index e67cd70da..c62187181 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts @@ -6,21 +6,23 @@ import { TransactionsSyncRole } from './TransactionsSync.role' import { UsersSyncRole } from './UsersSync.role' export async function syncDbWithBlockchainContext(context: Context, batchSize: number) { - const timeUsed = new Profiler() + const timeUsedDB = new Profiler() + const timeUsedBlockchain = new Profiler() const containers = [ new UsersSyncRole(context), new TransactionsSyncRole(context), new DeletedTransactionLinksSyncRole(context), new TransactionLinksSyncRole(context), ] - + let transactionsCount = 0 + let transactionsCountSinceLastLog = 0 while (true) { - timeUsed.reset() + timeUsedDB.reset() const results = await Promise.all(containers.map((c) => c.ensureFilled(batchSize))) const loadedItemsCount = results.reduce((acc, c) => acc + c, 0) // log only, if at least one new item was loaded - if (loadedItemsCount && context.logger.isInfoEnabled()) { - context.logger.info(`${loadedItemsCount} new items loaded from db in ${timeUsed.string()}`) + if (loadedItemsCount && context.logger.isDebugEnabled()) { + context.logger.debug(`${loadedItemsCount} new items loaded from db in ${timeUsedDB.string()}`) } // remove empty containers @@ -34,5 +36,14 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n available.sort((a, b) => a.getDate().getTime() - b.getDate().getTime()) } await available[0].toBlockchain() + process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`) + transactionsCount++ + transactionsCountSinceLastLog++ + if (transactionsCountSinceLastLog >= batchSize) { + context.logger.debug(`${transactionsCountSinceLastLog} transactions added to blockchain in ${timeUsedBlockchain.string()}`) + timeUsedBlockchain.reset() + transactionsCountSinceLastLog = 0 + } } + process.stdout.write(`\n`) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts index 26fc985c7..1fc99ec3b 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts @@ -21,7 +21,7 @@ export const userDbSchema = v.object({ communityUuid: uuidv4Schema, }) -export const transactionDbSchema = v.object({ +export const transactionDbSchema = v.pipe(v.object({ typeId: v.enum(TransactionTypeId), amount: gradidoAmountSchema, balanceDate: dateSchema, @@ -30,7 +30,12 @@ export const transactionDbSchema = v.object({ user: userDbSchema, linkedUser: userDbSchema, transactionLinkCode: v.nullish(identifierSeedSchema), -}) +}), v.custom((value: any) => { + if (value.user && value.linkedUser && !value.transactionLinkCode && value.user.gradidoId === value.linkedUser.gradidoId) { + throw new Error(`expect user to be different from linkedUser: ${JSON.stringify(value, null, 2)}`) + } + return value +})) export const transactionLinkDbSchema = v.object({ user: userDbSchema, From 4a72e65460cfcae986cfb35f0b217fe8c0e526d6 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 10 Dec 2025 14:38:55 +0100 Subject: [PATCH 08/50] add workarund for contribution resolver and binarization instead of InMemoryBlockchain for exporting testdata to gradido-blockchain --- .../blockchain.ts | 17 ++++++++++-- .../db-v2.7.0_to_blockchain-v3.5/database.ts | 18 ++++++++++++- .../drizzle.schema.ts | 27 ++++++++++++------- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts index fceecf541..128418cea 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts @@ -25,12 +25,14 @@ import { Community, Transaction } from '../../schemas/transaction.schema' import { identifierSeedSchema } from '../../schemas/typeGuard.schema' import { AbstractTransactionRole } from '../../interactions/sendToHiero/AbstractTransaction.role' import * as v from 'valibot' +import * as fs from 'node:fs' const logger = getLogger( `${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`, ) export const defaultHieroAccount = new HieroAccountId(0, 0, 2) let transactionAddedToBlockchainSum = 0 +const sizeBuffer = Buffer.alloc(2) function addToBlockchain( builder: GradidoTransactionBuilder, @@ -38,7 +40,18 @@ function addToBlockchain( createdAtTimestamp: Timestamp, ): boolean { const transaction = builder.build() - // TOD: use actual transaction id if exist in dlt_transactions table + /* const transactionSerializer = new InteractionSerialize(transaction) + const binTransaction = transactionSerializer.run() + if (!binTransaction) { + logger.error(`Failed to serialize transaction ${transaction.toJson(true)}`) + return false + } + const filePath = `${blockchain.getCommunityId()}.bin` + sizeBuffer.writeUInt16LE(binTransaction.size(), 0) + fs.appendFileSync(filePath, sizeBuffer) + fs.appendFileSync(filePath, binTransaction.data()) + */ + // TODO: use actual transaction id if exist in dlt_transactions table const transactionId = new HieroTransactionId(createdAtTimestamp, defaultHieroAccount) const interactionSerialize = new InteractionSerialize(transactionId) @@ -51,7 +64,7 @@ function addToBlockchain( return result } catch (error) { logger.error(`Transaction ${transaction.toJson(true)} not added: ${error}`) - return false + return true } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts index bfdcfe70e..8b3ef81dc 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts @@ -1,4 +1,4 @@ -import { and, asc, eq, inArray, isNotNull, lt, sql } from 'drizzle-orm' +import { and, asc, eq, inArray, isNotNull, lt, ne, sql } from 'drizzle-orm' import { alias } from 'drizzle-orm/mysql-core' import { MySql2Database } from 'drizzle-orm/mysql2' import { GradidoUnit } from 'gradido-blockchain-js' @@ -11,6 +11,7 @@ import { eventsTable, transactionLinksTable, transactionsTable, + userRolesTable, usersTable, } from './drizzle.schema' import { TransactionTypeId } from './TransactionTypeId' @@ -136,6 +137,21 @@ export async function loadTransactions( .limit(1) if (contribution && contribution.length > 0 && contribution[0].contributionLinkId) { linkedUser = contributionLinkModerators.get(contribution[0].contributionLinkId) + if (linkedUser?.gradidoId === user.gradidoId) { + const adminUser = await db + .select({ + user: usersTable + }) + .from(usersTable) + .leftJoin(userRolesTable, and(eq(usersTable.id, userRolesTable.userId), eq(userRolesTable.role, 'admin'))) + .orderBy(asc(userRolesTable.id)) + .where(ne(userRolesTable.userId, row.user.id)) + .limit(1) + if (!adminUser || !adminUser.length) { + throw new Error(`cannot find replace admin for contribution link`) + } + linkedUser = v.parse(createdUserDbSchema, adminUser[0].user) + } } } else { linkedUser = v.parse(createdUserDbSchema, row.linkedUser) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts index c984bb24e..c907c6c51 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts @@ -25,20 +25,20 @@ export const communitiesTable = mysqlTable( (table) => [unique('uuid_key').on(table.communityUuid)], ) -export const contributionsTable = mysqlTable("contributions", { +export const contributionsTable = mysqlTable('contributions', { id: int().autoincrement().notNull(), - contributionLinkId: int("contribution_link_id").default(sql`NULL`), - confirmedBy: int("confirmed_by").default(sql`NULL`), - confirmedAt: datetime("confirmed_at", { mode: 'string'}).default(sql`NULL`), - deletedAt: datetime("deleted_at", { mode: 'string'}).default(sql`NULL`), - transactionId: int("transaction_id").default(sql`NULL`), + contributionLinkId: int('contribution_link_id').default(sql`NULL`), + confirmedBy: int('confirmed_by').default(sql`NULL`), + confirmedAt: datetime('confirmed_at', { mode: 'string'}).default(sql`NULL`), + deletedAt: datetime('deleted_at', { mode: 'string'}).default(sql`NULL`), + transactionId: int('transaction_id').default(sql`NULL`), }) -export const eventsTable = mysqlTable("events", { +export const eventsTable = mysqlTable('events', { id: int().autoincrement().notNull(), type: varchar({ length: 100 }).notNull(), - actingUserId: int("acting_user_id").notNull(), - involvedContributionLinkId: int("involved_contribution_link_id").default(sql`NULL`), + actingUserId: int('acting_user_id').notNull(), + involvedContributionLinkId: int('involved_contribution_link_id').default(sql`NULL`), }) export const usersTable = mysqlTable( @@ -54,6 +54,15 @@ export const usersTable = mysqlTable( (table) => [unique('uuid_key').on(table.gradidoId, table.communityUuid)], ) +export const userRolesTable = mysqlTable('user_roles', { + id: int().autoincrement().notNull(), + userId: int('user_id').notNull(), + role: varchar({ length: 40 }).notNull(), +}, +(table) => [ + index('user_id').on(table.userId), +]) + export const transactionsTable = mysqlTable( 'transactions', { From b1214f8b6cff602d90b6c1c66b3ed940dca12bad Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 18 Dec 2025 07:50:27 +0100 Subject: [PATCH 09/50] use own account balances --- dlt-connector/bun.lock | 9 + dlt-connector/package.json | 2 + .../db-v2.7.0_to_blockchain-v3.5/Context.ts | 26 ++- .../blockchain.ts | 91 ++++---- .../db-v2.7.0_to_blockchain-v3.5/bootstrap.ts | 36 ++- .../db-v2.7.0_to_blockchain-v3.5/database.ts | 216 +++++++++++++----- .../drizzle.schema.ts | 12 +- .../db-v2.7.0_to_blockchain-v3.5/errors.ts | 6 + .../db-v2.7.0_to_blockchain-v3.5/index.ts | 2 +- .../syncDbWithBlockchain/AbstractSync.role.ts | 2 +- .../ContributionLinkTransactionSync.role.ts | 35 +++ .../DoubleLinkedTransactions.role.ts | 27 +++ ...InvalidContributionTransactionSync.role.ts | 27 +++ .../TransactionLinksSync.role.ts | 22 +- .../TransactionsSync.role.ts | 110 ++++++++- .../syncDbWithBlockchain/UsersSync.role.ts | 2 +- .../syncDbWithBlockchain.context.ts | 19 +- .../db-v2.7.0_to_blockchain-v3.5/utils.ts | 37 +++ .../valibot.schema.ts | 22 +- dlt-connector/src/schemas/typeGuard.schema.ts | 2 +- 20 files changed, 582 insertions(+), 123 deletions(-) create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DoubleLinkedTransactions.role.ts create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/InvalidContributionTransactionSync.role.ts diff --git a/dlt-connector/bun.lock b/dlt-connector/bun.lock index b6747359b..477af78c0 100644 --- a/dlt-connector/bun.lock +++ b/dlt-connector/bun.lock @@ -5,6 +5,7 @@ "": { "name": "dlt-connector", "dependencies": { + "cross-env": "^7.0.3", "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#f265dbb1780a912cf8b0418dfe3eaf5cdc5b51cf", }, "devDependencies": { @@ -19,8 +20,10 @@ "@types/uuid": "^8.3.4", "adm-zip": "^0.5.16", "async-mutex": "^0.5.0", + "decimal.js-light": "^2.5.1", "dotenv": "^10.0.0", "drizzle-orm": "^0.44.7", + "drizzle-valibot": "^0.4.2", "elysia": "1.3.8", "graphql-request": "^7.2.0", "jose": "^5.2.2", @@ -432,6 +435,8 @@ "cookie": ["cookie@1.0.2", "", {}, "sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA=="], + "cross-env": ["cross-env@7.0.3", "", { "dependencies": { "cross-spawn": "^7.0.1" }, "bin": { "cross-env": "src/bin/cross-env.js", "cross-env-shell": "src/bin/cross-env-shell.js" } }, "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw=="], + "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], "crypto-js": ["crypto-js@4.2.0", "", {}, "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q=="], @@ -444,6 +449,8 @@ "debug": ["debug@4.4.1", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ=="], + "decimal.js-light": ["decimal.js-light@2.5.1", "", {}, "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg=="], + "deep-extend": ["deep-extend@0.6.0", "", {}, "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="], "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="], @@ -460,6 +467,8 @@ "drizzle-orm": ["drizzle-orm@0.44.7", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@prisma/client": "*", "@tidbcloud/serverless": "*", "@types/better-sqlite3": "*", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=7", "bun-types": "*", "expo-sqlite": ">=14.0.0", "gel": ">=2", "knex": "*", "kysely": "*", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@tidbcloud/serverless", "@types/better-sqlite3", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "gel", "knex", "kysely", "mysql2", "pg", "postgres", "sql.js", "sqlite3"] }, "sha512-quIpnYznjU9lHshEOAYLoZ9s3jweleHlZIAWR/jX9gAWNg/JhQ1wj0KGRf7/Zm+obRrYd9GjPVJg790QY9N5AQ=="], + "drizzle-valibot": ["drizzle-valibot@0.4.2", "", { "peerDependencies": { "drizzle-orm": ">=0.36.0", "valibot": ">=1.0.0-beta.7" } }, "sha512-tzjT7g0Di/HX7426marIy8IDtWODjPgrwvgscdevLQRUe5rzYzRhx6bDsYLdDFF9VI/eaYgnjNeF/fznWJoUjg=="], + "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="], diff --git a/dlt-connector/package.json b/dlt-connector/package.json index 87566cbd2..1f4984b0b 100644 --- a/dlt-connector/package.json +++ b/dlt-connector/package.json @@ -33,8 +33,10 @@ "@types/uuid": "^8.3.4", "adm-zip": "^0.5.16", "async-mutex": "^0.5.0", + "decimal.js-light": "^2.5.1", "dotenv": "^10.0.0", "drizzle-orm": "^0.44.7", + "drizzle-valibot": "^0.4.2", "elysia": "1.3.8", "graphql-request": "^7.2.0", "jose": "^5.2.2", diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts index 58400a615..40931ef1e 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts @@ -1,4 +1,5 @@ import { heapStats } from 'bun:jsc' +import dotenv from 'dotenv' import { drizzle, MySql2Database } from 'drizzle-orm/mysql2' import { Filter, Profiler, SearchDirection_ASC } from 'gradido-blockchain-js' import { getLogger, Logger } from 'log4js' @@ -8,22 +9,27 @@ import { KeyPairCacheManager } from '../../cache/KeyPairCacheManager' import { CONFIG } from '../../config' import { LOG4JS_BASE_CATEGORY } from '../../config/const' import { Uuidv4 } from '../../schemas/typeGuard.schema' +import { loadUserByGradidoId } from './database' import { bytesToMbyte } from './utils' -import { CommunityContext } from './valibot.schema' +import { CommunityContext, CreatedUserDb } from './valibot.schema' + +dotenv.config() export class Context { public logger: Logger public db: MySql2Database public communities: Map public cache: KeyPairCacheManager + public balanceFixGradidoUser: CreatedUserDb | null private timeUsed: Profiler - constructor(logger: Logger, db: MySql2Database, cache: KeyPairCacheManager) { + constructor(logger: Logger, db: MySql2Database, cache: KeyPairCacheManager, balanceFixGradidoUser: CreatedUserDb | null) { this.logger = logger this.db = db this.cache = cache this.communities = new Map() this.timeUsed = new Profiler() + this.balanceFixGradidoUser = balanceFixGradidoUser } static async create(): Promise { @@ -36,10 +42,22 @@ export class Context { database: CONFIG.MYSQL_DATABASE, port: CONFIG.MYSQL_PORT, }) + const db = drizzle({ client: connection }) + const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5`) + let balanceFixGradidoUser: CreatedUserDb | null = null + if (process.env.MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID) { + balanceFixGradidoUser = await loadUserByGradidoId(db, process.env.MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID) + if (!balanceFixGradidoUser) { + logger.error(`MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID was set to ${process.env.MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID} but user not found`) + } + } else { + logger.debug(`MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID was not set`) + } return new Context( - getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5`), - drizzle({ client: connection }), + logger, + db, KeyPairCacheManager.getInstance(), + balanceFixGradidoUser, ) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts index 128418cea..e8ad337bc 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts @@ -1,20 +1,19 @@ +import * as fs from 'node:fs' import { + AccountBalances, Filter, GradidoTransactionBuilder, HieroAccountId, - HieroTransactionId, InMemoryBlockchain, InteractionSerialize, - Pagination, - Profiler, - SearchDirection_DESC, - Timestamp, TransactionType_DEFERRED_TRANSFER, } from 'gradido-blockchain-js' import { getLogger } from 'log4js' +import * as v from 'valibot' import { LOG4JS_BASE_CATEGORY } from '../../config/const' import { InputTransactionType } from '../../data/InputTransactionType.enum' import { LinkedTransactionKeyPairRole } from '../../interactions/resolveKeyPair/LinkedTransactionKeyPair.role' +import { AbstractTransactionRole } from '../../interactions/sendToHiero/AbstractTransaction.role' import { CommunityRootTransactionRole } from '../../interactions/sendToHiero/CommunityRootTransaction.role' import { CreationTransactionRole } from '../../interactions/sendToHiero/CreationTransaction.role' import { DeferredTransferTransactionRole } from '../../interactions/sendToHiero/DeferredTransferTransaction.role' @@ -23,61 +22,74 @@ import { RegisterAddressTransactionRole } from '../../interactions/sendToHiero/R import { TransferTransactionRole } from '../../interactions/sendToHiero/TransferTransaction.role' import { Community, Transaction } from '../../schemas/transaction.schema' import { identifierSeedSchema } from '../../schemas/typeGuard.schema' -import { AbstractTransactionRole } from '../../interactions/sendToHiero/AbstractTransaction.role' -import * as v from 'valibot' -import * as fs from 'node:fs' +import { NotEnoughGradidoBalanceError } from './errors' const logger = getLogger( `${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`, ) export const defaultHieroAccount = new HieroAccountId(0, 0, 2) let transactionAddedToBlockchainSum = 0 +let addToBlockchainSum = 0 const sizeBuffer = Buffer.alloc(2) function addToBlockchain( builder: GradidoTransactionBuilder, blockchain: InMemoryBlockchain, - createdAtTimestamp: Timestamp, + transactionId: number, + accountBalances: AccountBalances, ): boolean { const transaction = builder.build() - /* const transactionSerializer = new InteractionSerialize(transaction) + const transactionSerializer = new InteractionSerialize(transaction) const binTransaction = transactionSerializer.run() if (!binTransaction) { logger.error(`Failed to serialize transaction ${transaction.toJson(true)}`) return false } const filePath = `${blockchain.getCommunityId()}.bin` + if (!addToBlockchainSum) { + // clear file + fs.writeFileSync(filePath, Buffer.alloc(0)) + } sizeBuffer.writeUInt16LE(binTransaction.size(), 0) fs.appendFileSync(filePath, sizeBuffer) fs.appendFileSync(filePath, binTransaction.data()) - */ - // TODO: use actual transaction id if exist in dlt_transactions table - const transactionId = new HieroTransactionId(createdAtTimestamp, defaultHieroAccount) - const interactionSerialize = new InteractionSerialize(transactionId) - + // + try { - const result = blockchain.createAndAddConfirmedTransaction( + const result = blockchain.createAndAddConfirmedTransactionExtern( transaction, - interactionSerialize.run(), - createdAtTimestamp, + transactionId, + accountBalances, ) + // logger.info(`${transactionTypeToString(transaction.getTransactionBody()?.getTransactionType()!)} Transaction added in ${timeUsed.string()}`) + addToBlockchainSum++ return result } catch (error) { - logger.error(`Transaction ${transaction.toJson(true)} not added: ${error}`) - return true + if (error instanceof Error) { + const matches = error.message.match(/not enough Gradido Balance for (send coins|operation), needed: -?(\d+\.\d+), exist: (\d+\.\d+)/) + if (matches) { + const needed = parseFloat(matches[2]) + const exist = parseFloat(matches[3]) + throw new NotEnoughGradidoBalanceError(needed, exist) + } + } + const lastTransaction = blockchain.findOne(Filter.LAST_TRANSACTION) + throw new Error(`Transaction ${transaction.toJson(true)} not added: ${error}, last transaction was: ${lastTransaction?.getConfirmedTransaction()?.toJson(true)}`) } } export async function addCommunityRootTransaction( blockchain: InMemoryBlockchain, community: Community, + accountBalances: AccountBalances ): Promise { const communityRootTransactionRole = new CommunityRootTransactionRole(community) if ( addToBlockchain( await communityRootTransactionRole.getGradidoTransactionBuilder(), blockchain, - new Timestamp(community.creationDate), + 0, + accountBalances, ) ) { logger.info(`Community Root Transaction added`) @@ -90,17 +102,17 @@ export async function addTransaction( senderBlockchain: InMemoryBlockchain, _recipientBlockchain: InMemoryBlockchain, transaction: Transaction, + transactionId: number, + accountBalances: AccountBalances, ): Promise { let debugTmpStr = '' - - const createdAtTimestamp = new Timestamp(transaction.createdAt) let role: AbstractTransactionRole if (transaction.type === InputTransactionType.GRADIDO_CREATION) { role = new CreationTransactionRole(transaction) } else if (transaction.type === InputTransactionType.GRADIDO_TRANSFER) { role = new TransferTransactionRole(transaction) - } else if (transaction.type == InputTransactionType.REGISTER_ADDRESS) { + } else if (transaction.type === InputTransactionType.REGISTER_ADDRESS) { role = new RegisterAddressTransactionRole(transaction) } else if (transaction.type === InputTransactionType.GRADIDO_DEFERRED_TRANSFER) { role = new DeferredTransferTransactionRole(transaction) @@ -117,7 +129,7 @@ export async function addTransaction( `redeem deferred transfer: couldn't find parent deferred transfer on Gradido Node for ${JSON.stringify(transaction, null, 2)} and public key from seed: ${f.involvedPublicKey?.convertToHex()}`, ) } - if (deferredTransactions.size() != 1) { + if (deferredTransactions.size() !== 1) { logger.error( `redeem deferred transfer: found ${deferredTransactions.size()} parent deferred transfer on Gradido Node for ${JSON.stringify(transaction, null, 2)} and public key from seed: ${f.involvedPublicKey?.convertToHex()}`, ) @@ -144,19 +156,20 @@ export async function addTransaction( const involvedUser = transaction.user.account ? transaction.user.account.userUuid : transaction.linkedUser?.account?.userUuid - if (addToBlockchain(await role.getGradidoTransactionBuilder(), senderBlockchain, createdAtTimestamp)) { - logger.debug(`${transaction.type} Transaction added for user ${involvedUser}`) - transactionAddedToBlockchainSum++ - } else { - logger.error(debugTmpStr) - /*const f = new Filter() - f.searchDirection = SearchDirection_DESC - f.pagination = new Pagination(15) - const transactions = senderBlockchain.findAll(f) - for(let i = transactions.size() - 1; i >= 0; i--) { - logger.error(`transaction ${i}: ${transactions.get(i)?.getConfirmedTransaction()?.toJson(true)}`) - }*/ - logger.error(`transaction: ${JSON.stringify(transaction, null, 2)}`) - throw new Error(`${transaction.type} Transaction not added for user ${involvedUser}, after ${transactionAddedToBlockchainSum} transactions`) + try { + if (addToBlockchain(await role.getGradidoTransactionBuilder(), senderBlockchain, transactionId, accountBalances)) { + // logger.debug(`${transaction.type} Transaction added for user ${involvedUser}`) + transactionAddedToBlockchainSum++ + } else { + logger.error(debugTmpStr) + logger.error(`transaction: ${JSON.stringify(transaction, null, 2)}`) + throw new Error(`${transaction.type} Transaction not added for user ${involvedUser}, after ${transactionAddedToBlockchainSum} transactions`) + } + } catch(e) { + if (e instanceof NotEnoughGradidoBalanceError) { + throw e + } + logger.error(`error adding transaction: ${JSON.stringify(transaction, null, 2)}`) + throw e } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts index 0f45d2197..a51f87f5d 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts @@ -1,17 +1,24 @@ -import { InMemoryBlockchainProvider } from 'gradido-blockchain-js' +import { AccountBalance, AccountBalances, GradidoUnit, InMemoryBlockchainProvider } from 'gradido-blockchain-js' import * as v from 'valibot' +import { KeyPairIdentifierLogic } from '../../data/KeyPairIdentifier.logic' +import { GradidoBlockchainCryptoError } from '../../errors' +import { ResolveKeyPair } from '../../interactions/resolveKeyPair/ResolveKeyPair.context' import { HieroId, hieroIdSchema } from '../../schemas/typeGuard.schema' +import { AUF_ACCOUNT_DERIVATION_INDEX, GMW_ACCOUNT_DERIVATION_INDEX, hardenDerivationIndex } from '../../utils/derivationHelper' import { addCommunityRootTransaction } from './blockchain' import { Context } from './Context' import { communityDbToCommunity } from './convert' -import { loadCommunities, loadContributionLinkModeratorCache } from './database' +import { loadAdminUsersCache, loadCommunities, loadContributionLinkModeratorCache } from './database' import { generateKeyPairCommunity } from './keyPair' import { CommunityContext } from './valibot.schema' export async function bootstrap(): Promise { const context = await Context.create() context.communities = await bootstrapCommunities(context) - await loadContributionLinkModeratorCache(context.db) + await Promise.all([ + loadContributionLinkModeratorCache(context.db), + loadAdminUsersCache(context.db) + ]) return context } @@ -50,7 +57,28 @@ async function bootstrapCommunities(context: Context): Promise() +export const contributionLinkModerators = new Map() +export const adminUsers = new Map() +const transactionIdSet = new Set() export async function loadContributionLinkModeratorCache(db: MySql2Database): Promise { const result = await db @@ -48,6 +53,20 @@ export async function loadContributionLinkModeratorCache(db: MySql2Database): Pr }) } +export async function loadAdminUsersCache(db: MySql2Database): Promise { + const result = await db + .select({ + user: usersTable, + }) + .from(userRolesTable) + .where(eq(userRolesTable.role, 'ADMIN')) + .leftJoin(usersTable, eq(userRolesTable.userId, usersTable.id)) + + result.map((row: any) => { + adminUsers.set(row.gradidoId, v.parse(createdUserDbSchema, row.user)) + }) +} + // queries export async function loadCommunities(db: MySql2Database): Promise { const result = await db @@ -96,23 +115,42 @@ export async function loadUsers( }) } +export async function loadUserByGradidoId(db: MySql2Database, gradidoId: string): Promise { + const result = await db + .select() + .from(usersTable) + .where(eq(usersTable.gradidoId, gradidoId)) + .limit(1) + + return result.length ? v.parse(createdUserDbSchema, result[0]) : null +} + export async function loadTransactions( db: MySql2Database, offset: number, count: number, ): Promise { const linkedUsers = alias(usersTable, 'linkedUser') + const linkedTransactions = alias(transactionsTable, 'linkedTransaction') const result = await db .select({ transaction: transactionsTable, user: usersTable, linkedUser: linkedUsers, - transactionLink: transactionLinksTable, + transactionLink: { + id: transactionLinksTable.id, + code: transactionLinksTable.code + }, + linkedUserBalance: linkedTransactions.balance, }) .from(transactionsTable) .where( - inArray(transactionsTable.typeId, [TransactionTypeId.CREATION, TransactionTypeId.RECEIVE]), + and( + inArray(transactionsTable.typeId, [TransactionTypeId.CREATION, TransactionTypeId.RECEIVE]), + isNotNull(transactionsTable.linkedUserId), + eq(usersTable.foreign, 0) + ) ) .leftJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) .leftJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id)) @@ -120,67 +158,25 @@ export async function loadTransactions( transactionLinksTable, eq(transactionsTable.transactionLinkId, transactionLinksTable.id), ) + .leftJoin(linkedTransactions, eq(transactionsTable.linkedTransactionId, linkedTransactions.id)) .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) .limit(count) .offset(offset) - return await Promise.all(result.map(async (row: any) => { + return result.map((row: any) => { // console.log(row) try { - const user = v.parse(createdUserDbSchema, row.user) - let linkedUser: CreatedUserDb | null | undefined = null - if (!row.linkedUser) { - const contribution = await db - .select({contributionLinkId: contributionsTable.contributionLinkId}) - .from(contributionsTable) - .where(eq(contributionsTable.transactionId, row.transaction.id)) - .limit(1) - if (contribution && contribution.length > 0 && contribution[0].contributionLinkId) { - linkedUser = contributionLinkModerators.get(contribution[0].contributionLinkId) - if (linkedUser?.gradidoId === user.gradidoId) { - const adminUser = await db - .select({ - user: usersTable - }) - .from(usersTable) - .leftJoin(userRolesTable, and(eq(usersTable.id, userRolesTable.userId), eq(userRolesTable.role, 'admin'))) - .orderBy(asc(userRolesTable.id)) - .where(ne(userRolesTable.userId, row.user.id)) - .limit(1) - if (!adminUser || !adminUser.length) { - throw new Error(`cannot find replace admin for contribution link`) - } - linkedUser = v.parse(createdUserDbSchema, adminUser[0].user) - } - } - } else { - linkedUser = v.parse(createdUserDbSchema, row.linkedUser) - } - if (!linkedUser) { - throw new Error(`linked user not found for transaction ${row.transaction.id}`) - } - - // check for consistent data beforehand - const balanceDate = new Date(row.transaction.balanceDate) - if ( - user.createdAt.getTime() > balanceDate.getTime() || - linkedUser?.createdAt.getTime() > balanceDate.getTime() - ) { - logger.error(`table row: `, row) - throw new Error( - 'at least one user was created after transaction balance date, logic error!', - ) - } - - let amount = GradidoUnit.fromString(row.transaction.amount) - if (row.transaction.typeId === TransactionTypeId.SEND) { - amount = amount.mul(new GradidoUnit(-1)) + /*if (transactionIdSet.has(row.transaction.id)) { + throw new Error(`transaction ${row.transaction.id} already loaded`) } + transactionIdSet.add(row.transaction.id) + */ return v.parse(transactionDbSchema, { ...row.transaction, transactionLinkCode: row.transactionLink ? row.transactionLink.code : null, - user, - linkedUser, + user: row.user, + linkedUser: row.linkedUser, + linkedUserBalance: row.linkedUserBalance, }) } catch (e) { logger.error(`table row: ${JSON.stringify(row, null, 2)}`) @@ -189,7 +185,108 @@ export async function loadTransactions( } throw e } - })) + }) +} + +export async function loadInvalidContributionTransactions( + db: MySql2Database, + offset: number, + count: number, +): Promise<{ id: number, balanceDate: Date }[]> { + const result = await db + .select({ + id: transactionsTable.id, + balanceDate: transactionsTable.balanceDate, + }) + .from(transactionsTable) + .where( + and( + eq(transactionsTable.typeId, TransactionTypeId.CREATION), + sql`NOT EXISTS (SELECT 1 FROM contributions WHERE contributions.transaction_id = transactions.id)`, + ) + ) + .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) + .limit(count) + .offset(offset) + + return result.map((row: any) => { + return { + id: row.id, + balanceDate: new Date(row.balanceDate), + } + }) +} + +export async function loadDoubleLinkedTransactions( + db: MySql2Database, + offset: number, + rowsCount: number, +): Promise<{ id: number, balanceDate: Date }[]> { + const result = await db + .select({ + id: transactionsTable.id, + balanceDate: transactionsTable.balanceDate, + transactionLinkId: transactionsTable.transactionLinkId, + cnt: count(), + }) + .from(transactionsTable) + .where( + and( + eq(transactionsTable.typeId, TransactionTypeId.RECEIVE), + isNotNull(transactionsTable.transactionLinkId), + ) + ) + .groupBy(transactionsTable.transactionLinkId) + .having(gt(count(), 1)) + .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) + .limit(rowsCount) + .offset(offset) + + // logger.info(`loadDoubleLinkedTransactions ${result.length}: ${timeUsed.string()}`) + + return result.map((row: any) => { + return { + id: row.transactionLinkId, + balanceDate: new Date(row.balanceDate), + } + }) +} + +export async function loadContributionLinkTransactions( + db: MySql2Database, + offset: number, + count: number, +): Promise<{ transaction: TransactionSelect, user: UserSelect, contributionLinkId: number }[]> { + const result = await db + .select({ + transaction: transactionsTable, + user: usersTable, + contributionLinkId: contributionsTable.contributionLinkId, + }) + .from(contributionsTable) + .where( + and( + isNotNull(contributionsTable.contributionLinkId), + isNull(transactionsTable.linkedUserId) + ) + ) + .leftJoin(transactionsTable, eq(contributionsTable.transactionId, transactionsTable.id)) + .leftJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) + .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) + .limit(count) + .offset(offset) + + return result.map((row: any) => { + if (transactionIdSet.has(row.transaction.id)) { + throw new Error(`transaction ${row.transaction.id} already loaded`) + } + transactionIdSet.add(row.transaction.id) + return { + transaction: v.parse(transactionSelectSchema, row.transaction), + user: v.parse(userSelectSchema, row.user), + contributionLinkId: row.contributionLinkId, + } + }) } export async function loadTransactionLinks( @@ -232,6 +329,7 @@ export async function loadDeletedTransactionLinks( return result.map((row: any) => { return v.parse(transactionDbSchema, { + id: row.transaction_links.id, typeId: TransactionTypeId.RECEIVE, amount: row.transaction_links.amount, balanceDate: new Date(row.transaction_links.deletedAt), diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts index c907c6c51..cb99275e9 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts @@ -1,6 +1,5 @@ import { sql } from 'drizzle-orm' import { - bigint, char, datetime, decimal, @@ -11,6 +10,8 @@ import { unique, varchar, } from 'drizzle-orm/mysql-core' +import { createSelectSchema } from 'drizzle-valibot' +import * as v from 'valibot' // use only fields needed in the migration, after update the rest of the project, import database instead export const communitiesTable = mysqlTable( @@ -45,6 +46,7 @@ export const usersTable = mysqlTable( 'users', { id: int().autoincrement().notNull(), + foreign: tinyint().default(0).notNull(), gradidoId: char('gradido_id', { length: 36 }).notNull(), communityUuid: varchar('community_uuid', { length: 36 }).default(sql`NULL`), createdAt: datetime('created_at', { mode: 'string', fsp: 3 }) @@ -54,6 +56,9 @@ export const usersTable = mysqlTable( (table) => [unique('uuid_key').on(table.gradidoId, table.communityUuid)], ) +export const userSelectSchema = createSelectSchema(usersTable) +export type UserSelect = v.InferOutput + export const userRolesTable = mysqlTable('user_roles', { id: int().autoincrement().notNull(), userId: int('user_id').notNull(), @@ -70,6 +75,7 @@ export const transactionsTable = mysqlTable( typeId: int('type_id').default(sql`NULL`), transactionLinkId: int('transaction_link_id').default(sql`NULL`), amount: decimal({ precision: 40, scale: 20 }).default(sql`NULL`), + balance: decimal({ precision: 40, scale: 20 }).default(sql`NULL`), balanceDate: datetime('balance_date', { mode: 'string', fsp: 3 }) .default(sql`current_timestamp(3)`) .notNull(), @@ -77,10 +83,14 @@ export const transactionsTable = mysqlTable( creationDate: datetime('creation_date', { mode: 'string', fsp: 3 }).default(sql`NULL`), userId: int('user_id').notNull(), linkedUserId: int('linked_user_id').default(sql`NULL`), + linkedTransactionId: int('linked_transaction_id').default(sql`NULL`), }, (table) => [index('user_id').on(table.userId)], ) +export const transactionSelectSchema = createSelectSchema(transactionsTable) +export type TransactionSelect = v.InferOutput + export const transactionLinksTable = mysqlTable('transaction_links', { id: int().autoincrement().notNull(), userId: int().notNull(), diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts new file mode 100644 index 000000000..85c5be50a --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts @@ -0,0 +1,6 @@ +export class NotEnoughGradidoBalanceError extends Error { + constructor(public needed: number, public exist: number) { + super(`Not enough Gradido Balance for send coins, needed: ${needed} Gradido, exist: ${exist} Gradido`) + } +} + \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts index 09da8abf8..9803429e2 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts @@ -4,7 +4,7 @@ import { exportAllCommunities } from './binaryExport' import { bootstrap } from './bootstrap' import { syncDbWithBlockchainContext } from './interaction/syncDbWithBlockchain/syncDbWithBlockchain.context' -const BATCH_SIZE = 250 +const BATCH_SIZE = 1000 async function main() { // prepare in memory blockchains diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts index 90523fa1b..9038e0e49 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts @@ -36,7 +36,7 @@ export abstract class AbstractSyncRole { return this.items.length } return 0 - } + } async toBlockchain(): Promise { if (this.isEmpty()) { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts new file mode 100644 index 000000000..27908fa19 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts @@ -0,0 +1,35 @@ +import * as v from 'valibot' +import { Context } from '../../Context' +import { adminUsers, contributionLinkModerators, loadContributionLinkTransactions } from '../../database' +import { CreatedUserDb, TransactionDb, transactionDbSchema } from '../../valibot.schema' +import { TransactionsSyncRole } from './TransactionsSync.role' + +export class ContributionLinkTransactionSyncRole extends TransactionsSyncRole { + constructor(readonly context: Context) { + super(context) + } + itemTypeName(): string { + return 'contributionLinkTransaction' + } + + async loadFromDb(offset: number, count: number): Promise { + const transactionUsers = await loadContributionLinkTransactions(this.context.db, offset, count) + return transactionUsers.map((transactionUser) => { + let linkedUser: CreatedUserDb | null | undefined = null + linkedUser = contributionLinkModerators.get(transactionUser.contributionLinkId) + if (linkedUser?.gradidoId === transactionUser.user.gradidoId) { + for (const adminUser of adminUsers.values()) { + if (adminUser.gradidoId !== transactionUser.user.gradidoId) { + linkedUser = adminUser + break + } + } + } + return v.parse(transactionDbSchema, { + ...transactionUser.transaction, + user: transactionUser.user, + linkedUser, + }) + }) + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DoubleLinkedTransactions.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DoubleLinkedTransactions.role.ts new file mode 100644 index 000000000..136155a2c --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DoubleLinkedTransactions.role.ts @@ -0,0 +1,27 @@ +import { Context } from '../../Context' +import { loadDoubleLinkedTransactions } from '../../database' +import { AbstractSyncRole } from './AbstractSync.role' + +export class DoubleLinkedTransactionsSyncRole extends AbstractSyncRole<{ id: number, balanceDate: Date }> { + static allTransactionIds: number[] = [] + constructor(readonly context: Context) { + super(context) + } + itemTypeName(): string { + return 'doubleLinkedTransaction' + } + + async loadFromDb(offset: number, count: number): Promise<{ id: number, balanceDate: Date }[]> { + const result = await loadDoubleLinkedTransactions(this.context.db, offset, count) + DoubleLinkedTransactionsSyncRole.allTransactionIds.push(...result.map((r) => r.id)) + return result + } + + getDate(): Date { + return this.peek().balanceDate + } + + async pushToBlockchain(item: { id: number, balanceDate: Date }): Promise { + this.logger.warn(`Double transaction_links ${item.id} found.`) + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/InvalidContributionTransactionSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/InvalidContributionTransactionSync.role.ts new file mode 100644 index 000000000..616dd1fe4 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/InvalidContributionTransactionSync.role.ts @@ -0,0 +1,27 @@ +import { Context } from '../../Context' +import { loadInvalidContributionTransactions } from '../../database' +import { AbstractSyncRole } from './AbstractSync.role' + +export class InvalidContributionTransactionSyncRole extends AbstractSyncRole<{ id: number, balanceDate: Date }> { + static allTransactionIds: number[] = [] + constructor(readonly context: Context) { + super(context) + } + itemTypeName(): string { + return 'invalidContributionTransaction' + } + + async loadFromDb(offset: number, count: number): Promise<{ id: number, balanceDate: Date }[]> { + const result = await loadInvalidContributionTransactions(this.context.db, offset, count) + InvalidContributionTransactionSyncRole.allTransactionIds.push(...result.map((r) => r.id)) + return result + } + + getDate(): Date { + return this.peek().balanceDate + } + + async pushToBlockchain(item: { id: number, balanceDate: Date }): Promise { + this.logger.warn(`Invalid contribution transaction ${item.id} found.`) + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinksSync.role.ts index 847be89ba..6920732cc 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinksSync.role.ts @@ -1,3 +1,6 @@ +import { AccountBalance, AccountBalances, Filter, MemoryBlockPtr, SearchDirection_DESC } from 'gradido-blockchain-js' +import { KeyPairIdentifierLogic } from '../../../../data/KeyPairIdentifier.logic' +import { ResolveKeyPair } from '../../../../interactions/resolveKeyPair/ResolveKeyPair.context' import { addTransaction } from '../../blockchain' import { transactionLinkDbToTransaction } from '../../convert' import { loadTransactionLinks } from '../../database' @@ -20,6 +23,23 @@ export class TransactionLinksSyncRole extends AbstractSyncRole { const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) const transaction = transactionLinkDbToTransaction(item, communityContext.topicId) - await addTransaction(communityContext.blockchain, communityContext.blockchain, transaction) + // I use the receiving part of transaction pair, so the user is the recipient and the linked user the sender + const accountBalances = new AccountBalances() + const senderKeyPair = await ResolveKeyPair( + new KeyPairIdentifierLogic(transaction.user), + ) + const recipientKeyPair = await ResolveKeyPair( + new KeyPairIdentifierLogic(transaction.linkedUser!), + ) + const f = new Filter() + f.involvedPublicKey = senderKeyPair.getPublicKey() + f.pagination.size = 1 + f.searchDirection = SearchDirection_DESC + communityContext.blockchain.findOne(f) + accountBalances.add(new AccountBalance(senderKeyPair.getPublicKey(), item.linkedUserBalance, '')) + accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), item.amount, '')) + + + await addTransaction(communityContext.blockchain, communityContext.blockchain, transaction, item.id, accountBalances) } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionsSync.role.ts index df8709ac8..43e6c2cd7 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionsSync.role.ts @@ -1,10 +1,28 @@ +import Decimal from 'decimal.js-light' +import { AccountBalance, AccountBalances, GradidoUnit } from 'gradido-blockchain-js' +import { InputTransactionType } from '../../../../data/InputTransactionType.enum' +import { KeyPairIdentifierLogic } from '../../../../data/KeyPairIdentifier.logic' +import { GradidoBlockchainCryptoError } from '../../../../errors' +import { ResolveKeyPair } from '../../../../interactions/resolveKeyPair/ResolveKeyPair.context' +import { AUF_ACCOUNT_DERIVATION_INDEX, GMW_ACCOUNT_DERIVATION_INDEX, hardenDerivationIndex } from '../../../../utils/derivationHelper' import { addTransaction } from '../../blockchain' import { transactionDbToTransaction } from '../../convert' import { loadTransactions } from '../../database' +import { legacyCalculateDecay } from '../../utils' import { TransactionDb } from '../../valibot.schema' import { AbstractSyncRole } from './AbstractSync.role' +type BalanceDate = { + balance: Decimal + date: Date +} + export class TransactionsSyncRole extends AbstractSyncRole { + private static transactionLinkCodes = new Set() + static doubleTransactionLinkCodes: string[] = [] + static gmwBalance: BalanceDate | undefined = undefined + static aufBalance: BalanceDate | undefined = undefined + getDate(): Date { return this.peek().balanceDate } @@ -14,7 +32,34 @@ export class TransactionsSyncRole extends AbstractSyncRole { } async loadFromDb(offset: number, count: number): Promise { - return await loadTransactions(this.context.db, offset, count) + const result = await loadTransactions(this.context.db, offset, count) + return result.filter((item) => { + if (item.transactionLinkCode) { + if (TransactionsSyncRole.transactionLinkCodes.has(item.transactionLinkCode)) { + TransactionsSyncRole.doubleTransactionLinkCodes.push(item.transactionLinkCode) + return false + } + TransactionsSyncRole.transactionLinkCodes.add(item.transactionLinkCode) + } + return true + }) + } + + updateGmwAuf(amount: Decimal, date: Date) { + if(!TransactionsSyncRole.gmwBalance) { + TransactionsSyncRole.gmwBalance = { balance: amount, date } + } else { + const oldGmwBalanceDate = TransactionsSyncRole.gmwBalance + const newBalance = legacyCalculateDecay(oldGmwBalanceDate.balance, oldGmwBalanceDate.date, date ) + TransactionsSyncRole.gmwBalance = { balance: newBalance, date } + } + if(!TransactionsSyncRole.aufBalance) { + TransactionsSyncRole.aufBalance = { balance: amount, date } + } else { + const oldAufBalanceDate = TransactionsSyncRole.aufBalance + const newBalance = legacyCalculateDecay(oldAufBalanceDate.balance, oldAufBalanceDate.date, date ) + TransactionsSyncRole.aufBalance = { balance: newBalance, date } + } } async pushToBlockchain(item: TransactionDb): Promise { @@ -28,10 +73,63 @@ export class TransactionsSyncRole extends AbstractSyncRole { senderCommunityContext.topicId, recipientCommunityContext.topicId, ) - await addTransaction( - senderCommunityContext.blockchain, - recipientCommunityContext.blockchain, - transaction, - ) + const accountBalances = new AccountBalances() + if (InputTransactionType.GRADIDO_CREATION === transaction.type) { + const recipientKeyPair = await ResolveKeyPair( + new KeyPairIdentifierLogic(transaction.linkedUser!), + ) + accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), item.balance, '')) + // update gmw and auf + this.updateGmwAuf(new Decimal(item.amount.toString(4)), item.balanceDate) + const communityKeyPair = await ResolveKeyPair(new KeyPairIdentifierLogic({ communityTopicId: senderCommunityContext.topicId })) + const gmwKeyPair = communityKeyPair.deriveChild( + hardenDerivationIndex(GMW_ACCOUNT_DERIVATION_INDEX), + ) + if (!gmwKeyPair) { + throw new GradidoBlockchainCryptoError( + `KeyPairEd25519 child derivation failed, has private key: ${communityKeyPair.hasPrivateKey()} for community: ${senderCommunityContext.communityId}`, + ) + } + const aufKeyPair = communityKeyPair.deriveChild( + hardenDerivationIndex(AUF_ACCOUNT_DERIVATION_INDEX), + ) + if (!aufKeyPair) { + throw new GradidoBlockchainCryptoError( + `KeyPairEd25519 child derivation failed, has private key: ${communityKeyPair.hasPrivateKey()} for community: ${senderCommunityContext.communityId}`, + ) + } + accountBalances.add(new AccountBalance(gmwKeyPair.getPublicKey(), GradidoUnit.fromString( + TransactionsSyncRole.gmwBalance!.balance.toString()), '')) + accountBalances.add(new AccountBalance(aufKeyPair.getPublicKey(), GradidoUnit.fromString( + TransactionsSyncRole.aufBalance!.balance.toString()), '')) + } else if (InputTransactionType.REGISTER_ADDRESS === transaction.type) { + const recipientKeyPair = await ResolveKeyPair( + new KeyPairIdentifierLogic(transaction.user), + ) + accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), GradidoUnit.zero(), '')) + } else { + // I use the receiving part of transaction pair, so the user is the recipient and the linked user the sender + const senderKeyPair = await ResolveKeyPair( + new KeyPairIdentifierLogic(transaction.linkedUser!), + ) + const recipientKeyPair = await ResolveKeyPair( + new KeyPairIdentifierLogic(transaction.user), + ) + accountBalances.add(new AccountBalance(senderKeyPair.getPublicKey(), item.linkedUserBalance, '')) + accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), item.balance, '')) + } + + try { + await addTransaction( + senderCommunityContext.blockchain, + recipientCommunityContext.blockchain, + transaction, + item.id, + accountBalances, + ) + } catch(e) { + this.context.logger.error(`error adding transaction: ${JSON.stringify(transaction, null, 2)}`) + throw e + } } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts index 4c1b9ca2c..0d6c79b38 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts @@ -26,6 +26,6 @@ export class UsersSyncRole extends AbstractSyncRole { async pushToBlockchain(item: CreatedUserDb): Promise { const communityContext = this.context.getCommunityContextByUuid(item.communityUuid) const transaction = userDbToTransaction(item, communityContext.topicId) - return await addTransaction(communityContext.blockchain, communityContext.blockchain, transaction) + return await addTransaction(communityContext.blockchain, communityContext.blockchain, transaction, item.id) } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts index c62187181..53e0fcf39 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts @@ -1,6 +1,8 @@ import { Profiler } from 'gradido-blockchain-js' import { Context } from '../../Context' +import { ContributionLinkTransactionSyncRole } from './ContributionLinkTransactionSync.role' import { DeletedTransactionLinksSyncRole } from './DeletedTransactionLinksSync.role' +import { InvalidContributionTransactionSyncRole } from './InvalidContributionTransactionSync.role' import { TransactionLinksSyncRole } from './TransactionLinksSync.role' import { TransactionsSyncRole } from './TransactionsSync.role' import { UsersSyncRole } from './UsersSync.role' @@ -8,17 +10,21 @@ import { UsersSyncRole } from './UsersSync.role' export async function syncDbWithBlockchainContext(context: Context, batchSize: number) { const timeUsedDB = new Profiler() const timeUsedBlockchain = new Profiler() + const timeUsedAll = new Profiler() const containers = [ new UsersSyncRole(context), new TransactionsSyncRole(context), new DeletedTransactionLinksSyncRole(context), new TransactionLinksSyncRole(context), + new InvalidContributionTransactionSyncRole(context), + new ContributionLinkTransactionSyncRole(context), ] let transactionsCount = 0 let transactionsCountSinceLastLog = 0 + let available = containers while (true) { timeUsedDB.reset() - const results = await Promise.all(containers.map((c) => c.ensureFilled(batchSize))) + const results = await Promise.all(available.map((c) => c.ensureFilled(batchSize))) const loadedItemsCount = results.reduce((acc, c) => acc + c, 0) // log only, if at least one new item was loaded if (loadedItemsCount && context.logger.isDebugEnabled()) { @@ -26,7 +32,7 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n } // remove empty containers - const available = containers.filter((c) => !c.isEmpty()) + available = available.filter((c) => !c.isEmpty()) if (available.length === 0) { break } @@ -46,4 +52,13 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n } } process.stdout.write(`\n`) + context.logger.info(`Synced ${transactionsCount} transactions to blockchain in ${(timeUsedAll.seconds() / 60).toFixed(2)} minutes`) + context.logger.info(`Invalid contribution transactions: ${InvalidContributionTransactionSyncRole.allTransactionIds.length}`) + if (context.logger.isDebugEnabled()) { + context.logger.debug(InvalidContributionTransactionSyncRole.allTransactionIds.join(', ')) + } + context.logger.info(`Double linked transactions: ${TransactionsSyncRole.doubleTransactionLinkCodes.length}`) + if (context.logger.isDebugEnabled()) { + context.logger.debug(TransactionsSyncRole.doubleTransactionLinkCodes.join(', ')) + } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts index c9b4eccb9..dc44af15f 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts @@ -1,3 +1,4 @@ +import Decimal from 'decimal.js-light' import { crypto_generichash_batch, crypto_generichash_KEYBYTES } from 'sodium-native' export function bytesToMbyte(bytes: number): string { @@ -13,3 +14,39 @@ export function calculateOneHashStep(hash: Buffer, data: Buffer): Buffer toMs) { + return amount + } + // decay started before start date; decay for full duration + let duration = (toMs - fromMs) / 1000 + + // decay started between start and end date; decay from decay start till end date + if (startBlockMs >= fromMs) { + duration = (toMs - startBlockMs) / 1000 + } + return legacyDecayFormula(amount, duration) +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts index 1fc99ec3b..f7d66b6ff 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts @@ -11,6 +11,7 @@ import { import { TransactionTypeId } from './TransactionTypeId' export const createdUserDbSchema = v.object({ + id: v.pipe(v.number(), v.minValue(1)), gradidoId: uuidv4Schema, communityUuid: uuidv4Schema, createdAt: dateSchema, @@ -22,22 +23,37 @@ export const userDbSchema = v.object({ }) export const transactionDbSchema = v.pipe(v.object({ + id: v.pipe(v.number(), v.minValue(1)), typeId: v.enum(TransactionTypeId), amount: gradidoAmountSchema, balanceDate: dateSchema, + balance: gradidoAmountSchema, + linkedUserBalance: gradidoAmountSchema, memo: memoSchema, creationDate: v.nullish(dateSchema), - user: userDbSchema, - linkedUser: userDbSchema, + user: createdUserDbSchema, + linkedUser: createdUserDbSchema, transactionLinkCode: v.nullish(identifierSeedSchema), }), v.custom((value: any) => { if (value.user && value.linkedUser && !value.transactionLinkCode && value.user.gradidoId === value.linkedUser.gradidoId) { throw new Error(`expect user to be different from linkedUser: ${JSON.stringify(value, null, 2)}`) } + // check that user and linked user exist before transaction balance date + const balanceDate = new Date(value.balanceDate) + if ( + value.user.createdAt.getTime() >= balanceDate.getTime() || + value.linkedUser?.createdAt.getTime() >= balanceDate.getTime() + ) { + throw new Error( + `at least one user was created after transaction balance date, logic error! ${JSON.stringify(value, null, 2)}`, + ) + } + return value })) export const transactionLinkDbSchema = v.object({ + id: v.pipe(v.number(), v.minValue(1)), user: userDbSchema, code: identifierSeedSchema, amount: gradidoAmountSchema, @@ -62,7 +78,7 @@ export const communityContextSchema = v.object({ folder: v.pipe( v.string(), v.minLength(1, 'expect string length >= 1'), - v.maxLength(255, 'expect string length <= 255'), + v.maxLength(512, 'expect string length <= 512'), v.regex(/^[a-zA-Z0-9-_]+$/, 'expect string to be a valid (alphanumeric, _, -) folder name'), ), }) diff --git a/dlt-connector/src/schemas/typeGuard.schema.ts b/dlt-connector/src/schemas/typeGuard.schema.ts index 3a93ac62f..1f9f12eaf 100644 --- a/dlt-connector/src/schemas/typeGuard.schema.ts +++ b/dlt-connector/src/schemas/typeGuard.schema.ts @@ -163,7 +163,7 @@ export type HieroTransactionIdInput = v.InferInput Date: Fri, 19 Dec 2025 08:48:16 +0100 Subject: [PATCH 10/50] make interaction for accountBalances --- .../db-v2.7.0_to_blockchain-v3.5/bootstrap.ts | 5 +- .../db-v2.7.0_to_blockchain-v3.5/convert.ts | 2 +- .../data/Balance.ts | 25 +++++++ .../{ => data}/TransactionTypeId.ts | 0 .../{ => data}/keyPair.ts | 16 ++--- .../db-v2.7.0_to_blockchain-v3.5/database.ts | 2 +- .../accountBalances/AbstractBalances.role.ts | 10 +++ .../accountBalances/CreationBalances.role.ts | 5 ++ .../accountBalances/TransferBalances.role.ts | 0 .../accountBalances.context.ts | 70 +++++++++++++++++++ .../TransactionsSync.role.ts | 19 ----- .../syncDbWithBlockchain/UsersSync.role.ts | 2 +- .../valibot.schema.ts | 6 +- 13 files changed, 130 insertions(+), 32 deletions(-) create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/Balance.ts rename dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/{ => data}/TransactionTypeId.ts (100%) rename dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/{ => data}/keyPair.ts (80%) create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/AbstractBalances.role.ts create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/CreationBalances.role.ts create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/TransferBalances.role.ts create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/accountBalances.context.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts index a51f87f5d..5e03fbff6 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts @@ -9,8 +9,9 @@ import { addCommunityRootTransaction } from './blockchain' import { Context } from './Context' import { communityDbToCommunity } from './convert' import { loadAdminUsersCache, loadCommunities, loadContributionLinkModeratorCache } from './database' -import { generateKeyPairCommunity } from './keyPair' +import { generateKeyPairCommunity } from './data/keyPair' import { CommunityContext } from './valibot.schema' +import { Balance } from './data/Balance' export async function bootstrap(): Promise { const context = await Context.create() @@ -47,6 +48,8 @@ async function bootstrapCommunities(context: Context): Promise { private static transactionLinkCodes = new Set() static doubleTransactionLinkCodes: string[] = [] - static gmwBalance: BalanceDate | undefined = undefined - static aufBalance: BalanceDate | undefined = undefined getDate(): Date { return this.peek().balanceDate @@ -45,23 +43,6 @@ export class TransactionsSyncRole extends AbstractSyncRole { }) } - updateGmwAuf(amount: Decimal, date: Date) { - if(!TransactionsSyncRole.gmwBalance) { - TransactionsSyncRole.gmwBalance = { balance: amount, date } - } else { - const oldGmwBalanceDate = TransactionsSyncRole.gmwBalance - const newBalance = legacyCalculateDecay(oldGmwBalanceDate.balance, oldGmwBalanceDate.date, date ) - TransactionsSyncRole.gmwBalance = { balance: newBalance, date } - } - if(!TransactionsSyncRole.aufBalance) { - TransactionsSyncRole.aufBalance = { balance: amount, date } - } else { - const oldAufBalanceDate = TransactionsSyncRole.aufBalance - const newBalance = legacyCalculateDecay(oldAufBalanceDate.balance, oldAufBalanceDate.date, date ) - TransactionsSyncRole.aufBalance = { balance: newBalance, date } - } - } - async pushToBlockchain(item: TransactionDb): Promise { const senderCommunityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) const recipientCommunityContext = this.context.getCommunityContextByUuid( diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts index 0d6c79b38..03563468e 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts @@ -1,7 +1,7 @@ import { addTransaction } from '../../blockchain' import { userDbToTransaction } from '../../convert' import { loadUsers } from '../../database' -import { generateKeyPairUserAccount } from '../../keyPair' +import { generateKeyPairUserAccount } from '../../data/keyPair' import { CreatedUserDb } from '../../valibot.schema' import { AbstractSyncRole } from './AbstractSync.role' diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts index f7d66b6ff..aa4a34685 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts @@ -8,7 +8,8 @@ import { memoSchema, uuidv4Schema, } from '../../schemas/typeGuard.schema' -import { TransactionTypeId } from './TransactionTypeId' +import { TransactionTypeId } from './data/TransactionTypeId' +import { Balance } from './data/Balance' export const createdUserDbSchema = v.object({ id: v.pipe(v.number(), v.minValue(1)), @@ -81,6 +82,8 @@ export const communityContextSchema = v.object({ v.maxLength(512, 'expect string length <= 512'), v.regex(/^[a-zA-Z0-9-_]+$/, 'expect string to be a valid (alphanumeric, _, -) folder name'), ), + gmwBalance: v.instance(Balance), + aufBalance: v.instance(Balance), }) export type TransactionDb = v.InferOutput @@ -88,4 +91,5 @@ export type UserDb = v.InferOutput export type CreatedUserDb = v.InferOutput export type TransactionLinkDb = v.InferOutput export type CommunityDb = v.InferOutput +export type Balance = v.InferOutput export type CommunityContext = v.InferOutput From 02483a5993c441ee68b0b412066b2b88a15959d0 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 23 Dec 2025 15:33:42 +0100 Subject: [PATCH 11/50] refactor, make it easier to read --- ...097-fix_production_data_for_blockchain2.ts | 49 ++++++ .../src/cache/KeyPairCacheManager.ts | 13 ++ dlt-connector/src/data/deriveKeyPair.ts | 45 +++++ .../db-v2.7.0_to_blockchain-v3.5/Context.ts | 8 +- .../blockchain.ts | 4 +- .../db-v2.7.0_to_blockchain-v3.5/bootstrap.ts | 101 ++++++----- .../data/Balance.ts | 50 +++++- .../data/ContributionStatus.ts | 7 + .../db-v2.7.0_to_blockchain-v3.5/database.ts | 126 ++++++++++---- .../drizzle.schema.ts | 10 +- .../db-v2.7.0_to_blockchain-v3.5/errors.ts | 40 ++++- .../db-v2.7.0_to_blockchain-v3.5/index.ts | 12 +- .../accountBalances/AbstractBalances.role.ts | 40 ++++- .../accountBalances/CreationBalances.role.ts | 32 +++- .../DeferredTransferBalances.role.ts | 33 ++++ .../RedeemDeferredTransferBalances.role.ts | 30 ++++ .../RegisterAddressBalances.role.ts | 17 ++ .../accountBalances/TransferBalances.role.ts | 30 ++++ .../accountBalances.context.ts | 86 +++------- .../syncDbWithBlockchain/AbstractSync.role.ts | 37 +++- .../ContributionLinkTransactionSync.role.ts | 66 ++++--- .../CreationsSync.role.ts | 132 ++++++++++++++ .../DeletedTransactionLinksSync.role.ts | 147 +++++++++++++++- .../LocalTransactionsSync.role.ts | 151 ++++++++++++++++ .../RedeemTransactionLinksSync.role.ts | 161 ++++++++++++++++++ .../TransactionLinkFundingsSync.role.ts | 138 +++++++++++++++ .../TransactionLinksSync.role.ts | 45 ----- .../TransactionsSync.role.ts | 116 ------------- .../syncDbWithBlockchain/UsersSync.role.ts | 93 ++++++++-- .../syncDbWithBlockchain.context.ts | 29 ++-- .../valibot.schema.ts | 80 ++++++--- 31 files changed, 1515 insertions(+), 413 deletions(-) create mode 100644 dlt-connector/src/data/deriveKeyPair.ts create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/ContributionStatus.ts create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/DeferredTransferBalances.role.ts create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RedeemDeferredTransferBalances.role.ts create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RegisterAddressBalances.role.ts create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/CreationsSync.role.ts create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinksSync.role.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionsSync.role.ts diff --git a/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts b/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts index 004f5ccd3..14ed3f826 100644 --- a/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts +++ b/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts @@ -68,6 +68,16 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis AND t.user_id <> u.id ;`) + await queryFn(` + UPDATE contributions c + JOIN users u ON(c.moderator_id = u.id) + SET c.confirmed_by = u.id + WHERE c.contribution_status = 'CONFIRMED' + AND c.user_id = c.confirmed_by + AND u.created_at < c.confirmed_at + AND c.user_id <> u.id + ;`) + /** * Fix 2: Replace invalid moderators with the earliest ADMIN. * @@ -112,6 +122,25 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis AND t.user_id <> moderator.id ;`) + // similar but with confirmed by user + await queryFn(` + UPDATE contributions c + JOIN ( + SELECT c_sub.id as sub_c_id, u_sub.created_at, u_sub.id + FROM contributions c_sub + JOIN users u_sub ON (c_sub.confirmed_by <> u_sub.id AND c_sub.user_id <> u_sub.id) + JOIN user_roles r_sub ON (u_sub.id = r_sub.user_id) + WHERE r_sub.role IN ('ADMIN', 'MODERATOR') + GROUP BY c_sub.id + ORDER BY r_sub.created_at ASC + ) confirmingUser ON (c.id = confirmingUser.sub_c_id) + LEFT JOIN users u on(c.confirmed_by = u.id) + SET c.confirmed_by = confirmingUser.id + WHERE c.confirmed_at <= u.created_at + AND c.confirmed_at > confirmingUser.created_at + AND c.user_id <> confirmingUser.id + ;`) + /** * Fix 3: Update user creation dates to ensure historical consistency. * @@ -174,6 +203,26 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis WHERE CHAR_LENGTH(t.memo) < 5 ;`) + await queryFn(` + UPDATE contributions t + SET t.memo = CASE + WHEN CHAR_LENGTH(t.memo) = 0 THEN 'empty empty' + WHEN CHAR_LENGTH(t.memo) < 5 THEN LPAD(t.memo, 5, ' ') + ELSE t.memo + END + WHERE CHAR_LENGTH(t.memo) < 5 + ;`) + + await queryFn(` + UPDATE transaction_links t + SET t.memo = CASE + WHEN CHAR_LENGTH(t.memo) = 0 THEN 'empty empty' + WHEN CHAR_LENGTH(t.memo) < 5 THEN LPAD(t.memo, 5, ' ') + ELSE t.memo + END + WHERE CHAR_LENGTH(t.memo) < 5 + ;`) + /** * Fix 5: Insert missing 'ADMIN_CONTRIBUTION_LINK_CREATE' events for contribution_links. * diff --git a/dlt-connector/src/cache/KeyPairCacheManager.ts b/dlt-connector/src/cache/KeyPairCacheManager.ts index 8d7c3bf56..f38342904 100644 --- a/dlt-connector/src/cache/KeyPairCacheManager.ts +++ b/dlt-connector/src/cache/KeyPairCacheManager.ts @@ -76,4 +76,17 @@ export class KeyPairCacheManager { } return keyPair } + + public getKeyPairSync( + input: string, + createKeyPair: () => KeyPairEd25519, + ): KeyPairEd25519 { + const keyPair = this.cache.get(input) + if (!keyPair) { + const keyPair = createKeyPair() + this.cache.set(input, keyPair) + return keyPair + } + return keyPair + } } diff --git a/dlt-connector/src/data/deriveKeyPair.ts b/dlt-connector/src/data/deriveKeyPair.ts new file mode 100644 index 000000000..f0880b207 --- /dev/null +++ b/dlt-connector/src/data/deriveKeyPair.ts @@ -0,0 +1,45 @@ +import { KeyPairEd25519, MemoryBlock } from 'gradido-blockchain-js' +import { GradidoBlockchainCryptoError, ParameterError } from '../errors' +import { Hex32, Uuidv4 } from '../schemas/typeGuard.schema' +import { hardenDerivationIndex } from '../utils/derivationHelper' + +export function deriveFromSeed(seed: Hex32): KeyPairEd25519 { + const keyPair = KeyPairEd25519.create(MemoryBlock.fromHex(seed)) + if (!keyPair) { + throw new Error(`couldn't create keyPair from seed: ${seed}`) + } + return keyPair +} + +export function deriveFromCode(code: string): KeyPairEd25519 { + // code is expected to be 24 bytes long, but we need 32 + // so hash the seed with blake2 and we have 32 Bytes + const hash = new MemoryBlock(code).calculateHash() + const keyPair = KeyPairEd25519.create(hash) + if (!keyPair) { + throw new ParameterError( + `error creating Ed25519 KeyPair from seed: ${code.substring(0, 5)}...`, + ) + } + return keyPair +} + +export function deriveFromKeyPairAndUuid(keyPair: KeyPairEd25519, uuid: Uuidv4): KeyPairEd25519 { + const wholeHex = Buffer.from(uuid.replace(/-/g, ''), 'hex') + const parts = [] + for (let i = 0; i < 4; i++) { + parts[i] = hardenDerivationIndex(wholeHex.subarray(i * 4, (i + 1) * 4).readUInt32BE()) + } + // parts: [2206563009, 2629978174, 2324817329, 2405141782] + return parts.reduce((keyPair: KeyPairEd25519, node: number) => deriveFromKeyPairAndIndex(keyPair, node), keyPair) +} + +export function deriveFromKeyPairAndIndex(keyPair: KeyPairEd25519, index: number): KeyPairEd25519 { + const localKeyPair = keyPair.deriveChild(index) + if (!localKeyPair) { + throw new GradidoBlockchainCryptoError( + `KeyPairEd25519 child derivation failed, has private key: ${keyPair.hasPrivateKey()}, index: ${index}`, + ) + } + return localKeyPair +} \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts index 40931ef1e..ea854ce7a 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts @@ -11,7 +11,7 @@ import { LOG4JS_BASE_CATEGORY } from '../../config/const' import { Uuidv4 } from '../../schemas/typeGuard.schema' import { loadUserByGradidoId } from './database' import { bytesToMbyte } from './utils' -import { CommunityContext, CreatedUserDb } from './valibot.schema' +import { CommunityContext, UserDb } from './valibot.schema' dotenv.config() @@ -20,10 +20,10 @@ export class Context { public db: MySql2Database public communities: Map public cache: KeyPairCacheManager - public balanceFixGradidoUser: CreatedUserDb | null + public balanceFixGradidoUser: UserDb | null private timeUsed: Profiler - constructor(logger: Logger, db: MySql2Database, cache: KeyPairCacheManager, balanceFixGradidoUser: CreatedUserDb | null) { + constructor(logger: Logger, db: MySql2Database, cache: KeyPairCacheManager, balanceFixGradidoUser: UserDb | null) { this.logger = logger this.db = db this.cache = cache @@ -44,7 +44,7 @@ export class Context { }) const db = drizzle({ client: connection }) const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5`) - let balanceFixGradidoUser: CreatedUserDb | null = null + let balanceFixGradidoUser: UserDb | null = null if (process.env.MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID) { balanceFixGradidoUser = await loadUserByGradidoId(db, process.env.MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID) if (!balanceFixGradidoUser) { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts index e8ad337bc..1d473a1b2 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts @@ -32,7 +32,7 @@ let transactionAddedToBlockchainSum = 0 let addToBlockchainSum = 0 const sizeBuffer = Buffer.alloc(2) -function addToBlockchain( +export function addToBlockchain( builder: GradidoTransactionBuilder, blockchain: InMemoryBlockchain, transactionId: number, @@ -53,7 +53,7 @@ function addToBlockchain( sizeBuffer.writeUInt16LE(binTransaction.size(), 0) fs.appendFileSync(filePath, sizeBuffer) fs.appendFileSync(filePath, binTransaction.data()) - // + //*/ try { const result = blockchain.createAndAddConfirmedTransactionExtern( diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts index 5e03fbff6..cd34bd8eb 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts @@ -1,17 +1,15 @@ -import { AccountBalance, AccountBalances, GradidoUnit, InMemoryBlockchainProvider } from 'gradido-blockchain-js' +import { randomBytes } from 'node:crypto' +import { AccountBalances, GradidoTransactionBuilder, InMemoryBlockchainProvider } from 'gradido-blockchain-js' import * as v from 'valibot' -import { KeyPairIdentifierLogic } from '../../data/KeyPairIdentifier.logic' -import { GradidoBlockchainCryptoError } from '../../errors' -import { ResolveKeyPair } from '../../interactions/resolveKeyPair/ResolveKeyPair.context' -import { HieroId, hieroIdSchema } from '../../schemas/typeGuard.schema' +import { CONFIG } from '../../config' +import { deriveFromSeed } from '../../data/deriveKeyPair' +import { Hex32, hex32Schema } from '../../schemas/typeGuard.schema' import { AUF_ACCOUNT_DERIVATION_INDEX, GMW_ACCOUNT_DERIVATION_INDEX, hardenDerivationIndex } from '../../utils/derivationHelper' -import { addCommunityRootTransaction } from './blockchain' +import { addToBlockchain } from './blockchain' import { Context } from './Context' -import { communityDbToCommunity } from './convert' -import { loadAdminUsersCache, loadCommunities, loadContributionLinkModeratorCache } from './database' -import { generateKeyPairCommunity } from './data/keyPair' -import { CommunityContext } from './valibot.schema' import { Balance } from './data/Balance' +import { loadAdminUsersCache, loadCommunities, loadContributionLinkModeratorCache } from './database' +import { CommunityContext } from './valibot.schema' export async function bootstrap(): Promise { const context = await Context.create() @@ -26,62 +24,63 @@ export async function bootstrap(): Promise { async function bootstrapCommunities(context: Context): Promise> { const communities = new Map() const communitiesDb = await loadCommunities(context.db) - const topicIds = new Set() + const communityNames = new Set() for (const communityDb of communitiesDb) { + let alias = communityDb.name + if (communityNames.has(communityDb.name)) { + alias = communityDb.communityUuid + } else { + communityNames.add(communityDb.name) + } const blockchain = InMemoryBlockchainProvider.getInstance().findBlockchain( - communityDb.uniqueAlias, + alias, ) if (!blockchain) { - throw new Error(`Couldn't create Blockchain for community ${communityDb.communityUuid}`) + throw new Error(`Couldn't create Blockchain for community ${alias}`) } - context.logger.info(`Blockchain for community '${communityDb.uniqueAlias}' created`) - // make sure topic id is unique - let topicId: HieroId - do { - topicId = v.parse(hieroIdSchema, '0.0.' + Math.floor(Math.random() * 10000)) - } while (topicIds.has(topicId)) - topicIds.add(topicId) - - communities.set(communityDb.communityUuid, { - communityId: communityDb.uniqueAlias, - blockchain, - topicId, - folder: communityDb.uniqueAlias.replace(/[^a-zA-Z0-9]/g, '_'), - gmwBalance: new Balance(), - aufBalance: new Balance(), - }) - - generateKeyPairCommunity(communityDb, context.cache, topicId) + context.logger.info(`Blockchain for community '${alias}' created`) + let seed: Hex32 + if (!communityDb.foreign) { + seed = v.parse(hex32Schema, CONFIG.HOME_COMMUNITY_SEED.convertToHex()) + } else { + seed = v.parse(hex32Schema, randomBytes(32).toString('hex')) + } + let creationDate = communityDb.creationDate if (communityDb.userMinCreatedAt && communityDb.userMinCreatedAt < communityDb.creationDate) { // create community root transaction 1 minute before first user creationDate = new Date(new Date(communityDb.userMinCreatedAt).getTime() - 1000 * 60) } - // community from db to community format the dlt connector normally uses - const community = communityDbToCommunity(topicId, communityDb, creationDate) - // TODO: remove code for gmw and auf key somewhere else - const communityKeyPair = await ResolveKeyPair(new KeyPairIdentifierLogic({ communityTopicId: topicId })) - const gmwKeyPair = communityKeyPair.deriveChild( - hardenDerivationIndex(GMW_ACCOUNT_DERIVATION_INDEX), - ) - if (!gmwKeyPair) { - throw new GradidoBlockchainCryptoError( - `KeyPairEd25519 child derivation failed, has private key: ${communityKeyPair.hasPrivateKey()} for community: ${communityDb.communityUuid}`, - ) + const communityKeyPair = deriveFromSeed(seed) + const gmwKeyPair = communityKeyPair.deriveChild(hardenDerivationIndex(GMW_ACCOUNT_DERIVATION_INDEX)) + const aufKeyPair = communityKeyPair.deriveChild(hardenDerivationIndex(AUF_ACCOUNT_DERIVATION_INDEX)) + if (!communityKeyPair || !gmwKeyPair || !aufKeyPair) { + throw new Error(`Error on creating key pair for community ${JSON.stringify(communityDb, null, 2)}`) } - const aufKeyPair = communityKeyPair.deriveChild( - hardenDerivationIndex(AUF_ACCOUNT_DERIVATION_INDEX), - ) - if (!aufKeyPair) { - throw new GradidoBlockchainCryptoError( - `KeyPairEd25519 child derivation failed, has private key: ${communityKeyPair.hasPrivateKey()} for community: ${communityDb.communityUuid}`, + const builder = new GradidoTransactionBuilder() + builder + .setCreatedAt(creationDate) + .setCommunityRoot( + communityKeyPair.getPublicKey(), + gmwKeyPair.getPublicKey(), + aufKeyPair.getPublicKey(), ) + .sign(communityKeyPair) + + const communityContext: CommunityContext = { + communityId: alias, + blockchain, + keyPair: communityKeyPair, + folder: alias.replace(/[^a-zA-Z0-9]/g, '_'), + gmwBalance: new Balance(gmwKeyPair.getPublicKey()!), + aufBalance: new Balance(aufKeyPair.getPublicKey()!), } + communities.set(communityDb.communityUuid, communityContext) const accountBalances = new AccountBalances() - accountBalances.add(new AccountBalance(gmwKeyPair.getPublicKey(), GradidoUnit.zero(), '')) - accountBalances.add(new AccountBalance(aufKeyPair.getPublicKey(), GradidoUnit.zero(), '')) - await addCommunityRootTransaction(blockchain, community, accountBalances) + accountBalances.add(communityContext.aufBalance.getAccountBalance()) + accountBalances.add(communityContext.gmwBalance.getAccountBalance()) + addToBlockchain(builder, blockchain, communityDb.id, accountBalances) } return communities } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/Balance.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/Balance.ts index adc13860d..718be3ff9 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/Balance.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/Balance.ts @@ -1,25 +1,63 @@ import Decimal from 'decimal.js-light' -import { GradidoUnit } from 'gradido-blockchain-js' +import { AccountBalance, GradidoUnit, MemoryBlockPtr } from 'gradido-blockchain-js' import { legacyCalculateDecay } from '../utils' export class Balance { private balance: GradidoUnit private date: Date - public constructor() - { + private publicKey: MemoryBlockPtr + private communityId?: string | null + + constructor(publicKey: MemoryBlockPtr, communityId?: string | null) { this.balance = new GradidoUnit(0) this.date = new Date() + this.publicKey = publicKey + this.communityId = communityId } - public update(amount: Decimal, date: Date) { + static fromAccountBalance(accountBalance: AccountBalance, confirmedAt: Date): Balance { + const balance = new Balance(accountBalance.getPublicKey()!, accountBalance.getCommunityId() || null) + balance.update(accountBalance.getBalance(), confirmedAt) + return balance + } + + getBalance(): GradidoUnit { + return this.balance + } + + updateLegacyDecay(amount: GradidoUnit, date: Date) { + const previousBalanceString = this.balance.toString() if (this.balance.equal(GradidoUnit.zero())) { - this.balance = GradidoUnit.fromString(amount.toString()) + this.balance = amount this.date = date } else { const decayedBalance = legacyCalculateDecay(new Decimal(this.balance.toString()), this.date, date ) - const newBalance = decayedBalance.add(amount) + const newBalance = decayedBalance.add(new Decimal(amount.toString())) this.balance = GradidoUnit.fromString(newBalance.toString()) this.date = date } + if (this.balance.lt(GradidoUnit.zero())) { + throw new Error(`negative Gradido amount detected in Balance.updateLegacyDecay, previous balance: ${previousBalanceString}, amount: ${amount.toString()}`) + } + } + + update(amount: GradidoUnit, date: Date) { + const previousBalanceString = this.balance.toString() + if (this.balance.equal(GradidoUnit.zero())) { + this.balance = amount + this.date = date + } else { + this.balance = this.balance + .calculateDecay(this.date, date) + .add(amount) + this.date = date + } + if (this.balance.lt(GradidoUnit.zero())) { + throw new Error(`negative Gradido amount detected in Balance.update, previous balance: ${previousBalanceString}, amount: ${amount.toString()}`) + } + } + + getAccountBalance(): AccountBalance { + return new AccountBalance(this.publicKey, this.balance, this.communityId || '') } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/ContributionStatus.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/ContributionStatus.ts new file mode 100644 index 000000000..a4bbc55f3 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/ContributionStatus.ts @@ -0,0 +1,7 @@ +export enum ContributionStatus { + PENDING = 'PENDING', + DELETED = 'DELETED', + IN_PROGRESS = 'IN_PROGRESS', + DENIED = 'DENIED', + CONFIRMED = 'CONFIRMED', +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts index 5000a22fc..a4a81b826 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts @@ -4,6 +4,8 @@ import { MySql2Database } from 'drizzle-orm/mysql2' import { getLogger } from 'log4js' import * as v from 'valibot' import { LOG4JS_BASE_CATEGORY } from '../../config/const' +import { ContributionStatus } from './data/ContributionStatus' +import { TransactionTypeId } from './data/TransactionTypeId' import { communitiesTable, contributionsTable, @@ -17,12 +19,14 @@ import { userSelectSchema, usersTable } from './drizzle.schema' -import { TransactionTypeId } from './data/TransactionTypeId' +import { DatabaseError } from './errors' import { CommunityDb, - CreatedUserDb, + UserDb, + CreationTransactionDb, communityDbSchema, - createdUserDbSchema, + userDbSchema, + creationTransactionDbSchema, TransactionDb, TransactionLinkDb, transactionDbSchema, @@ -33,8 +37,8 @@ const logger = getLogger( `${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`, ) -export const contributionLinkModerators = new Map() -export const adminUsers = new Map() +export const contributionLinkModerators = new Map() +export const adminUsers = new Map() const transactionIdSet = new Set() export async function loadContributionLinkModeratorCache(db: MySql2Database): Promise { @@ -49,7 +53,7 @@ export async function loadContributionLinkModeratorCache(db: MySql2Database): Pr .orderBy(asc(eventsTable.id)) result.map((row: any) => { - contributionLinkModerators.set(row.event.involvedContributionLinkId, v.parse(createdUserDbSchema, row.user)) + contributionLinkModerators.set(row.event.involvedContributionLinkId, v.parse(userDbSchema, row.user)) }) } @@ -63,7 +67,7 @@ export async function loadAdminUsersCache(db: MySql2Database): Promise { .leftJoin(usersTable, eq(userRolesTable.userId, usersTable.id)) result.map((row: any) => { - adminUsers.set(row.gradidoId, v.parse(createdUserDbSchema, row.user)) + adminUsers.set(row.gradidoId, v.parse(userDbSchema, row.user)) }) } @@ -71,6 +75,7 @@ export async function loadAdminUsersCache(db: MySql2Database): Promise { export async function loadCommunities(db: MySql2Database): Promise { const result = await db .select({ + id: communitiesTable.id, foreign: communitiesTable.foreign, communityUuid: communitiesTable.communityUuid, name: communitiesTable.name, @@ -83,18 +88,8 @@ export async function loadCommunities(db: MySql2Database): Promise() return result.map((row: any) => { - let alias = row.name - if (communityNames.has(row.name)) { - alias = row.community_uuid - } else { - communityNames.add(row.name) - } - return v.parse(communityDbSchema, { - ...row, - uniqueAlias: alias, - }) + return v.parse(communityDbSchema, row) }) } @@ -102,7 +97,7 @@ export async function loadUsers( db: MySql2Database, offset: number, count: number, -): Promise { +): Promise { const result = await db .select() .from(usersTable) @@ -110,19 +105,59 @@ export async function loadUsers( .limit(count) .offset(offset) - return result.map((row: any) => { - return v.parse(createdUserDbSchema, row) - }) + return result.map((row: any) => v.parse(userDbSchema, row)) } -export async function loadUserByGradidoId(db: MySql2Database, gradidoId: string): Promise { +export async function loadUserByGradidoId(db: MySql2Database, gradidoId: string): Promise { const result = await db .select() .from(usersTable) .where(eq(usersTable.gradidoId, gradidoId)) .limit(1) - return result.length ? v.parse(createdUserDbSchema, result[0]) : null + return result.length ? v.parse(userDbSchema, result[0]) : null +} + +export async function loadLocalTransferTransactions( + db: MySql2Database, + offset: number, + count: number, +): Promise { + const linkedUsers = alias(usersTable, 'linkedUser') + const result = await db + .select({ + transaction: transactionsTable, + user: usersTable, + linkedUser: linkedUsers, + }) + .from(transactionsTable) + .where( + and( + eq(transactionsTable.typeId, TransactionTypeId.RECEIVE), + isNull(transactionsTable.transactionLinkId), + isNotNull(transactionsTable.linkedUserId), + eq(usersTable.foreign, 0), + eq(linkedUsers.foreign, 0), + ) + ) + .leftJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) + .leftJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id)) + .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) + .limit(count) + .offset(offset) + + return result.map((row: any) => { + const item = { + ...row.transaction, + user: row.user, + linkedUser: row.linkedUser, + } + try { + return v.parse(transactionDbSchema, item) + } catch (e) { + throw new DatabaseError('loadLocalTransferTransactions', item, e as Error) + } + }) } export async function loadTransactions( @@ -131,8 +166,7 @@ export async function loadTransactions( count: number, ): Promise { const linkedUsers = alias(usersTable, 'linkedUser') - const linkedTransactions = alias(transactionsTable, 'linkedTransaction') - + const result = await db .select({ transaction: transactionsTable, @@ -142,7 +176,6 @@ export async function loadTransactions( id: transactionLinksTable.id, code: transactionLinksTable.code }, - linkedUserBalance: linkedTransactions.balance, }) .from(transactionsTable) .where( @@ -158,7 +191,6 @@ export async function loadTransactions( transactionLinksTable, eq(transactionsTable.transactionLinkId, transactionLinksTable.id), ) - .leftJoin(linkedTransactions, eq(transactionsTable.linkedTransactionId, linkedTransactions.id)) .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) .limit(count) .offset(offset) @@ -176,7 +208,6 @@ export async function loadTransactions( transactionLinkCode: row.transactionLink ? row.transactionLink.code : null, user: row.user, linkedUser: row.linkedUser, - linkedUserBalance: row.linkedUserBalance, }) } catch (e) { logger.error(`table row: ${JSON.stringify(row, null, 2)}`) @@ -188,6 +219,43 @@ export async function loadTransactions( }) } +export async function loadCreations( + db: MySql2Database, + offset: number, + count: number, +): Promise { + const confirmedByUsers = alias(usersTable, 'confirmedByUser') + const result = await db + .select({ + contribution: contributionsTable, + user: usersTable, + confirmedByUser: confirmedByUsers, + }) + .from(contributionsTable) + .where(and( + isNull(contributionsTable.contributionLinkId), + eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED), + )) + .leftJoin(usersTable, eq(contributionsTable.userId, usersTable.id)) + .leftJoin(confirmedByUsers, eq(contributionsTable.confirmedBy, confirmedByUsers.id)) + .orderBy(asc(contributionsTable.confirmedAt), asc(contributionsTable.id)) + .limit(count) + .offset(offset) + + return result.map((row) => { + const creationTransactionDb = { + ...row.contribution, + user: row.user, + confirmedByUser: row.confirmedByUser, + } + try { + return v.parse(creationTransactionDbSchema, creationTransactionDb) + } catch (e) { + throw new DatabaseError('loadCreations', creationTransactionDb, e as Error) + } + }) +} + export async function loadInvalidContributionTransactions( db: MySql2Database, offset: number, diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts index cb99275e9..3b7b2ec45 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts @@ -28,10 +28,14 @@ export const communitiesTable = mysqlTable( export const contributionsTable = mysqlTable('contributions', { id: int().autoincrement().notNull(), + userId: int('user_id').default(sql`NULL`), + contributionDate: datetime("contribution_date", { mode: 'string'}).default(sql`NULL`), + memo: varchar({ length: 512 }).notNull(), + amount: decimal({ precision: 40, scale: 20 }).notNull(), contributionLinkId: int('contribution_link_id').default(sql`NULL`), confirmedBy: int('confirmed_by').default(sql`NULL`), confirmedAt: datetime('confirmed_at', { mode: 'string'}).default(sql`NULL`), - deletedAt: datetime('deleted_at', { mode: 'string'}).default(sql`NULL`), + contributionStatus: varchar('contribution_status', { length: 12 }).default('\'PENDING\'').notNull(), transactionId: int('transaction_id').default(sql`NULL`), }) @@ -49,9 +53,11 @@ export const usersTable = mysqlTable( foreign: tinyint().default(0).notNull(), gradidoId: char('gradido_id', { length: 36 }).notNull(), communityUuid: varchar('community_uuid', { length: 36 }).default(sql`NULL`), + deletedAt: datetime('deleted_at', { mode: 'string', fsp: 3 }).default(sql`NULL`), createdAt: datetime('created_at', { mode: 'string', fsp: 3 }) .default(sql`current_timestamp(3)`) .notNull(), + }, (table) => [unique('uuid_key').on(table.gradidoId, table.communityUuid)], ) @@ -100,4 +106,6 @@ export const transactionLinksTable = mysqlTable('transaction_links', { createdAt: datetime({ mode: 'string' }).notNull(), deletedAt: datetime({ mode: 'string' }).default(sql`NULL`), validUntil: datetime({ mode: 'string' }).notNull(), + redeemedAt: datetime({ mode: 'string'}).default(sql`NULL`), + redeemedBy: int().default(sql`NULL`), }) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts index 85c5be50a..504ce76a4 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts @@ -1,6 +1,44 @@ +import * as v from 'valibot' + export class NotEnoughGradidoBalanceError extends Error { constructor(public needed: number, public exist: number) { super(`Not enough Gradido Balance for send coins, needed: ${needed} Gradido, exist: ${exist} Gradido`) } } - \ No newline at end of file + +export class DatabaseError extends Error { + constructor(message: string, rows: any, originalError: Error) { + const parts: string[] = [`DatabaseError in ${message}`] + + // Valibot-specific + if (originalError instanceof v.ValiError) { + const flattened = v.flatten(originalError.issues) + parts.push('Validation errors:') + parts.push(JSON.stringify(flattened, null, 2)) + } else { + parts.push(`Original error: ${originalError.message}`) + } + + parts.push('Rows:') + parts.push(JSON.stringify(rows, null, 2)) + + super(parts.join('\n\n')) + + this.name = 'DatabaseError' + } +} + +export class BlockchainError extends Error { + constructor(message: string, item: any, originalError: Error) { + const parts: string[] = [`BlockchainError in ${message}`] + + parts.push(`Original error: ${originalError.message}`) + parts.push('Item:') + parts.push(JSON.stringify(item, null, 2)) + + super(parts.join('\n\n')) + + this.name = 'BlockchainError' + this.stack = originalError.stack + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts index 9803429e2..0d7f69805 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts @@ -1,10 +1,11 @@ -import { Filter } from 'gradido-blockchain-js' +import * as v from 'valibot' import { onShutdown } from '../../../../shared/src/helper/onShutdown' +import { uuidv4Schema } from '../../schemas/typeGuard.schema' import { exportAllCommunities } from './binaryExport' import { bootstrap } from './bootstrap' import { syncDbWithBlockchainContext } from './interaction/syncDbWithBlockchain/syncDbWithBlockchain.context' -const BATCH_SIZE = 1000 +const BATCH_SIZE = 500 async function main() { // prepare in memory blockchains @@ -17,7 +18,12 @@ async function main() { }) // synchronize to in memory blockchain - await syncDbWithBlockchainContext(context, BATCH_SIZE) + try { + await syncDbWithBlockchainContext(context, BATCH_SIZE) + } catch(e) { + console.error(e) + //context.logBlogchain(v.parse(uuidv4Schema, 'e70da33e-5976-4767-bade-aa4e4fa1c01a')) + } // write as binary file for GradidoNode exportAllCommunities(context, BATCH_SIZE) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/AbstractBalances.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/AbstractBalances.role.ts index f2b2cbaf8..b2433240e 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/AbstractBalances.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/AbstractBalances.role.ts @@ -1,10 +1,38 @@ -import { AccountBalances } from 'gradido-blockchain-js' +import { AccountBalances, Filter, InMemoryBlockchain, SearchDirection_DESC } from 'gradido-blockchain-js' +import { KeyPairIdentifierLogic } from '../../../../data/KeyPairIdentifier.logic' +import { ResolveKeyPair } from '../../../../interactions/resolveKeyPair/ResolveKeyPair.context' +import { IdentifierAccount } from '../../../../schemas/account.schema' +import { Transaction } from '../../../../schemas/transaction.schema' +import { Context } from '../../Context' +import { Balance } from '../../data/Balance' -export class AbstractBalancesRole { - public accountBalances: AccountBalances +export abstract class AbstractBalancesRole { + public constructor(protected transaction: Transaction) {} - public constructor() { - this.accountBalances = new AccountBalances() + abstract getAccountBalances(context: Context): Promise + + async getLastBalanceForUser(identifierAccount: IdentifierAccount, blockchain: InMemoryBlockchain): Promise { + const userKeyPair = await ResolveKeyPair( + new KeyPairIdentifierLogic(identifierAccount), + ) + const f = new Filter() + f.involvedPublicKey = userKeyPair.getPublicKey() + f.pagination.size = 1 + f.searchDirection = SearchDirection_DESC + const lastSenderTransaction = blockchain.findOne(f) + if (!lastSenderTransaction) { + throw new Error(`no last transaction found for user: ${JSON.stringify(identifierAccount, null, 2)}`) } - + const lastConfirmedTransaction = lastSenderTransaction.getConfirmedTransaction() + if (!lastConfirmedTransaction) { + throw new Error(`invalid transaction, getConfirmedTransaction call failed for transaction nr: ${lastSenderTransaction.getTransactionNr()}`) + } + const senderLastAccountBalance = lastConfirmedTransaction.getAccountBalance(userKeyPair.getPublicKey(), '') + if (!senderLastAccountBalance) { + throw new Error( + `no sender account balance found for transaction nr: ${lastSenderTransaction.getTransactionNr()} and public key: ${userKeyPair.getPublicKey()?.convertToHex()}` + ) + } + return Balance.fromAccountBalance(senderLastAccountBalance, lastConfirmedTransaction.getConfirmedAt().getDate()) + } } \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/CreationBalances.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/CreationBalances.role.ts index a596832d6..45d8e2308 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/CreationBalances.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/CreationBalances.role.ts @@ -1,5 +1,33 @@ +import { AccountBalances } from 'gradido-blockchain-js' +import { Transaction } from '../../../../schemas/transaction.schema' +import { Context } from '../../Context' +import { TransactionDb } from '../../valibot.schema' import { AbstractBalancesRole } from './AbstractBalances.role' export class CreationBalancesRole extends AbstractBalancesRole { - -} \ No newline at end of file + + constructor(transaction: Transaction, protected dbTransaction: TransactionDb) { + super(transaction) + } + + async getAccountBalances(context: Context): Promise { + if (this.dbTransaction.linkedUser.communityUuid !== this.dbTransaction.user.communityUuid) { + throw new Error('creation: both recipient and signer must belong to same community') + } + + const accountBalances = new AccountBalances() + const communityContext = context.getCommunityContextByUuid(this.dbTransaction.user.communityUuid) + const balance = await this.getLastBalanceForUser(this.transaction.user, communityContext.blockchain) + + // calculate decay since last balance with legacy calculation method + balance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate) + communityContext.aufBalance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate) + communityContext.gmwBalance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate) + + accountBalances.add(balance.getAccountBalance()) + accountBalances.add(communityContext.aufBalance.getAccountBalance()) + accountBalances.add(communityContext.gmwBalance.getAccountBalance()) + + return accountBalances + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/DeferredTransferBalances.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/DeferredTransferBalances.role.ts new file mode 100644 index 000000000..077c20ca7 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/DeferredTransferBalances.role.ts @@ -0,0 +1,33 @@ +import { AccountBalance, AccountBalances, GradidoUnit } from 'gradido-blockchain-js' +import { KeyPairIdentifierLogic } from '../../../../data/KeyPairIdentifier.logic' +import { ResolveKeyPair } from '../../../../interactions/resolveKeyPair/ResolveKeyPair.context' +import { Transaction } from '../../../../schemas/transaction.schema' +import { Context } from '../../Context' +import { TransactionLinkDb } from '../../valibot.schema' +import { AbstractBalancesRole } from './AbstractBalances.role' + + +export class DeferredTransferBalancesRole extends AbstractBalancesRole { + constructor(transaction: Transaction, protected dbTransactionLink: TransactionLinkDb) { + super(transaction) + } + + async getAccountBalances(context: Context): Promise { + const senderCommunityContext = context.getCommunityContextByUuid(this.dbTransactionLink.user.communityUuid) + const accountBalances = new AccountBalances() + + const seededIdentifier = new KeyPairIdentifierLogic(this.transaction.linkedUser!) + if (!seededIdentifier.isSeedKeyPair()) { + throw new Error(`linked user is not a seed: ${JSON.stringify(this.transaction, null, 2)}`) + } + const seedKeyPair = await ResolveKeyPair(seededIdentifier) + const senderAccountBalance = await this.getLastBalanceForUser(this.transaction.user, senderCommunityContext.blockchain) + + let amount = GradidoUnit.fromString(this.dbTransactionLink.amount.toString()) + amount = amount.calculateCompoundInterest((this.dbTransactionLink.validUntil.getTime() - this.dbTransactionLink.createdAt.getTime()) / 60000) + senderAccountBalance.updateLegacyDecay(amount.negated(), this.dbTransactionLink.createdAt) + accountBalances.add(senderAccountBalance.getAccountBalance()) + accountBalances.add(new AccountBalance(seedKeyPair.getPublicKey(), amount, '')) + return accountBalances + } +} \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RedeemDeferredTransferBalances.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RedeemDeferredTransferBalances.role.ts new file mode 100644 index 000000000..4024c3dd3 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RedeemDeferredTransferBalances.role.ts @@ -0,0 +1,30 @@ +import { AccountBalances } from 'gradido-blockchain-js' +import { Transaction } from '../../../../schemas/transaction.schema' +import { Context } from '../../Context' +import { TransactionDb } from '../../valibot.schema' +import { AbstractBalancesRole } from './AbstractBalances.role' + +export class RedeemDeferredTransferBalancesRole extends AbstractBalancesRole { + constructor(transaction: Transaction, protected dbTransaction: TransactionDb) { + super(transaction) + } + + async getAccountBalances(context: Context): Promise { + // I use the receiving part of transaction pair, so the user is the recipient and the linked user the sender and amount is positive + const senderCommunityContext = context.getCommunityContextByUuid(this.dbTransaction.linkedUser.communityUuid) + const recipientCommunityContext = context.getCommunityContextByUuid(this.dbTransaction.user.communityUuid) + const accountBalances = new AccountBalances() + + context.cache.setHomeCommunityTopicId(senderCommunityContext.topicId) + const senderLastBalance = await this.getLastBalanceForUser(this.transaction.linkedUser!, senderCommunityContext.blockchain) + context.cache.setHomeCommunityTopicId(recipientCommunityContext.topicId) + const recipientLastBalance = await this.getLastBalanceForUser(this.transaction.user, recipientCommunityContext.blockchain) + + senderLastBalance.updateLegacyDecay(this.dbTransaction.amount.negate(), this.dbTransaction.balanceDate) + recipientLastBalance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate) + + accountBalances.add(senderLastBalance.getAccountBalance()) + accountBalances.add(recipientLastBalance.getAccountBalance()) + return accountBalances + } +} \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RegisterAddressBalances.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RegisterAddressBalances.role.ts new file mode 100644 index 000000000..0c4c53454 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RegisterAddressBalances.role.ts @@ -0,0 +1,17 @@ +import { AccountBalance, AccountBalances, GradidoUnit } from 'gradido-blockchain-js' +import { KeyPairIdentifierLogic } from '../../../../data/KeyPairIdentifier.logic' +import { ResolveKeyPair } from '../../../../interactions/resolveKeyPair/ResolveKeyPair.context' +import { Context } from '../../Context' +import { AbstractBalancesRole } from './AbstractBalances.role' + + +export class RegisterAddressBalancesRole extends AbstractBalancesRole { + async getAccountBalances(_context: Context): Promise { + const accountBalances = new AccountBalances() + const recipientKeyPair = await ResolveKeyPair( + new KeyPairIdentifierLogic(this.transaction.user), + ) + accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), GradidoUnit.zero(), '')) + return accountBalances + } +} \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/TransferBalances.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/TransferBalances.role.ts index e69de29bb..c5182627d 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/TransferBalances.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/TransferBalances.role.ts @@ -0,0 +1,30 @@ +import { AccountBalances } from 'gradido-blockchain-js' +import { Transaction } from '../../../../schemas/transaction.schema' +import { Context } from '../../Context' +import { TransactionDb } from '../../valibot.schema' +import { AbstractBalancesRole } from './AbstractBalances.role' + +export class TransferBalancesRole extends AbstractBalancesRole { + constructor(transaction: Transaction, protected dbTransaction: TransactionDb) { + super(transaction) + } + + async getAccountBalances(context: Context): Promise { + // I use the receiving part of transaction pair, so the user is the recipient and the linked user the sender and amount is positive + const senderCommunityContext = context.getCommunityContextByUuid(this.dbTransaction.linkedUser.communityUuid) + const recipientCommunityContext = context.getCommunityContextByUuid(this.dbTransaction.user.communityUuid) + const accountBalances = new AccountBalances() + + context.cache.setHomeCommunityTopicId(senderCommunityContext.topicId) + const senderLastBalance = await this.getLastBalanceForUser(this.transaction.linkedUser!, senderCommunityContext.blockchain) + context.cache.setHomeCommunityTopicId(recipientCommunityContext.topicId) + const recipientLastBalance = await this.getLastBalanceForUser(this.transaction.user, recipientCommunityContext.blockchain) + + senderLastBalance.updateLegacyDecay(this.dbTransaction.amount.negate(), this.dbTransaction.balanceDate) + recipientLastBalance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate) + + accountBalances.add(senderLastBalance.getAccountBalance()) + accountBalances.add(recipientLastBalance.getAccountBalance()) + return accountBalances + } +} \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/accountBalances.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/accountBalances.context.ts index 348e5a8b6..6a2f63604 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/accountBalances.context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/accountBalances.context.ts @@ -1,70 +1,32 @@ +import { AccountBalances } from 'gradido-blockchain-js' +import * as v from 'valibot' import { InputTransactionType } from '../../../../data/InputTransactionType.enum' import { Transaction } from '../../../../schemas/transaction.schema' import { Context } from '../../Context' -import { TransactionDb } from '../../valibot.schema' +import { TransactionDb, TransactionLinkDb, transactionDbSchema, transactionLinkDbSchema } from '../../valibot.schema' import { AbstractBalancesRole } from './AbstractBalances.role' import { CreationBalancesRole } from './CreationBalances.role' +import { DeferredTransferBalancesRole } from './DeferredTransferBalances.role' +import { RedeemDeferredTransferBalancesRole } from './RedeemDeferredTransferBalances.role' +import { RegisterAddressBalancesRole } from './RegisterAddressBalances.role' +import { TransferBalancesRole } from './TransferBalances.role' -export function accountBalancesContext(transaction: Transaction, dbTransaction: TransactionDb, context: Context) { - let role: AbstractBalancesRole | null = null - if (InputTransactionType.GRADIDO_CREATION === transaction.type) { - role = new CreationBalancesRole() - } +export async function accountBalancesContext(transaction: Transaction, item: TransactionDb | TransactionLinkDb, context: Context): Promise { + let role: AbstractBalancesRole | null = null + if (InputTransactionType.GRADIDO_CREATION === transaction.type) { + role = new CreationBalancesRole(transaction, v.parse(transactionDbSchema, item)) + } else if (InputTransactionType.GRADIDO_TRANSFER === transaction.type) { + role = new TransferBalancesRole(transaction, v.parse(transactionDbSchema, item)) + } else if (InputTransactionType.GRADIDO_DEFERRED_TRANSFER === transaction.type) { + role = new DeferredTransferBalancesRole(transaction, v.parse(transactionLinkDbSchema, item)) + } else if (InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER === transaction.type) { + role = new RedeemDeferredTransferBalancesRole(transaction, v.parse(transactionDbSchema, item)) + } else if (InputTransactionType.REGISTER_ADDRESS === transaction.type) { + role = new RegisterAddressBalancesRole(transaction) + } + if (!role) { + throw new Error(`No role found for transaction type ${transaction.type}`) + } + return await role.getAccountBalances(context) } -/* -const senderCommunityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) - const recipientCommunityContext = this.context.getCommunityContextByUuid( - item.linkedUser.communityUuid, - ) - this.context.cache.setHomeCommunityTopicId(senderCommunityContext.topicId) - const transaction = transactionDbToTransaction( - item, - senderCommunityContext.topicId, - recipientCommunityContext.topicId, - ) - const accountBalances = new AccountBalances() - if (InputTransactionType.GRADIDO_CREATION === transaction.type) { - const recipientKeyPair = await ResolveKeyPair( - new KeyPairIdentifierLogic(transaction.linkedUser!), - ) - accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), item.balance, '')) - // update gmw and auf - this.updateGmwAuf(new Decimal(item.amount.toString(4)), item.balanceDate) - const communityKeyPair = await ResolveKeyPair(new KeyPairIdentifierLogic({ communityTopicId: senderCommunityContext.topicId })) - const gmwKeyPair = communityKeyPair.deriveChild( - hardenDerivationIndex(GMW_ACCOUNT_DERIVATION_INDEX), - ) - if (!gmwKeyPair) { - throw new GradidoBlockchainCryptoError( - `KeyPairEd25519 child derivation failed, has private key: ${communityKeyPair.hasPrivateKey()} for community: ${senderCommunityContext.communityId}`, - ) - } - const aufKeyPair = communityKeyPair.deriveChild( - hardenDerivationIndex(AUF_ACCOUNT_DERIVATION_INDEX), - ) - if (!aufKeyPair) { - throw new GradidoBlockchainCryptoError( - `KeyPairEd25519 child derivation failed, has private key: ${communityKeyPair.hasPrivateKey()} for community: ${senderCommunityContext.communityId}`, - ) - } - accountBalances.add(new AccountBalance(gmwKeyPair.getPublicKey(), GradidoUnit.fromString( - TransactionsSyncRole.gmwBalance!.balance.toString()), '')) - accountBalances.add(new AccountBalance(aufKeyPair.getPublicKey(), GradidoUnit.fromString( - TransactionsSyncRole.aufBalance!.balance.toString()), '')) - } else if (InputTransactionType.REGISTER_ADDRESS === transaction.type) { - const recipientKeyPair = await ResolveKeyPair( - new KeyPairIdentifierLogic(transaction.user), - ) - accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), GradidoUnit.zero(), '')) - } else { - // I use the receiving part of transaction pair, so the user is the recipient and the linked user the sender - const senderKeyPair = await ResolveKeyPair( - new KeyPairIdentifierLogic(transaction.linkedUser!), - ) - const recipientKeyPair = await ResolveKeyPair( - new KeyPairIdentifierLogic(transaction.user), - ) - accountBalances.add(new AccountBalance(senderKeyPair.getPublicKey(), item.linkedUserBalance, '')) - accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), item.balance, '')) - */ \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts index 9038e0e49..df6ba0f19 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts @@ -1,7 +1,11 @@ -import { Profiler } from 'gradido-blockchain-js' +import { Filter, InMemoryBlockchain, KeyPairEd25519, MemoryBlockPtr, Profiler, SearchDirection_DESC } from 'gradido-blockchain-js' import { getLogger, Logger } from 'log4js' import { LOG4JS_BASE_CATEGORY } from '../../../../config/const' +import { deriveFromKeyPairAndIndex, deriveFromKeyPairAndUuid } from '../../../../data/deriveKeyPair' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' import { Context } from '../../Context' +import { Balance } from '../../data/Balance' +import { CommunityContext } from '../../valibot.schema' export abstract class AbstractSyncRole { private items: T[] = [] @@ -14,9 +18,34 @@ export abstract class AbstractSyncRole { ) } + getAccountKeyPair(communityContext: CommunityContext, gradidoId: Uuidv4): KeyPairEd25519 { + return this.context.cache.getKeyPairSync(gradidoId, () => { + return deriveFromKeyPairAndIndex(deriveFromKeyPairAndUuid(communityContext.keyPair, gradidoId), 1) + }) + } + + getLastBalanceForUser(publicKey: MemoryBlockPtr, blockchain: InMemoryBlockchain, communityId: string = ''): Balance { + if (publicKey.isEmpty()) { + throw new Error('publicKey is empty') + } + const lastSenderTransaction = blockchain.findOne(Filter.lastBalanceFor(publicKey)) + if (!lastSenderTransaction) { + return new Balance(publicKey, communityId) + } + const lastConfirmedTransaction = lastSenderTransaction.getConfirmedTransaction() + if (!lastConfirmedTransaction) { + throw new Error(`invalid transaction, getConfirmedTransaction call failed for transaction nr: ${lastSenderTransaction.getTransactionNr()}`) + } + const senderLastAccountBalance = lastConfirmedTransaction.getAccountBalance(publicKey, communityId) + if (!senderLastAccountBalance) { + return new Balance(publicKey, communityId) + } + return Balance.fromAccountBalance(senderLastAccountBalance, lastConfirmedTransaction.getConfirmedAt().getDate()) + } + abstract getDate(): Date abstract loadFromDb(offset: number, count: number): Promise - abstract pushToBlockchain(item: T): Promise + abstract pushToBlockchain(item: T): void abstract itemTypeName(): string // return count of new loaded items @@ -38,11 +67,11 @@ export abstract class AbstractSyncRole { return 0 } - async toBlockchain(): Promise { + toBlockchain(): void { if (this.isEmpty()) { throw new Error(`[toBlockchain] No items, please call this only if isEmpty returns false`) } - await this.pushToBlockchain(this.shift()) + this.pushToBlockchain(this.shift()) } peek(): T { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts index 27908fa19..41f488d4b 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts @@ -1,10 +1,14 @@ +import { and, asc, eq, isNotNull } from 'drizzle-orm' import * as v from 'valibot' import { Context } from '../../Context' -import { adminUsers, contributionLinkModerators, loadContributionLinkTransactions } from '../../database' -import { CreatedUserDb, TransactionDb, transactionDbSchema } from '../../valibot.schema' -import { TransactionsSyncRole } from './TransactionsSync.role' +import { contributionLinkModerators } from '../../database' +import { CreationTransactionDb, creationTransactionDbSchema } from '../../valibot.schema' +import { CreationsSyncRole } from './CreationsSync.role' +import { contributionsTable, usersTable } from '../../drizzle.schema' +import { ContributionStatus } from '../../data/ContributionStatus' +import { DatabaseError } from '../../errors' -export class ContributionLinkTransactionSyncRole extends TransactionsSyncRole { +export class ContributionLinkTransactionSyncRole extends CreationsSyncRole { constructor(readonly context: Context) { super(context) } @@ -12,24 +16,42 @@ export class ContributionLinkTransactionSyncRole extends TransactionsSyncRole { return 'contributionLinkTransaction' } - async loadFromDb(offset: number, count: number): Promise { - const transactionUsers = await loadContributionLinkTransactions(this.context.db, offset, count) - return transactionUsers.map((transactionUser) => { - let linkedUser: CreatedUserDb | null | undefined = null - linkedUser = contributionLinkModerators.get(transactionUser.contributionLinkId) - if (linkedUser?.gradidoId === transactionUser.user.gradidoId) { - for (const adminUser of adminUsers.values()) { - if (adminUser.gradidoId !== transactionUser.user.gradidoId) { - linkedUser = adminUser - break - } - } - } - return v.parse(transactionDbSchema, { - ...transactionUser.transaction, - user: transactionUser.user, - linkedUser, + async loadFromDb(offset: number, count: number): Promise { + const result = await this.context.db + .select({ + contribution: contributionsTable, + user: usersTable, }) - }) + .from(contributionsTable) + .where(and( + isNotNull(contributionsTable.contributionLinkId), + eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED), + )) + .innerJoin(usersTable, eq(contributionsTable.userId, usersTable.id)) + .orderBy(asc(contributionsTable.confirmedAt), asc(contributionsTable.transactionId)) + .limit(count) + .offset(offset) + + const verifiedCreationTransactions: CreationTransactionDb[] = [] + for(const row of result) { + if (!row.contribution.contributionLinkId) { + throw new Error(`expect contributionLinkId to be set: ${JSON.stringify(row.contribution, null, 2)}`) + } + const item = { + ...row.contribution, + user: row.user, + confirmedByUser: contributionLinkModerators.get(row.contribution.contributionLinkId) + } + if (!item.confirmedByUser || item.userId === item.confirmedByUser.id) { + this.context.logger.warn(`skipped Contribution Link Transaction ${row.contribution.id}`) + continue + } + try { + verifiedCreationTransactions.push(v.parse(creationTransactionDbSchema, item)) + } catch (e) { + throw new DatabaseError('load contributions with contribution link id', item, e as Error) + } + } + return verifiedCreationTransactions } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/CreationsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/CreationsSync.role.ts new file mode 100644 index 000000000..ff23e1fcb --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/CreationsSync.role.ts @@ -0,0 +1,132 @@ +import { and, asc, eq, isNull } from 'drizzle-orm' +import { alias } from 'drizzle-orm/mysql-core' +import { + AccountBalances, + AuthenticatedEncryption, + EncryptedMemo, + GradidoTransactionBuilder, + KeyPairEd25519, + MemoryBlockPtr, + TransferAmount +} from 'gradido-blockchain-js' +import * as v from 'valibot' +import { addToBlockchain } from '../../blockchain' +import { ContributionStatus } from '../../data/ContributionStatus' +import { + contributionsTable, + usersTable +} from '../../drizzle.schema' +import { BlockchainError, DatabaseError } from '../../errors' +import { CommunityContext, CreationTransactionDb, creationTransactionDbSchema } from '../../valibot.schema' +import { AbstractSyncRole } from './AbstractSync.role' + +export class CreationsSyncRole extends AbstractSyncRole { + + getDate(): Date { + return this.peek().confirmedAt + } + + itemTypeName(): string { + return 'creationTransactions' + } + + async loadFromDb(offset: number, count: number): Promise { + const confirmedByUsers = alias(usersTable, 'confirmedByUser') + const result = await this.context.db + .select({ + contribution: contributionsTable, + user: usersTable, + confirmedByUser: confirmedByUsers, + }) + .from(contributionsTable) + .where(and( + isNull(contributionsTable.contributionLinkId), + eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED), + )) + .innerJoin(usersTable, eq(contributionsTable.userId, usersTable.id)) + .innerJoin(confirmedByUsers, eq(contributionsTable.confirmedBy, confirmedByUsers.id)) + .orderBy(asc(contributionsTable.confirmedAt), asc(contributionsTable.transactionId)) + .limit(count) + .offset(offset) + + return result.map((row) => { + const item = { + ...row.contribution, + user: row.user, + confirmedByUser: row.confirmedByUser, + } + try { + return v.parse(creationTransactionDbSchema, item) + } catch (e) { + throw new DatabaseError('loadCreations', item, e as Error) + } + }) + } + + buildTransaction( + item: CreationTransactionDb, + communityContext: CommunityContext, + recipientKeyPair: KeyPairEd25519, + signerKeyPair: KeyPairEd25519 + ): GradidoTransactionBuilder { + return new GradidoTransactionBuilder() + .setCreatedAt(item.confirmedAt) + .addMemo( + new EncryptedMemo( + item.memo, + new AuthenticatedEncryption(communityContext.keyPair), + new AuthenticatedEncryption(recipientKeyPair), + ), + ) + .setTransactionCreation( + new TransferAmount(recipientKeyPair.getPublicKey(), item.amount), + item.contributionDate, + ) + .sign(signerKeyPair) + } + + calculateAccountBalances( + item: CreationTransactionDb, + communityContext: CommunityContext, + recipientPublicKey: MemoryBlockPtr + ): AccountBalances { + const accountBalances = new AccountBalances() + const balance = this.getLastBalanceForUser(recipientPublicKey, communityContext.blockchain) + + // calculate decay since last balance with legacy calculation method + balance.updateLegacyDecay(item.amount, item.confirmedAt) + communityContext.aufBalance.updateLegacyDecay(item.amount, item.confirmedAt) + communityContext.gmwBalance.updateLegacyDecay(item.amount, item.confirmedAt) + + accountBalances.add(balance.getAccountBalance()) + accountBalances.add(communityContext.aufBalance.getAccountBalance()) + accountBalances.add(communityContext.gmwBalance.getAccountBalance()) + return accountBalances + } + + pushToBlockchain(item: CreationTransactionDb): void { + const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) + const blockchain = communityContext.blockchain + if (item.confirmedByUser.communityUuid !== item.user.communityUuid) { + throw new Error(`contribution was confirmed from other community: ${JSON.stringify(item, null, 2)}`) + } + + const recipientKeyPair = this.getAccountKeyPair(communityContext, item.user.gradidoId) + const recipientPublicKey = recipientKeyPair.getPublicKey() + const signerKeyPair = this.getAccountKeyPair(communityContext, item.confirmedByUser.gradidoId) + if (!recipientKeyPair || !signerKeyPair || !recipientPublicKey) { + throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`) + } + + try { + addToBlockchain( + this.buildTransaction(item, communityContext, recipientKeyPair, signerKeyPair), + blockchain, + item.id, + this.calculateAccountBalances(item, communityContext, recipientPublicKey), + ) + } catch(e) { + throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) + } + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts index fd1ae225f..c1c12e724 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts @@ -1,13 +1,148 @@ -import { loadDeletedTransactionLinks } from '../../database' -import { TransactionDb } from '../../valibot.schema' -import { TransactionsSyncRole } from './TransactionsSync.role' +import { CommunityContext, DeletedTransactionLinkDb, deletedTransactionLinKDbSchema } from '../../valibot.schema' +import { AbstractSyncRole } from './AbstractSync.role' +import { transactionLinksTable, usersTable } from '../../drizzle.schema' +import { and, lt, asc, isNotNull, eq } from 'drizzle-orm' +import * as v from 'valibot' +import { + AccountBalance, + AccountBalances, + Filter, + GradidoDeferredTransfer, + GradidoTransactionBuilder, + GradidoTransfer, + GradidoUnit, + KeyPairEd25519, + MemoryBlockPtr, + TransferAmount +} from 'gradido-blockchain-js' +import { deriveFromCode } from '../../../../data/deriveKeyPair' +import { addToBlockchain } from '../../blockchain' +import { BlockchainError, DatabaseError } from '../../errors' +import { Balance } from '../../data/Balance' + +export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { + getDate(): Date { + return this.peek().deletedAt + } -export class DeletedTransactionLinksSyncRole extends TransactionsSyncRole { itemTypeName(): string { return 'deletedTransactionLinks' } - async loadFromDb(offset: number, count: number): Promise { - return await loadDeletedTransactionLinks(this.context.db, offset, count) + async loadFromDb(offset: number, count: number): Promise { + const result = await this.context.db + .select({ + transactionLink: transactionLinksTable, + user: usersTable, + }) + .from(transactionLinksTable) + .where( + and( + isNotNull(transactionLinksTable.deletedAt), + lt(transactionLinksTable.deletedAt, transactionLinksTable.validUntil) + ) + ) + .innerJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id)) + .orderBy(asc(transactionLinksTable.deletedAt), asc(transactionLinksTable.id)) + .limit(count) + .offset(offset) + + return result.map((row) => { + const item = { + ...row.transactionLink, + user: row.user, + } + try { + return v.parse(deletedTransactionLinKDbSchema, item) + } catch (e) { + throw new DatabaseError('loadDeletedTransactionLinks', item, e as Error) + } + }) } + + buildTransaction( + item: DeletedTransactionLinkDb, + linkFundingTransactionNr: number, + restAmount: GradidoUnit, + senderKeyPair: KeyPairEd25519, + linkFundingPublicKey: MemoryBlockPtr, + ): GradidoTransactionBuilder { + return new GradidoTransactionBuilder() + .setCreatedAt(item.deletedAt) + .setRedeemDeferredTransfer( + linkFundingTransactionNr, + new GradidoTransfer( + new TransferAmount(senderKeyPair.getPublicKey(), restAmount), + linkFundingPublicKey, + ), + ) + .sign(senderKeyPair) + } + + calculateBalances( + item: DeletedTransactionLinkDb, + fundingTransaction: GradidoDeferredTransfer, + senderLastBalance: Balance, + communityContext: CommunityContext, + senderPublicKey: MemoryBlockPtr, + ): AccountBalances { + const accountBalances = new AccountBalances() + + const fundingUserLastBalance = this.getLastBalanceForUser(fundingTransaction.getSenderPublicKey()!, communityContext.blockchain) + fundingUserLastBalance.updateLegacyDecay(senderLastBalance.getBalance(), item.deletedAt) + + // account of link is set to zero, gdd will be send back to initiator + accountBalances.add(new AccountBalance(senderPublicKey, GradidoUnit.zero(), '')) + accountBalances.add(fundingUserLastBalance.getAccountBalance()) + return accountBalances + } + + pushToBlockchain(item: DeletedTransactionLinkDb): void { + const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) + const blockchain = communityContext.blockchain + + const senderKeyPair = deriveFromCode(item.code) + const senderPublicKey = senderKeyPair.getPublicKey() + + if (!senderKeyPair || !senderPublicKey) { + throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`) + } + + const transaction = blockchain.findOne(Filter.lastBalanceFor(senderPublicKey)) + if (!transaction) { + throw new Error(`expect transaction for code: ${item.code}`) + } + // should be funding transaction + if (!transaction.isDeferredTransfer()) { + throw new Error(`expect funding transaction: ${transaction.getConfirmedTransaction()?.toJson(true)}`) + } + const body = transaction.getConfirmedTransaction()?.getGradidoTransaction()?.getTransactionBody(); + const deferredTransfer = body?.getDeferredTransfer() + if (!deferredTransfer || !deferredTransfer.getRecipientPublicKey()?.equal(senderPublicKey)) { + throw new Error(`expect funding transaction to belong to code: ${item.code}: ${transaction.getConfirmedTransaction()?.toJson(true)}`) + } + const linkFundingPublicKey = deferredTransfer.getSenderPublicKey() + if (!linkFundingPublicKey) { + throw new Error(`missing sender public key of transaction link founder`) + } + const senderLastBalance = this.getLastBalanceForUser(senderPublicKey, communityContext.blockchain) + senderLastBalance.updateLegacyDecay(GradidoUnit.zero(), item.deletedAt) + + try { + addToBlockchain( + this.buildTransaction( + item, transaction.getTransactionNr(), + senderLastBalance.getBalance(), + senderKeyPair, + linkFundingPublicKey, + ), + blockchain, + item.id, + this.calculateBalances(item, deferredTransfer, senderLastBalance, communityContext, senderPublicKey), + ) + } catch(e) { + throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) + } + } + } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts new file mode 100644 index 000000000..63b03ff36 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts @@ -0,0 +1,151 @@ +import { and, asc, eq, isNotNull, isNull } from 'drizzle-orm' +import { alias } from 'drizzle-orm/mysql-core' +import { + AccountBalances, + AuthenticatedEncryption, + EncryptedMemo, + Filter, + GradidoTransactionBuilder, + KeyPairEd25519, + MemoryBlockPtr, + SearchDirection_DESC, + TransferAmount +} from 'gradido-blockchain-js' +import * as v from 'valibot' +import { addToBlockchain } from '../../blockchain' +import { TransactionTypeId } from '../../data/TransactionTypeId' +import { transactionsTable, usersTable } from '../../drizzle.schema' +import { BlockchainError, DatabaseError } from '../../errors' +import { CommunityContext, TransactionDb, transactionDbSchema } from '../../valibot.schema' +import { AbstractSyncRole } from './AbstractSync.role' + +export class LocalTransactionsSyncRole extends AbstractSyncRole { + + getDate(): Date { + return this.peek().balanceDate + } + + itemTypeName(): string { + return 'localTransactions' + } + + async loadFromDb(offset: number, count: number): Promise { + const linkedUsers = alias(usersTable, 'linkedUser') + const result = await this.context.db + .select({ + transaction: transactionsTable, + user: usersTable, + linkedUser: linkedUsers, + }) + .from(transactionsTable) + .where( + and( + eq(transactionsTable.typeId, TransactionTypeId.RECEIVE), + isNull(transactionsTable.transactionLinkId), + isNotNull(transactionsTable.linkedUserId), + eq(usersTable.foreign, 0), + eq(linkedUsers.foreign, 0), + ) + ) + .innerJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) + .innerJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id)) + .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) + .limit(count) + .offset(offset) + + return result.map((row) => { + const item = { + ...row.transaction, + user: row.user, + linkedUser: row.linkedUser, + } + try { + return v.parse(transactionDbSchema, item) + } catch (e) { + throw new DatabaseError('loadLocalTransferTransactions', item, e as Error) + } + }) + } + + buildTransaction( + item: TransactionDb, + senderKeyPair: KeyPairEd25519, + recipientKeyPair: KeyPairEd25519, + ): GradidoTransactionBuilder { + return new GradidoTransactionBuilder() + .setCreatedAt(item.balanceDate) + .addMemo( + new EncryptedMemo( + item.memo, + new AuthenticatedEncryption(senderKeyPair), + new AuthenticatedEncryption(recipientKeyPair), + ), + ) + .setTransactionTransfer( + new TransferAmount(senderKeyPair.getPublicKey(), item.amount), + recipientKeyPair.getPublicKey(), + ) + .sign(senderKeyPair) + } + + calculateBalances( + item: TransactionDb, + communityContext: CommunityContext, + senderPublicKey: MemoryBlockPtr, + recipientPublicKey: MemoryBlockPtr, + ): AccountBalances { + const accountBalances = new AccountBalances() + + const senderLastBalance = this.getLastBalanceForUser(senderPublicKey, communityContext.blockchain) + const recipientLastBalance = this.getLastBalanceForUser(recipientPublicKey, communityContext.blockchain) + + if (senderLastBalance.getAccountBalance().getBalance().lt(item.amount)) { + const f = new Filter() + f.updatedBalancePublicKey = senderPublicKey + f.searchDirection = SearchDirection_DESC + f.pagination.size = 5 + const lastTransactions = communityContext.blockchain.findAll(f) + for (let i = lastTransactions.size() - 1; i >= 0; i--) { + const tx = lastTransactions.get(i) + this.context.logger.error(`${tx?.getConfirmedTransaction()!.toJson(true)}`) + } + throw new Error(`sender has not enough balance (${senderLastBalance.getAccountBalance().getBalance().toString()}) to send ${item.amount.toString()} to ${recipientPublicKey.convertToHex()}`) + } + + senderLastBalance.updateLegacyDecay(item.amount.negated(), item.balanceDate) + recipientLastBalance.updateLegacyDecay(item.amount, item.balanceDate) + + accountBalances.add(senderLastBalance.getAccountBalance()) + accountBalances.add(recipientLastBalance.getAccountBalance()) + return accountBalances + } + + pushToBlockchain(item: TransactionDb): void { + const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) + const blockchain = communityContext.blockchain + if (item.linkedUser.communityUuid !== item.user.communityUuid) { + throw new Error(`transfer between user from different communities: ${JSON.stringify(item, null, 2)}`) + } + + // I use the received transaction so user and linked user are swapped and user is recipient and linkedUser ist sender + const senderKeyPair = this.getAccountKeyPair(communityContext, item.linkedUser.gradidoId) + const senderPublicKey = senderKeyPair.getPublicKey() + const recipientKeyPair = this.getAccountKeyPair(communityContext, item.user.gradidoId) + const recipientPublicKey = recipientKeyPair.getPublicKey() + + if (!senderKeyPair || !senderPublicKey || !recipientKeyPair || !recipientPublicKey) { + throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`) + } + + try { + addToBlockchain( + this.buildTransaction(item, senderKeyPair, recipientKeyPair), + blockchain, + item.id, + this.calculateBalances(item, communityContext, senderPublicKey, recipientPublicKey), + ) + } catch(e) { + throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) + } + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts new file mode 100644 index 000000000..b2d0fd707 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts @@ -0,0 +1,161 @@ +import { and, asc, eq, isNotNull, isNull } from 'drizzle-orm' +import { + AccountBalance, + AccountBalances, + AuthenticatedEncryption, + EncryptedMemo, + Filter, + GradidoDeferredTransfer, + GradidoTransactionBuilder, + GradidoTransfer, + GradidoUnit, + KeyPairEd25519, + MemoryBlockPtr, + TransferAmount +} from 'gradido-blockchain-js' +import * as v from 'valibot' +import { addToBlockchain } from '../../blockchain' +import { transactionLinksTable, usersTable } from '../../drizzle.schema' +import { BlockchainError, DatabaseError } from '../../errors' +import { CommunityContext, RedeemedTransactionLinkDb, redeemedTransactionLinkDbSchema } from '../../valibot.schema' +import { AbstractSyncRole } from './AbstractSync.role' +import { deriveFromCode } from '../../../../data/deriveKeyPair' +import { alias } from 'drizzle-orm/mysql-core' + +export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { + getDate(): Date { + return this.peek().redeemedAt + } + + itemTypeName(): string { + return 'redeemTransactionLinks' + } + + async loadFromDb(offset: number, count: number): Promise { + const redeemedByUser = alias(usersTable, 'redeemedByUser') + const result = await this.context.db + .select({ + transactionLink: transactionLinksTable, + user: usersTable, + redeemedBy: redeemedByUser, + }) + .from(transactionLinksTable) + .where( + and( + isNull(transactionLinksTable.deletedAt), + isNotNull(transactionLinksTable.redeemedAt), + eq(usersTable.foreign, 0), + eq(redeemedByUser.foreign, 0) + ) + ) + .innerJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id)) + .innerJoin(redeemedByUser, eq(transactionLinksTable.redeemedBy, redeemedByUser.id)) + .orderBy(asc(transactionLinksTable.redeemedAt), asc(transactionLinksTable.id)) + .limit(count) + .offset(offset) + + return result.map((row) => { + const item = { + ...row.transactionLink, + redeemedBy: row.redeemedBy, + user: row.user, + } + try { + return v.parse(redeemedTransactionLinkDbSchema, item) + } catch (e) { + throw new DatabaseError('loadRedeemTransactionLinks', item, e as Error) + } + }) + } + + buildTransaction( + item: RedeemedTransactionLinkDb, + linkFundingTransactionNr: number, + senderKeyPair: KeyPairEd25519, + recipientKeyPair: KeyPairEd25519, + ): GradidoTransactionBuilder { + return new GradidoTransactionBuilder() + .setCreatedAt(item.redeemedAt) + .addMemo( + new EncryptedMemo( + item.memo, + new AuthenticatedEncryption(senderKeyPair), + new AuthenticatedEncryption(recipientKeyPair), + ), + ) + .setRedeemDeferredTransfer( + linkFundingTransactionNr, + new GradidoTransfer( + new TransferAmount(senderKeyPair.getPublicKey(), item.amount), + recipientKeyPair.getPublicKey(), + ), + ) + .sign(senderKeyPair) + } + + calculateBalances( + item: RedeemedTransactionLinkDb, + fundingTransaction: GradidoDeferredTransfer, + communityContext: CommunityContext, + senderPublicKey: MemoryBlockPtr, + recipientPublicKey: MemoryBlockPtr, + ): AccountBalances { + const accountBalances = new AccountBalances() + + const senderLastBalance = this.getLastBalanceForUser(senderPublicKey, communityContext.blockchain) + const fundingUserLastBalance = this.getLastBalanceForUser(fundingTransaction.getSenderPublicKey()!, communityContext.blockchain) + const recipientLastBalance = this.getLastBalanceForUser(recipientPublicKey, communityContext.blockchain) + + if (senderLastBalance.getAccountBalance().getBalance().lt(item.amount)) { + throw new Error(`sender has not enough balance (${senderLastBalance.getAccountBalance().getBalance().toString()}) to send ${item.amount.toString()} to ${recipientPublicKey.convertToHex()}`) + } + senderLastBalance.updateLegacyDecay(item.amount.negated(), item.redeemedAt) + fundingUserLastBalance.updateLegacyDecay(senderLastBalance.getBalance(), item.redeemedAt) + recipientLastBalance.updateLegacyDecay(item.amount, item.redeemedAt) + + // account of link is set to zero, and change send back to link creator + accountBalances.add(new AccountBalance(senderPublicKey, GradidoUnit.zero(), '')) + accountBalances.add(recipientLastBalance.getAccountBalance()) + accountBalances.add(fundingUserLastBalance.getAccountBalance()) + return accountBalances + } + + pushToBlockchain(item: RedeemedTransactionLinkDb): void { + const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) + const blockchain = communityContext.blockchain + + const senderKeyPair = deriveFromCode(item.code) + const senderPublicKey = senderKeyPair.getPublicKey() + const recipientKeyPair = this.getAccountKeyPair(communityContext, item.redeemedBy.gradidoId) + const recipientPublicKey = recipientKeyPair.getPublicKey() + + if (!senderKeyPair || !senderPublicKey || !recipientKeyPair || !recipientPublicKey) { + throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`) + } + + const transaction = blockchain.findOne(Filter.lastBalanceFor(senderPublicKey)) + if (!transaction) { + throw new Error(`expect transaction for code: ${item.code}`) + } + // should be funding transaction + if (!transaction.isDeferredTransfer()) { + throw new Error(`expect funding transaction: ${transaction.getConfirmedTransaction()?.toJson(true)}`) + } + const body = transaction.getConfirmedTransaction()?.getGradidoTransaction()?.getTransactionBody(); + const deferredTransfer = body?.getDeferredTransfer() + if (!deferredTransfer || !deferredTransfer.getRecipientPublicKey()?.equal(senderPublicKey)) { + throw new Error(`expect funding transaction to belong to code: ${item.code}: ${transaction.getConfirmedTransaction()?.toJson(true)}`) + } + + try { + addToBlockchain( + this.buildTransaction(item, transaction.getTransactionNr(), senderKeyPair, recipientKeyPair), + blockchain, + item.id, + this.calculateBalances(item, deferredTransfer, communityContext, senderPublicKey, recipientPublicKey), + ) + } catch(e) { + throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) + } + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts new file mode 100644 index 000000000..a0dd98593 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts @@ -0,0 +1,138 @@ +import { asc, eq } from 'drizzle-orm' +import { + AccountBalance, + AccountBalances, + AuthenticatedEncryption, + DurationSeconds, + EncryptedMemo, + Filter, + GradidoTransactionBuilder, + GradidoTransfer, + GradidoUnit, + KeyPairEd25519, + MemoryBlockPtr, + SearchDirection_DESC, + TransferAmount +} from 'gradido-blockchain-js' +import * as v from 'valibot' +import { addToBlockchain } from '../../blockchain' +import { transactionLinksTable, usersTable } from '../../drizzle.schema' +import { BlockchainError, DatabaseError } from '../../errors' +import { CommunityContext, TransactionLinkDb, transactionLinkDbSchema } from '../../valibot.schema' +import { AbstractSyncRole } from './AbstractSync.role' +import { deriveFromCode } from '../../../../data/deriveKeyPair' + +export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { + getDate(): Date { + return this.peek().createdAt + } + + itemTypeName(): string { + return 'transactionLinkFundings' + } + + async loadFromDb(offset: number, count: number): Promise { + const result = await this.context.db + .select() + .from(transactionLinksTable) + .innerJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id)) + .orderBy(asc(transactionLinksTable.createdAt), asc(transactionLinksTable.id)) + .limit(count) + .offset(offset) + + return result.map((row) => { + const item = { + ...row.transaction_links, + user: row.users, + } + try { + return v.parse(transactionLinkDbSchema, item) + } catch (e) { + throw new DatabaseError('loadTransactionLinkFundings', item, e as Error) + } + }) + } + + buildTransaction( + item: TransactionLinkDb, + blockedAmount: GradidoUnit, + duration: DurationSeconds, + senderKeyPair: KeyPairEd25519, + recipientKeyPair: KeyPairEd25519, + ): GradidoTransactionBuilder { + return new GradidoTransactionBuilder() + .setCreatedAt(item.createdAt) + .addMemo( + new EncryptedMemo( + item.memo, + new AuthenticatedEncryption(senderKeyPair), + new AuthenticatedEncryption(recipientKeyPair), + ), + ) + .setDeferredTransfer( + new GradidoTransfer( + new TransferAmount(senderKeyPair.getPublicKey(), blockedAmount), + recipientKeyPair.getPublicKey(), + ), + duration, + ) + .sign(senderKeyPair) + } + + calculateBalances( + item: TransactionLinkDb, + blockedAmount: GradidoUnit, + communityContext: CommunityContext, + senderPublicKey: MemoryBlockPtr, + recipientPublicKey: MemoryBlockPtr, + ): AccountBalances { + const accountBalances = new AccountBalances() + + const senderLastBalance = this.getLastBalanceForUser(senderPublicKey, communityContext.blockchain) + if (senderLastBalance.getBalance().lt(blockedAmount)) { + const f = new Filter() + f.updatedBalancePublicKey = senderPublicKey + f.pagination.size = 4 + f.searchDirection = SearchDirection_DESC + const lastSenderTransactions = communityContext.blockchain.findAll(f) + this.context.logger.error(`sender hasn't enough balance: ${senderPublicKey.convertToHex()}, last ${lastSenderTransactions.size()} balance changing transactions:`) + for(let i = lastSenderTransactions.size() - 1; i >= 0; i--) { + const lastSenderTransaction = lastSenderTransactions.get(i) + this.context.logger.error(`${lastSenderTransaction?.getConfirmedTransaction()?.toJson(true)}`) + } + } + senderLastBalance.updateLegacyDecay(blockedAmount.negated(), item.createdAt) + + accountBalances.add(senderLastBalance.getAccountBalance()) + accountBalances.add(new AccountBalance(recipientPublicKey, blockedAmount, '')) + return accountBalances + } + + pushToBlockchain(item: TransactionLinkDb): void { + const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) + const blockchain = communityContext.blockchain + + const senderKeyPair = this.getAccountKeyPair(communityContext, item.user.gradidoId) + const senderPublicKey = senderKeyPair.getPublicKey() + const recipientKeyPair = deriveFromCode(item.code) + const recipientPublicKey = recipientKeyPair.getPublicKey() + + if (!senderKeyPair || !senderPublicKey || !recipientKeyPair || !recipientPublicKey) { + throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`) + } + + const duration = new DurationSeconds((item.validUntil.getTime() - item.createdAt.getTime()) / 1000) + const blockedAmount = item.amount.calculateCompoundInterest(duration.getSeconds()) + + try { + addToBlockchain( + this.buildTransaction(item, blockedAmount, duration, senderKeyPair, recipientKeyPair), + blockchain, + item.id, + this.calculateBalances(item, blockedAmount, communityContext, senderPublicKey, recipientPublicKey), + ) + } catch(e) { + throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) + } + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinksSync.role.ts deleted file mode 100644 index 6920732cc..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinksSync.role.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { AccountBalance, AccountBalances, Filter, MemoryBlockPtr, SearchDirection_DESC } from 'gradido-blockchain-js' -import { KeyPairIdentifierLogic } from '../../../../data/KeyPairIdentifier.logic' -import { ResolveKeyPair } from '../../../../interactions/resolveKeyPair/ResolveKeyPair.context' -import { addTransaction } from '../../blockchain' -import { transactionLinkDbToTransaction } from '../../convert' -import { loadTransactionLinks } from '../../database' -import { TransactionLinkDb } from '../../valibot.schema' -import { AbstractSyncRole } from './AbstractSync.role' - -export class TransactionLinksSyncRole extends AbstractSyncRole { - getDate(): Date { - return this.peek().createdAt - } - - itemTypeName(): string { - return 'transactionLinks' - } - - async loadFromDb(offset: number, count: number): Promise { - return await loadTransactionLinks(this.context.db, offset, count) - } - - async pushToBlockchain(item: TransactionLinkDb): Promise { - const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) - const transaction = transactionLinkDbToTransaction(item, communityContext.topicId) - // I use the receiving part of transaction pair, so the user is the recipient and the linked user the sender - const accountBalances = new AccountBalances() - const senderKeyPair = await ResolveKeyPair( - new KeyPairIdentifierLogic(transaction.user), - ) - const recipientKeyPair = await ResolveKeyPair( - new KeyPairIdentifierLogic(transaction.linkedUser!), - ) - const f = new Filter() - f.involvedPublicKey = senderKeyPair.getPublicKey() - f.pagination.size = 1 - f.searchDirection = SearchDirection_DESC - communityContext.blockchain.findOne(f) - accountBalances.add(new AccountBalance(senderKeyPair.getPublicKey(), item.linkedUserBalance, '')) - accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), item.amount, '')) - - - await addTransaction(communityContext.blockchain, communityContext.blockchain, transaction, item.id, accountBalances) - } -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionsSync.role.ts deleted file mode 100644 index 2ebab70ef..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionsSync.role.ts +++ /dev/null @@ -1,116 +0,0 @@ -import Decimal from 'decimal.js-light' -import { AccountBalance, AccountBalances, GradidoUnit } from 'gradido-blockchain-js' -import { InputTransactionType } from '../../../../data/InputTransactionType.enum' -import { KeyPairIdentifierLogic } from '../../../../data/KeyPairIdentifier.logic' -import { GradidoBlockchainCryptoError } from '../../../../errors' -import { ResolveKeyPair } from '../../../../interactions/resolveKeyPair/ResolveKeyPair.context' -import { AUF_ACCOUNT_DERIVATION_INDEX, GMW_ACCOUNT_DERIVATION_INDEX, hardenDerivationIndex } from '../../../../utils/derivationHelper' -import { addTransaction } from '../../blockchain' -import { transactionDbToTransaction } from '../../convert' -import { loadTransactions } from '../../database' -import { legacyCalculateDecay } from '../../utils' -import { TransactionDb } from '../../valibot.schema' -import { AbstractSyncRole } from './AbstractSync.role' - -type BalanceDate = { - balance: Decimal - date: Date -} - -export class TransactionsSyncRole extends AbstractSyncRole { - private static transactionLinkCodes = new Set() - static doubleTransactionLinkCodes: string[] = [] - - getDate(): Date { - return this.peek().balanceDate - } - - itemTypeName(): string { - return 'transactions' - } - - async loadFromDb(offset: number, count: number): Promise { - const result = await loadTransactions(this.context.db, offset, count) - return result.filter((item) => { - if (item.transactionLinkCode) { - if (TransactionsSyncRole.transactionLinkCodes.has(item.transactionLinkCode)) { - TransactionsSyncRole.doubleTransactionLinkCodes.push(item.transactionLinkCode) - return false - } - TransactionsSyncRole.transactionLinkCodes.add(item.transactionLinkCode) - } - return true - }) - } - - async pushToBlockchain(item: TransactionDb): Promise { - const senderCommunityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) - const recipientCommunityContext = this.context.getCommunityContextByUuid( - item.linkedUser.communityUuid, - ) - this.context.cache.setHomeCommunityTopicId(senderCommunityContext.topicId) - const transaction = transactionDbToTransaction( - item, - senderCommunityContext.topicId, - recipientCommunityContext.topicId, - ) - const accountBalances = new AccountBalances() - if (InputTransactionType.GRADIDO_CREATION === transaction.type) { - const recipientKeyPair = await ResolveKeyPair( - new KeyPairIdentifierLogic(transaction.linkedUser!), - ) - accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), item.balance, '')) - // update gmw and auf - this.updateGmwAuf(new Decimal(item.amount.toString(4)), item.balanceDate) - const communityKeyPair = await ResolveKeyPair(new KeyPairIdentifierLogic({ communityTopicId: senderCommunityContext.topicId })) - const gmwKeyPair = communityKeyPair.deriveChild( - hardenDerivationIndex(GMW_ACCOUNT_DERIVATION_INDEX), - ) - if (!gmwKeyPair) { - throw new GradidoBlockchainCryptoError( - `KeyPairEd25519 child derivation failed, has private key: ${communityKeyPair.hasPrivateKey()} for community: ${senderCommunityContext.communityId}`, - ) - } - const aufKeyPair = communityKeyPair.deriveChild( - hardenDerivationIndex(AUF_ACCOUNT_DERIVATION_INDEX), - ) - if (!aufKeyPair) { - throw new GradidoBlockchainCryptoError( - `KeyPairEd25519 child derivation failed, has private key: ${communityKeyPair.hasPrivateKey()} for community: ${senderCommunityContext.communityId}`, - ) - } - accountBalances.add(new AccountBalance(gmwKeyPair.getPublicKey(), GradidoUnit.fromString( - TransactionsSyncRole.gmwBalance!.balance.toString()), '')) - accountBalances.add(new AccountBalance(aufKeyPair.getPublicKey(), GradidoUnit.fromString( - TransactionsSyncRole.aufBalance!.balance.toString()), '')) - } else if (InputTransactionType.REGISTER_ADDRESS === transaction.type) { - const recipientKeyPair = await ResolveKeyPair( - new KeyPairIdentifierLogic(transaction.user), - ) - accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), GradidoUnit.zero(), '')) - } else { - // I use the receiving part of transaction pair, so the user is the recipient and the linked user the sender - const senderKeyPair = await ResolveKeyPair( - new KeyPairIdentifierLogic(transaction.linkedUser!), - ) - const recipientKeyPair = await ResolveKeyPair( - new KeyPairIdentifierLogic(transaction.user), - ) - accountBalances.add(new AccountBalance(senderKeyPair.getPublicKey(), item.linkedUserBalance, '')) - accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), item.balance, '')) - } - - try { - await addTransaction( - senderCommunityContext.blockchain, - recipientCommunityContext.blockchain, - transaction, - item.id, - accountBalances, - ) - } catch(e) { - this.context.logger.error(`error adding transaction: ${JSON.stringify(transaction, null, 2)}`) - throw e - } - } -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts index 03563468e..7f33a814d 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts @@ -1,11 +1,24 @@ -import { addTransaction } from '../../blockchain' -import { userDbToTransaction } from '../../convert' -import { loadUsers } from '../../database' -import { generateKeyPairUserAccount } from '../../data/keyPair' -import { CreatedUserDb } from '../../valibot.schema' +import { asc } from 'drizzle-orm' +import { + AccountBalance, + AccountBalances, + AddressType_COMMUNITY_HUMAN, + GradidoTransactionBuilder, + GradidoUnit, + KeyPairEd25519, + MemoryBlockPtr +} from 'gradido-blockchain-js' +import * as v from 'valibot' +import { deriveFromKeyPairAndUuid } from '../../../../data/deriveKeyPair' +import { Uuidv4Hash } from '../../../../data/Uuidv4Hash' +import { addToBlockchain } from '../../blockchain' +import { usersTable } from '../../drizzle.schema' +import { BlockchainError, DatabaseError } from '../../errors' +import { UserDb, userDbSchema } from '../../valibot.schema' import { AbstractSyncRole } from './AbstractSync.role' -export class UsersSyncRole extends AbstractSyncRole { +export class UsersSyncRole extends AbstractSyncRole { + getDate(): Date { return this.peek().createdAt } @@ -14,18 +27,66 @@ export class UsersSyncRole extends AbstractSyncRole { return 'users' } - async loadFromDb(offset: number, count: number): Promise { - const users = await loadUsers(this.context.db, offset, count) - for (const user of users) { - const communityContext = this.context.getCommunityContextByUuid(user.communityUuid) - await generateKeyPairUserAccount(user, this.context.cache, communityContext.topicId) - } - return users + async loadFromDb(offset: number, count: number): Promise { + const result = await this.context.db + .select() + .from(usersTable) + .orderBy(asc(usersTable.createdAt), asc(usersTable.id)) + .limit(count) + .offset(offset) + + return result.map((row) => { + try { + return v.parse(userDbSchema, row) + } catch (e) { + throw new DatabaseError('loadUsers', row, e as Error) + } + }) } - async pushToBlockchain(item: CreatedUserDb): Promise { + buildTransaction( + item: UserDb, + communityKeyPair: KeyPairEd25519, + accountKeyPair: KeyPairEd25519, + userKeyPair: KeyPairEd25519 + ): GradidoTransactionBuilder { + return new GradidoTransactionBuilder() + .setCreatedAt(item.createdAt) + .setRegisterAddress( + userKeyPair.getPublicKey(), + AddressType_COMMUNITY_HUMAN, + new Uuidv4Hash(item.gradidoId).getAsMemoryBlock(), + accountKeyPair.getPublicKey(), + ) + .sign(communityKeyPair) + .sign(accountKeyPair) + .sign(userKeyPair) + } + + calculateAccountBalances(accountPublicKey: MemoryBlockPtr): AccountBalances { + const accountBalances = new AccountBalances() + accountBalances.add(new AccountBalance(accountPublicKey, GradidoUnit.zero(), '')) + return accountBalances + } + + pushToBlockchain(item: UserDb): void { const communityContext = this.context.getCommunityContextByUuid(item.communityUuid) - const transaction = userDbToTransaction(item, communityContext.topicId) - return await addTransaction(communityContext.blockchain, communityContext.blockchain, transaction, item.id) + const userKeyPair = deriveFromKeyPairAndUuid(communityContext.keyPair, item.gradidoId) + const accountKeyPair = this.getAccountKeyPair(communityContext, item.gradidoId) + const accountPublicKey = accountKeyPair.getPublicKey() + if (!userKeyPair || !accountKeyPair || !accountPublicKey) { + throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`) + } + + try { + addToBlockchain( + this.buildTransaction(item, communityContext.keyPair, accountKeyPair, userKeyPair), + communityContext.blockchain, + item.id, + this.calculateAccountBalances(accountPublicKey), + ) + } catch (e) { + throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) + } } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts index 53e0fcf39..e3b2d059f 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts @@ -1,11 +1,13 @@ import { Profiler } from 'gradido-blockchain-js' import { Context } from '../../Context' +import { CreationsSyncRole } from './CreationsSync.role' +import { InvalidContributionTransactionSyncRole } from './InvalidContributionTransactionSync.role' +import { LocalTransactionsSyncRole } from './LocalTransactionsSync.role' +import { UsersSyncRole } from './UsersSync.role' +import { TransactionLinkFundingsSyncRole } from './TransactionLinkFundingsSync.role' +import { RedeemTransactionLinksSyncRole } from './RedeemTransactionLinksSync.role' import { ContributionLinkTransactionSyncRole } from './ContributionLinkTransactionSync.role' import { DeletedTransactionLinksSyncRole } from './DeletedTransactionLinksSync.role' -import { InvalidContributionTransactionSyncRole } from './InvalidContributionTransactionSync.role' -import { TransactionLinksSyncRole } from './TransactionLinksSync.role' -import { TransactionsSyncRole } from './TransactionsSync.role' -import { UsersSyncRole } from './UsersSync.role' export async function syncDbWithBlockchainContext(context: Context, batchSize: number) { const timeUsedDB = new Profiler() @@ -13,11 +15,12 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n const timeUsedAll = new Profiler() const containers = [ new UsersSyncRole(context), - new TransactionsSyncRole(context), - new DeletedTransactionLinksSyncRole(context), - new TransactionLinksSyncRole(context), - new InvalidContributionTransactionSyncRole(context), + new CreationsSyncRole(context), + new LocalTransactionsSyncRole(context), + new TransactionLinkFundingsSyncRole(context), + new RedeemTransactionLinksSyncRole(context), new ContributionLinkTransactionSyncRole(context), + new DeletedTransactionLinksSyncRole(context), ] let transactionsCount = 0 let transactionsCountSinceLastLog = 0 @@ -38,10 +41,12 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n } // sort by date, to ensure container on index 0 is the one with the smallest date - if (available.length > 0) { + if (available.length > 1) { + // const sortTime = new Profiler() available.sort((a, b) => a.getDate().getTime() - b.getDate().getTime()) + // context.logger.debug(`sorted ${available.length} containers in ${sortTime.string()}`) } - await available[0].toBlockchain() + available[0].toBlockchain() process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`) transactionsCount++ transactionsCountSinceLastLog++ @@ -57,8 +62,8 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n if (context.logger.isDebugEnabled()) { context.logger.debug(InvalidContributionTransactionSyncRole.allTransactionIds.join(', ')) } - context.logger.info(`Double linked transactions: ${TransactionsSyncRole.doubleTransactionLinkCodes.length}`) + /*context.logger.info(`Double linked transactions: ${TransactionsSyncRole.doubleTransactionLinkCodes.length}`) if (context.logger.isDebugEnabled()) { context.logger.debug(TransactionsSyncRole.doubleTransactionLinkCodes.join(', ')) - } + }*/ } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts index aa4a34685..581f77e3a 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts @@ -1,40 +1,36 @@ -import { InMemoryBlockchain } from 'gradido-blockchain-js' +import { InMemoryBlockchain, KeyPairEd25519 } from 'gradido-blockchain-js' import * as v from 'valibot' import { booleanSchema, dateSchema } from '../../schemas/typeConverter.schema' import { gradidoAmountSchema, - hieroIdSchema, identifierSeedSchema, memoSchema, uuidv4Schema, } from '../../schemas/typeGuard.schema' -import { TransactionTypeId } from './data/TransactionTypeId' import { Balance } from './data/Balance' +import { TransactionTypeId } from './data/TransactionTypeId' -export const createdUserDbSchema = v.object({ - id: v.pipe(v.number(), v.minValue(1)), +const positiveNumberSchema = v.pipe(v.number(), v.minValue(1)) + +export const userDbSchema = v.object({ + id: positiveNumberSchema, gradidoId: uuidv4Schema, communityUuid: uuidv4Schema, createdAt: dateSchema, }) -export const userDbSchema = v.object({ - gradidoId: uuidv4Schema, - communityUuid: uuidv4Schema, +export const transactionBaseSchema = v.object({ + id: positiveNumberSchema, + amount: gradidoAmountSchema, + memo: memoSchema, + user: userDbSchema, }) export const transactionDbSchema = v.pipe(v.object({ - id: v.pipe(v.number(), v.minValue(1)), + ...transactionBaseSchema.entries, typeId: v.enum(TransactionTypeId), - amount: gradidoAmountSchema, balanceDate: dateSchema, - balance: gradidoAmountSchema, - linkedUserBalance: gradidoAmountSchema, - memo: memoSchema, - creationDate: v.nullish(dateSchema), - user: createdUserDbSchema, - linkedUser: createdUserDbSchema, - transactionLinkCode: v.nullish(identifierSeedSchema), + linkedUser: userDbSchema, }), v.custom((value: any) => { if (value.user && value.linkedUser && !value.transactionLinkCode && value.user.gradidoId === value.linkedUser.gradidoId) { throw new Error(`expect user to be different from linkedUser: ${JSON.stringify(value, null, 2)}`) @@ -53,29 +49,62 @@ export const transactionDbSchema = v.pipe(v.object({ return value })) +export const creationTransactionDbSchema = v.pipe(v.object({ + ...transactionBaseSchema.entries, + contributionDate: dateSchema, + confirmedAt: dateSchema, + confirmedByUser: userDbSchema, +}), v.custom((value: any) => { + if (value.user && value.confirmedByUser && value.user.gradidoId === value.confirmedByUser.gradidoId) { + throw new Error(`expect user to be different from confirmedByUser: ${JSON.stringify(value, null, 2)}`) + } + // check that user and confirmedByUser exist before transaction balance date + const confirmedAt = new Date(value.confirmedAt) + if ( + value.user.createdAt.getTime() >= confirmedAt.getTime() || + value.confirmedByUser?.createdAt.getTime() >= confirmedAt.getTime() + ) { + throw new Error( + `at least one user was created after transaction confirmedAt date, logic error! ${JSON.stringify(value, null, 2)}`, + ) + } + + return value +})) + export const transactionLinkDbSchema = v.object({ - id: v.pipe(v.number(), v.minValue(1)), - user: userDbSchema, + ...transactionBaseSchema.entries, code: identifierSeedSchema, - amount: gradidoAmountSchema, - memo: memoSchema, createdAt: dateSchema, validUntil: dateSchema, }) +export const redeemedTransactionLinkDbSchema = v.object({ + ...transactionLinkDbSchema.entries, + redeemedAt: dateSchema, + redeemedBy: userDbSchema, +}) + +export const deletedTransactionLinKDbSchema = v.object({ + id: positiveNumberSchema, + user: userDbSchema, + code: identifierSeedSchema, + deletedAt: dateSchema, +}) + export const communityDbSchema = v.object({ + id: positiveNumberSchema, foreign: booleanSchema, communityUuid: uuidv4Schema, name: v.string(), creationDate: dateSchema, userMinCreatedAt: v.nullish(dateSchema), - uniqueAlias: v.string(), }) export const communityContextSchema = v.object({ communityId: v.string(), blockchain: v.instance(InMemoryBlockchain, 'expect InMemoryBlockchain type'), - topicId: hieroIdSchema, + keyPair: v.instance(KeyPairEd25519), folder: v.pipe( v.string(), v.minLength(1, 'expect string length >= 1'), @@ -87,9 +116,10 @@ export const communityContextSchema = v.object({ }) export type TransactionDb = v.InferOutput +export type CreationTransactionDb = v.InferOutput export type UserDb = v.InferOutput -export type CreatedUserDb = v.InferOutput export type TransactionLinkDb = v.InferOutput +export type RedeemedTransactionLinkDb = v.InferOutput +export type DeletedTransactionLinkDb = v.InferOutput export type CommunityDb = v.InferOutput -export type Balance = v.InferOutput export type CommunityContext = v.InferOutput From e1a94e3c9324eab1486ad26269ec6f88e490a0e8 Mon Sep 17 00:00:00 2001 From: Einhornimmond Date: Sun, 28 Dec 2025 15:31:29 +0100 Subject: [PATCH 12/50] more detailed error handling --- .../data/Balance.ts | 28 +++++++++-- .../db-v2.7.0_to_blockchain-v3.5/errors.ts | 12 +++++ .../syncDbWithBlockchain/AbstractSync.role.ts | 15 ++++++ .../LocalTransactionsSync.role.ts | 23 ++++----- .../TransactionLinkFundingsSync.role.ts | 50 ++++++++++++------- .../valibot.schema.ts | 2 + 6 files changed, 95 insertions(+), 35 deletions(-) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/Balance.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/Balance.ts index 718be3ff9..d9e36721f 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/Balance.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/Balance.ts @@ -1,6 +1,7 @@ import Decimal from 'decimal.js-light' import { AccountBalance, GradidoUnit, MemoryBlockPtr } from 'gradido-blockchain-js' import { legacyCalculateDecay } from '../utils' +import { NegativeBalanceError } from '../errors' export class Balance { private balance: GradidoUnit @@ -25,8 +26,15 @@ export class Balance { return this.balance } + getDate(): Date { + return this.date + } + updateLegacyDecay(amount: GradidoUnit, date: Date) { + // make sure to copy instead of referencing const previousBalanceString = this.balance.toString() + const previousDate = new Date(this.date.getTime()) + if (this.balance.equal(GradidoUnit.zero())) { this.balance = amount this.date = date @@ -37,12 +45,20 @@ export class Balance { this.date = date } if (this.balance.lt(GradidoUnit.zero())) { - throw new Error(`negative Gradido amount detected in Balance.updateLegacyDecay, previous balance: ${previousBalanceString}, amount: ${amount.toString()}`) + const previousDecayedBalance = legacyCalculateDecay(new Decimal(previousBalanceString), previousDate, date) + throw new NegativeBalanceError( + `negative Gradido amount detected in Balance.updateLegacyDecay`, + previousBalanceString, + amount.toString(), + previousDecayedBalance.toString(), + ) } } update(amount: GradidoUnit, date: Date) { - const previousBalanceString = this.balance.toString() + const previousBalance = new GradidoUnit(this.balance.toString()) + const previousDate = new Date(this.date.getTime()) + if (this.balance.equal(GradidoUnit.zero())) { this.balance = amount this.date = date @@ -53,7 +69,13 @@ export class Balance { this.date = date } if (this.balance.lt(GradidoUnit.zero())) { - throw new Error(`negative Gradido amount detected in Balance.update, previous balance: ${previousBalanceString}, amount: ${amount.toString()}`) + const previousDecayedBalance = this.balance.calculateDecay(previousDate, date) + throw new NegativeBalanceError( + `negative Gradido amount detected in Balance.update`, + previousBalance.toString(), + amount.toString(), + previousDecayedBalance.toString(), + ) } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts index 504ce76a4..64b3a28e7 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts @@ -3,6 +3,7 @@ import * as v from 'valibot' export class NotEnoughGradidoBalanceError extends Error { constructor(public needed: number, public exist: number) { super(`Not enough Gradido Balance for send coins, needed: ${needed} Gradido, exist: ${exist} Gradido`) + this.name = 'NotEnoughGradidoBalanceError' } } @@ -42,3 +43,14 @@ export class BlockchainError extends Error { this.stack = originalError.stack } } + +export class NegativeBalanceError extends Error { + constructor(message: string, previousBalanceString: string, amount: string, previousDecayedBalance: string) { + const parts: string[] = [`NegativeBalanceError in ${message}`] + parts.push(`Previous balance: ${previousBalanceString}`) + parts.push(`Amount: ${amount}`) + parts.push(`Previous decayed balance: ${previousDecayedBalance}`) + super(parts.join('\n')) + this.name = 'NegativeBalanceError' + } +} \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts index df6ba0f19..307b2933c 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts @@ -43,6 +43,21 @@ export abstract class AbstractSyncRole { return Balance.fromAccountBalance(senderLastAccountBalance, lastConfirmedTransaction.getConfirmedAt().getDate()) } + logLastBalanceChangingTransactions(publicKey: MemoryBlockPtr, blockchain: InMemoryBlockchain, transactionCount: number = 5) { + if (!this.context.logger.isDebugEnabled()) { + return + } + const f = new Filter() + f.updatedBalancePublicKey = publicKey + f.searchDirection = SearchDirection_DESC + f.pagination.size = transactionCount + const lastTransactions = blockchain.findAll(f) + for (let i = lastTransactions.size() - 1; i >= 0; i--) { + const tx = lastTransactions.get(i) + this.context.logger.debug(`${tx?.getConfirmedTransaction()!.toJson(true)}`) + } + } + abstract getDate(): Date abstract loadFromDb(offset: number, count: number): Promise abstract pushToBlockchain(item: T): void diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts index 63b03ff36..6476d886b 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts @@ -15,7 +15,7 @@ import * as v from 'valibot' import { addToBlockchain } from '../../blockchain' import { TransactionTypeId } from '../../data/TransactionTypeId' import { transactionsTable, usersTable } from '../../drizzle.schema' -import { BlockchainError, DatabaseError } from '../../errors' +import { BlockchainError, DatabaseError, NegativeBalanceError, NotEnoughGradidoBalanceError } from '../../errors' import { CommunityContext, TransactionDb, transactionDbSchema } from '../../valibot.schema' import { AbstractSyncRole } from './AbstractSync.role' @@ -99,20 +99,14 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { const senderLastBalance = this.getLastBalanceForUser(senderPublicKey, communityContext.blockchain) const recipientLastBalance = this.getLastBalanceForUser(recipientPublicKey, communityContext.blockchain) - if (senderLastBalance.getAccountBalance().getBalance().lt(item.amount)) { - const f = new Filter() - f.updatedBalancePublicKey = senderPublicKey - f.searchDirection = SearchDirection_DESC - f.pagination.size = 5 - const lastTransactions = communityContext.blockchain.findAll(f) - for (let i = lastTransactions.size() - 1; i >= 0; i--) { - const tx = lastTransactions.get(i) - this.context.logger.error(`${tx?.getConfirmedTransaction()!.toJson(true)}`) + try { + senderLastBalance.updateLegacyDecay(item.amount.negated(), item.balanceDate) + } catch(e) { + if (e instanceof NegativeBalanceError) { + this.logLastBalanceChangingTransactions(senderPublicKey, communityContext.blockchain) + throw e } - throw new Error(`sender has not enough balance (${senderLastBalance.getAccountBalance().getBalance().toString()}) to send ${item.amount.toString()} to ${recipientPublicKey.convertToHex()}`) } - - senderLastBalance.updateLegacyDecay(item.amount.negated(), item.balanceDate) recipientLastBalance.updateLegacyDecay(item.amount, item.balanceDate) accountBalances.add(senderLastBalance.getAccountBalance()) @@ -145,6 +139,9 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { this.calculateBalances(item, communityContext, senderPublicKey, recipientPublicKey), ) } catch(e) { + if (e instanceof NotEnoughGradidoBalanceError) { + this.logLastBalanceChangingTransactions(senderPublicKey, blockchain) + } throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts index a0dd98593..574c02235 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts @@ -17,10 +17,12 @@ import { import * as v from 'valibot' import { addToBlockchain } from '../../blockchain' import { transactionLinksTable, usersTable } from '../../drizzle.schema' -import { BlockchainError, DatabaseError } from '../../errors' +import { BlockchainError, DatabaseError, NegativeBalanceError } from '../../errors' import { CommunityContext, TransactionLinkDb, transactionLinkDbSchema } from '../../valibot.schema' import { AbstractSyncRole } from './AbstractSync.role' import { deriveFromCode } from '../../../../data/deriveKeyPair' +import { legacyCalculateDecay } from '../../utils' +import Decimal from 'decimal.js-light' export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { getDate(): Date { @@ -87,22 +89,9 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole= 0; i--) { - const lastSenderTransaction = lastSenderTransactions.get(i) - this.context.logger.error(`${lastSenderTransaction?.getConfirmedTransaction()?.toJson(true)}`) - } - } + let senderLastBalance = this.getLastBalanceForUser(senderPublicKey, communityContext.blockchain) senderLastBalance.updateLegacyDecay(blockedAmount.negated(), item.createdAt) - + accountBalances.add(senderLastBalance.getAccountBalance()) accountBalances.add(new AccountBalance(recipientPublicKey, blockedAmount, '')) return accountBalances @@ -122,16 +111,39 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole Date: Mon, 29 Dec 2025 19:54:17 +0100 Subject: [PATCH 13/50] refactor migration code, fix holdAvailable Amount from transaction links on the fly --- dlt-connector/bun.lock | 4 - dlt-connector/package.json | 1 - .../sendToHiero/SendToHiero.context.ts | 9 +- .../binaryExport.ts | 39 ++- .../blockchain.ts | 111 +----- .../db-v2.7.0_to_blockchain-v3.5/convert.ts | 136 -------- .../db-v2.7.0_to_blockchain-v3.5/database.ts | 324 +----------------- .../drizzle.schema.ts | 9 +- .../db-v2.7.0_to_blockchain-v3.5/errors.ts | 3 +- .../db-v2.7.0_to_blockchain-v3.5/index.ts | 4 +- .../accountBalances/AbstractBalances.role.ts | 38 -- .../accountBalances/CreationBalances.role.ts | 33 -- .../DeferredTransferBalances.role.ts | 33 -- .../RedeemDeferredTransferBalances.role.ts | 30 -- .../RegisterAddressBalances.role.ts | 17 - .../accountBalances/TransferBalances.role.ts | 30 -- .../accountBalances.context.ts | 32 -- .../syncDbWithBlockchain/AbstractSync.role.ts | 43 ++- .../ContributionLinkTransactionSync.role.ts | 14 +- .../CreationsSync.role.ts | 31 +- .../DeletedTransactionLinksSync.role.ts | 22 +- .../DoubleLinkedTransactions.role.ts | 27 -- ...InvalidContributionTransactionSync.role.ts | 27 -- .../LocalTransactionsSync.role.ts | 23 +- .../RedeemTransactionLinksSync.role.ts | 22 +- .../TransactionLinkFundingsSync.role.ts | 69 +++- .../syncDbWithBlockchain/UsersSync.role.ts | 20 +- .../syncDbWithBlockchain.context.ts | 39 ++- .../db-v2.7.0_to_blockchain-v3.5/utils.ts | 28 +- .../valibot.schema.ts | 2 + 30 files changed, 281 insertions(+), 939 deletions(-) delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/convert.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/AbstractBalances.role.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/CreationBalances.role.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/DeferredTransferBalances.role.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RedeemDeferredTransferBalances.role.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RegisterAddressBalances.role.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/TransferBalances.role.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/accountBalances.context.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DoubleLinkedTransactions.role.ts delete mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/InvalidContributionTransactionSync.role.ts diff --git a/dlt-connector/bun.lock b/dlt-connector/bun.lock index 477af78c0..5337098a9 100644 --- a/dlt-connector/bun.lock +++ b/dlt-connector/bun.lock @@ -1,6 +1,5 @@ { "lockfileVersion": 1, - "configVersion": 0, "workspaces": { "": { "name": "dlt-connector", @@ -23,7 +22,6 @@ "decimal.js-light": "^2.5.1", "dotenv": "^10.0.0", "drizzle-orm": "^0.44.7", - "drizzle-valibot": "^0.4.2", "elysia": "1.3.8", "graphql-request": "^7.2.0", "jose": "^5.2.2", @@ -467,8 +465,6 @@ "drizzle-orm": ["drizzle-orm@0.44.7", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@prisma/client": "*", "@tidbcloud/serverless": "*", "@types/better-sqlite3": "*", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=7", "bun-types": "*", "expo-sqlite": ">=14.0.0", "gel": ">=2", "knex": "*", "kysely": "*", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@tidbcloud/serverless", "@types/better-sqlite3", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "gel", "knex", "kysely", "mysql2", "pg", "postgres", "sql.js", "sqlite3"] }, "sha512-quIpnYznjU9lHshEOAYLoZ9s3jweleHlZIAWR/jX9gAWNg/JhQ1wj0KGRf7/Zm+obRrYd9GjPVJg790QY9N5AQ=="], - "drizzle-valibot": ["drizzle-valibot@0.4.2", "", { "peerDependencies": { "drizzle-orm": ">=0.36.0", "valibot": ">=1.0.0-beta.7" } }, "sha512-tzjT7g0Di/HX7426marIy8IDtWODjPgrwvgscdevLQRUe5rzYzRhx6bDsYLdDFF9VI/eaYgnjNeF/fznWJoUjg=="], - "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="], diff --git a/dlt-connector/package.json b/dlt-connector/package.json index 1f4984b0b..e173d6ede 100644 --- a/dlt-connector/package.json +++ b/dlt-connector/package.json @@ -36,7 +36,6 @@ "decimal.js-light": "^2.5.1", "dotenv": "^10.0.0", "drizzle-orm": "^0.44.7", - "drizzle-valibot": "^0.4.2", "elysia": "1.3.8", "graphql-request": "^7.2.0", "jose": "^5.2.2", diff --git a/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts b/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts index 622addb28..f96bfe402 100644 --- a/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts +++ b/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts @@ -1,8 +1,8 @@ import { GradidoTransaction, HieroTransactionId, - InteractionSerialize, InteractionValidate, + LedgerAnchor, ValidateType_SINGLE, } from 'gradido-blockchain-js' import { getLogger } from 'log4js' @@ -61,11 +61,8 @@ export async function SendToHieroContext( role.getSenderCommunityTopicId(), ) - // serialize Hiero transaction ID and attach it to the builder for the inbound transaction - const transactionIdSerializer = new InteractionSerialize( - new HieroTransactionId(outboundHieroTransactionIdString), - ) - builder.setParentMessageId(transactionIdSerializer.run()) + // attach Hiero transaction ID to the builder for the inbound transaction + builder.setParentLedgerAnchor(new LedgerAnchor(new HieroTransactionId(outboundHieroTransactionIdString))) // build and validate inbound transaction const inboundTransaction = builder.buildInbound() diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/binaryExport.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/binaryExport.ts index 43459372b..9d34f5588 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/binaryExport.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/binaryExport.ts @@ -9,12 +9,13 @@ import { } from 'gradido-blockchain-js' import { CONFIG } from '../../config' import { Context } from './Context' -import { bytesToKbyte, calculateOneHashStep } from './utils' +import { bytesString, calculateOneHashStep } from './utils' import { CommunityContext } from './valibot.schema' export function exportAllCommunities(context: Context, batchSize: number) { const timeUsed = new Profiler() for (const communityContext of context.communities.values()) { + context.logger.info(`exporting community ${communityContext.communityId} to binary file`) exportCommunity(communityContext, context, batchSize) } context.logger.info(`time used for exporting communities to binary file: ${timeUsed.string()}`) @@ -25,36 +26,66 @@ export function exportCommunity( context: Context, batchSize: number, ) { + const timeUsed = new Profiler() // write as binary file for GradidoNode const f = new Filter() f.pagination.size = batchSize f.pagination.page = 1 f.searchDirection = SearchDirection_ASC const binFilePath = prepareFolder(communityContext) + let count = 0 + let printCount = 0 let lastTransactionCount = 0 + let triggeredTransactionsCount = 0 let hash = Buffer.alloc(32, 0) + const isDebug = context.logger.isDebugEnabled() + const printConsole = () => { + if (triggeredTransactionsCount > 0) { + process.stdout.write(`exported ${count} transactions + ${triggeredTransactionsCount} triggered from timeouted transaction links\r`) + } else { + process.stdout.write(`exported ${count} transactions\r`) + } + } do { const transactions = communityContext.blockchain.findAll(f) lastTransactionCount = transactions.size() for (let i = 0; i < lastTransactionCount; i++) { const confirmedTransaction = transactions.get(i)?.getConfirmedTransaction() - const transactionNr = f.pagination.page * batchSize + i + const transactionNr = (f.pagination.page - 2) * batchSize + i if (!confirmedTransaction) { throw new Error(`invalid TransactionEntry at index: ${transactionNr} `) } hash = exportTransaction(confirmedTransaction, hash, binFilePath) + if (confirmedTransaction?.getGradidoTransaction()?.getTransactionBody()?.isTimeoutDeferredTransfer()) { + triggeredTransactionsCount++ + } else { + count++ + } + if (isDebug) { + printConsole() + } else { + printCount++ + if (printCount >= 100) { + printConsole() + printCount = 0 + } + } } - f.pagination.page++ + f.pagination.page++ } while (lastTransactionCount === batchSize) + printConsole() + process.stdout.write(`\n`) fs.appendFileSync(binFilePath, hash!) context.logger.info( `binary file for community ${communityContext.communityId} written to ${binFilePath}`, ) + const sumTransactionsCount = ((f.pagination.page - 2) * batchSize) + lastTransactionCount + const fileSize = fs.statSync(binFilePath).size context.logger.info( - `transactions count: ${(f.pagination.page - 1) * batchSize + lastTransactionCount}, size: ${bytesToKbyte(fs.statSync(binFilePath).size)} KByte`, + `exported ${sumTransactionsCount} transactions (${bytesString(fileSize)}) in ${timeUsed.string()}`, ) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts index 1d473a1b2..d689f22e1 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts @@ -1,36 +1,20 @@ -import * as fs from 'node:fs' import { AccountBalances, Filter, GradidoTransactionBuilder, HieroAccountId, InMemoryBlockchain, - InteractionSerialize, - TransactionType_DEFERRED_TRANSFER, } from 'gradido-blockchain-js' import { getLogger } from 'log4js' -import * as v from 'valibot' import { LOG4JS_BASE_CATEGORY } from '../../config/const' -import { InputTransactionType } from '../../data/InputTransactionType.enum' -import { LinkedTransactionKeyPairRole } from '../../interactions/resolveKeyPair/LinkedTransactionKeyPair.role' -import { AbstractTransactionRole } from '../../interactions/sendToHiero/AbstractTransaction.role' import { CommunityRootTransactionRole } from '../../interactions/sendToHiero/CommunityRootTransaction.role' -import { CreationTransactionRole } from '../../interactions/sendToHiero/CreationTransaction.role' -import { DeferredTransferTransactionRole } from '../../interactions/sendToHiero/DeferredTransferTransaction.role' -import { RedeemDeferredTransferTransactionRole } from '../../interactions/sendToHiero/RedeemDeferredTransferTransaction.role' -import { RegisterAddressTransactionRole } from '../../interactions/sendToHiero/RegisterAddressTransaction.role' -import { TransferTransactionRole } from '../../interactions/sendToHiero/TransferTransaction.role' -import { Community, Transaction } from '../../schemas/transaction.schema' -import { identifierSeedSchema } from '../../schemas/typeGuard.schema' +import { Community } from '../../schemas/transaction.schema' import { NotEnoughGradidoBalanceError } from './errors' const logger = getLogger( `${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`, ) export const defaultHieroAccount = new HieroAccountId(0, 0, 2) -let transactionAddedToBlockchainSum = 0 -let addToBlockchainSum = 0 -const sizeBuffer = Buffer.alloc(2) export function addToBlockchain( builder: GradidoTransactionBuilder, @@ -39,30 +23,12 @@ export function addToBlockchain( accountBalances: AccountBalances, ): boolean { const transaction = builder.build() - const transactionSerializer = new InteractionSerialize(transaction) - const binTransaction = transactionSerializer.run() - if (!binTransaction) { - logger.error(`Failed to serialize transaction ${transaction.toJson(true)}`) - return false - } - const filePath = `${blockchain.getCommunityId()}.bin` - if (!addToBlockchainSum) { - // clear file - fs.writeFileSync(filePath, Buffer.alloc(0)) - } - sizeBuffer.writeUInt16LE(binTransaction.size(), 0) - fs.appendFileSync(filePath, sizeBuffer) - fs.appendFileSync(filePath, binTransaction.data()) - //*/ - try { const result = blockchain.createAndAddConfirmedTransactionExtern( transaction, transactionId, accountBalances, ) - // logger.info(`${transactionTypeToString(transaction.getTransactionBody()?.getTransactionType()!)} Transaction added in ${timeUsed.string()}`) - addToBlockchainSum++ return result } catch (error) { if (error instanceof Error) { @@ -98,78 +64,3 @@ export async function addCommunityRootTransaction( } } -export async function addTransaction( - senderBlockchain: InMemoryBlockchain, - _recipientBlockchain: InMemoryBlockchain, - transaction: Transaction, - transactionId: number, - accountBalances: AccountBalances, -): Promise { - - let debugTmpStr = '' - let role: AbstractTransactionRole - if (transaction.type === InputTransactionType.GRADIDO_CREATION) { - role = new CreationTransactionRole(transaction) - } else if (transaction.type === InputTransactionType.GRADIDO_TRANSFER) { - role = new TransferTransactionRole(transaction) - } else if (transaction.type === InputTransactionType.REGISTER_ADDRESS) { - role = new RegisterAddressTransactionRole(transaction) - } else if (transaction.type === InputTransactionType.GRADIDO_DEFERRED_TRANSFER) { - role = new DeferredTransferTransactionRole(transaction) - } else if (transaction.type === InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER) { - const seedKeyPairRole = new LinkedTransactionKeyPairRole( - v.parse(identifierSeedSchema, transaction.user.seed), - ) - const f = new Filter() - f.involvedPublicKey = seedKeyPairRole.generateKeyPair().getPublicKey() - f.transactionType = TransactionType_DEFERRED_TRANSFER - const deferredTransactions = senderBlockchain.findAll(f) - if (!deferredTransactions) { - throw new Error( - `redeem deferred transfer: couldn't find parent deferred transfer on Gradido Node for ${JSON.stringify(transaction, null, 2)} and public key from seed: ${f.involvedPublicKey?.convertToHex()}`, - ) - } - if (deferredTransactions.size() !== 1) { - logger.error( - `redeem deferred transfer: found ${deferredTransactions.size()} parent deferred transfer on Gradido Node for ${JSON.stringify(transaction, null, 2)} and public key from seed: ${f.involvedPublicKey?.convertToHex()}`, - ) - for(let i = 0; i < deferredTransactions.size(); i++) { - logger.error(`deferred transaction ${i}: ${deferredTransactions.get(i)?.getConfirmedTransaction()?.toJson(true)}`) - } - throw new Error( - `redeem deferred transfer: found ${deferredTransactions.size()} parent deferred transfer on Gradido Node for ${JSON.stringify(transaction, null, 2)} and public key from seed: ${f.involvedPublicKey?.convertToHex()}`, - ) - } - const deferredTransaction = deferredTransactions.get(0)! - const confirmedDeferredTransaction = deferredTransaction.getConfirmedTransaction() - if (!confirmedDeferredTransaction) { - throw new Error('redeem deferred transfer: invalid TransactionEntry') - } - debugTmpStr += `\nconfirmed deferred transaction: ${confirmedDeferredTransaction.toJson(true)} with filter: ${f.toJson(true)}` - role = new RedeemDeferredTransferTransactionRole( - transaction, - confirmedDeferredTransaction, - ) - } else { - throw new Error(`Transaction type ${transaction.type} not supported`) - } - const involvedUser = transaction.user.account - ? transaction.user.account.userUuid - : transaction.linkedUser?.account?.userUuid - try { - if (addToBlockchain(await role.getGradidoTransactionBuilder(), senderBlockchain, transactionId, accountBalances)) { - // logger.debug(`${transaction.type} Transaction added for user ${involvedUser}`) - transactionAddedToBlockchainSum++ - } else { - logger.error(debugTmpStr) - logger.error(`transaction: ${JSON.stringify(transaction, null, 2)}`) - throw new Error(`${transaction.type} Transaction not added for user ${involvedUser}, after ${transactionAddedToBlockchainSum} transactions`) - } - } catch(e) { - if (e instanceof NotEnoughGradidoBalanceError) { - throw e - } - logger.error(`error adding transaction: ${JSON.stringify(transaction, null, 2)}`) - throw e - } -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/convert.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/convert.ts deleted file mode 100644 index d347bd8ff..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/convert.ts +++ /dev/null @@ -1,136 +0,0 @@ -import * as v from 'valibot' -import { AccountType } from '../../data/AccountType.enum' -import { InputTransactionType } from '../../data/InputTransactionType.enum' -import { - Community, - communitySchema, - Transaction, - TransactionInput, - transactionSchema, -} from '../../schemas/transaction.schema' -import { - gradidoAmountSchema, - HieroId, - memoSchema, - timeoutDurationSchema, -} from '../../schemas/typeGuard.schema' -import { TransactionTypeId } from './data/TransactionTypeId' -import { CommunityDb, CreatedUserDb, TransactionDb, TransactionLinkDb } from './valibot.schema' - -export function getInputTransactionTypeFromTypeId(typeId: TransactionTypeId): InputTransactionType { - switch (typeId) { - case TransactionTypeId.CREATION: - return InputTransactionType.GRADIDO_CREATION - case TransactionTypeId.SEND: - return InputTransactionType.GRADIDO_TRANSFER - case TransactionTypeId.RECEIVE: - throw new Error('not used') - default: - throw new Error('not implemented') - } -} - -export function communityDbToCommunity( - topicId: HieroId, - communityDb: CommunityDb, - creationDate: Date, -): Community { - return v.parse(communitySchema, { - hieroTopicId: topicId, - uuid: communityDb.communityUuid, - foreign: communityDb.foreign, - creationDate, - }) -} - -export function userDbToTransaction(userDb: CreatedUserDb, communityTopicId: HieroId): Transaction { - return v.parse(transactionSchema, { - user: { - communityTopicId: communityTopicId, - account: { userUuid: userDb.gradidoId }, - }, - type: InputTransactionType.REGISTER_ADDRESS, - accountType: AccountType.COMMUNITY_HUMAN, - createdAt: userDb.createdAt, - }) -} - -export function transactionDbToTransaction( - transactionDb: TransactionDb, - communityTopicId: HieroId, - recipientCommunityTopicId: HieroId, -): Transaction { - if ( - transactionDb.typeId !== TransactionTypeId.CREATION && - transactionDb.typeId !== TransactionTypeId.SEND && - transactionDb.typeId !== TransactionTypeId.RECEIVE - ) { - throw new Error('not implemented') - } - - const user = { - communityTopicId: communityTopicId, - account: { userUuid: transactionDb.user.gradidoId }, - } - const linkedUser = { - communityTopicId: recipientCommunityTopicId, - account: { userUuid: transactionDb.linkedUser.gradidoId }, - } - const transaction: TransactionInput = { - user, - linkedUser, - amount: v.parse(gradidoAmountSchema, transactionDb.amount), - memo: v.parse(memoSchema, transactionDb.memo), - type: InputTransactionType.GRADIDO_TRANSFER, - createdAt: transactionDb.balanceDate, - } - if (transactionDb.typeId === TransactionTypeId.CREATION) { - if (!transactionDb.creationDate) { - throw new Error('contribution transaction without creation date') - } - transaction.targetDate = transactionDb.creationDate - transaction.type = InputTransactionType.GRADIDO_CREATION - } else if (transactionDb.typeId === TransactionTypeId.RECEIVE) { - transaction.user = linkedUser - transaction.linkedUser = user - } - if (transactionDb.transactionLinkCode) { - if (transactionDb.typeId !== TransactionTypeId.RECEIVE) { - throw new Error( - "linked transaction which isn't receive, send will taken care of on link creation", - ) - } - transaction.user = { - communityTopicId: recipientCommunityTopicId, - seed: transactionDb.transactionLinkCode, - } - transaction.type = InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER - } - return v.parse(transactionSchema, transaction) -} - -export function transactionLinkDbToTransaction( - transactionLinkDb: TransactionLinkDb, - communityTopicId: HieroId, -): Transaction { - return v.parse(transactionSchema, { - user: { - communityTopicId: communityTopicId, - account: { userUuid: transactionLinkDb.user.gradidoId }, - }, - linkedUser: { - communityTopicId: communityTopicId, - seed: transactionLinkDb.code, - }, - type: InputTransactionType.GRADIDO_DEFERRED_TRANSFER, - amount: v.parse(gradidoAmountSchema, transactionLinkDb.amount), - memo: v.parse(memoSchema, transactionLinkDb.memo), - createdAt: transactionLinkDb.createdAt, - timeoutDuration: v.parse( - timeoutDurationSchema, - Math.round( - (transactionLinkDb.validUntil.getTime() - transactionLinkDb.createdAt.getTime()) / 1000, - ), - ), - }) -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts index a4a81b826..47b7372a2 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts @@ -1,36 +1,19 @@ -import { and, asc, count, eq, gt, inArray, isNotNull, isNull, lt, sql } from 'drizzle-orm' -import { alias } from 'drizzle-orm/mysql-core' +import { asc, eq, isNotNull, sql } from 'drizzle-orm' import { MySql2Database } from 'drizzle-orm/mysql2' import { getLogger } from 'log4js' import * as v from 'valibot' import { LOG4JS_BASE_CATEGORY } from '../../config/const' -import { ContributionStatus } from './data/ContributionStatus' -import { TransactionTypeId } from './data/TransactionTypeId' import { communitiesTable, - contributionsTable, eventsTable, - TransactionSelect, - transactionLinksTable, - transactionSelectSchema, - transactionsTable, - UserSelect, userRolesTable, - userSelectSchema, usersTable } from './drizzle.schema' -import { DatabaseError } from './errors' import { CommunityDb, UserDb, - CreationTransactionDb, communityDbSchema, userDbSchema, - creationTransactionDbSchema, - TransactionDb, - TransactionLinkDb, - transactionDbSchema, - transactionLinkDbSchema, } from './valibot.schema' const logger = getLogger( @@ -93,21 +76,6 @@ export async function loadCommunities(db: MySql2Database): Promise { - const result = await db - .select() - .from(usersTable) - .orderBy(asc(usersTable.createdAt), asc(usersTable.id)) - .limit(count) - .offset(offset) - - return result.map((row: any) => v.parse(userDbSchema, row)) -} - export async function loadUserByGradidoId(db: MySql2Database, gradidoId: string): Promise { const result = await db .select() @@ -118,293 +86,3 @@ export async function loadUserByGradidoId(db: MySql2Database, gradidoId: string) return result.length ? v.parse(userDbSchema, result[0]) : null } -export async function loadLocalTransferTransactions( - db: MySql2Database, - offset: number, - count: number, -): Promise { - const linkedUsers = alias(usersTable, 'linkedUser') - const result = await db - .select({ - transaction: transactionsTable, - user: usersTable, - linkedUser: linkedUsers, - }) - .from(transactionsTable) - .where( - and( - eq(transactionsTable.typeId, TransactionTypeId.RECEIVE), - isNull(transactionsTable.transactionLinkId), - isNotNull(transactionsTable.linkedUserId), - eq(usersTable.foreign, 0), - eq(linkedUsers.foreign, 0), - ) - ) - .leftJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) - .leftJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id)) - .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) - .limit(count) - .offset(offset) - - return result.map((row: any) => { - const item = { - ...row.transaction, - user: row.user, - linkedUser: row.linkedUser, - } - try { - return v.parse(transactionDbSchema, item) - } catch (e) { - throw new DatabaseError('loadLocalTransferTransactions', item, e as Error) - } - }) -} - -export async function loadTransactions( - db: MySql2Database, - offset: number, - count: number, -): Promise { - const linkedUsers = alias(usersTable, 'linkedUser') - - const result = await db - .select({ - transaction: transactionsTable, - user: usersTable, - linkedUser: linkedUsers, - transactionLink: { - id: transactionLinksTable.id, - code: transactionLinksTable.code - }, - }) - .from(transactionsTable) - .where( - and( - inArray(transactionsTable.typeId, [TransactionTypeId.CREATION, TransactionTypeId.RECEIVE]), - isNotNull(transactionsTable.linkedUserId), - eq(usersTable.foreign, 0) - ) - ) - .leftJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) - .leftJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id)) - .leftJoin( - transactionLinksTable, - eq(transactionsTable.transactionLinkId, transactionLinksTable.id), - ) - .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) - .limit(count) - .offset(offset) - - return result.map((row: any) => { - // console.log(row) - try { - /*if (transactionIdSet.has(row.transaction.id)) { - throw new Error(`transaction ${row.transaction.id} already loaded`) - } - transactionIdSet.add(row.transaction.id) - */ - return v.parse(transactionDbSchema, { - ...row.transaction, - transactionLinkCode: row.transactionLink ? row.transactionLink.code : null, - user: row.user, - linkedUser: row.linkedUser, - }) - } catch (e) { - logger.error(`table row: ${JSON.stringify(row, null, 2)}`) - if (e instanceof v.ValiError) { - logger.error(v.flatten(e.issues)) - } - throw e - } - }) -} - -export async function loadCreations( - db: MySql2Database, - offset: number, - count: number, -): Promise { - const confirmedByUsers = alias(usersTable, 'confirmedByUser') - const result = await db - .select({ - contribution: contributionsTable, - user: usersTable, - confirmedByUser: confirmedByUsers, - }) - .from(contributionsTable) - .where(and( - isNull(contributionsTable.contributionLinkId), - eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED), - )) - .leftJoin(usersTable, eq(contributionsTable.userId, usersTable.id)) - .leftJoin(confirmedByUsers, eq(contributionsTable.confirmedBy, confirmedByUsers.id)) - .orderBy(asc(contributionsTable.confirmedAt), asc(contributionsTable.id)) - .limit(count) - .offset(offset) - - return result.map((row) => { - const creationTransactionDb = { - ...row.contribution, - user: row.user, - confirmedByUser: row.confirmedByUser, - } - try { - return v.parse(creationTransactionDbSchema, creationTransactionDb) - } catch (e) { - throw new DatabaseError('loadCreations', creationTransactionDb, e as Error) - } - }) -} - -export async function loadInvalidContributionTransactions( - db: MySql2Database, - offset: number, - count: number, -): Promise<{ id: number, balanceDate: Date }[]> { - const result = await db - .select({ - id: transactionsTable.id, - balanceDate: transactionsTable.balanceDate, - }) - .from(transactionsTable) - .where( - and( - eq(transactionsTable.typeId, TransactionTypeId.CREATION), - sql`NOT EXISTS (SELECT 1 FROM contributions WHERE contributions.transaction_id = transactions.id)`, - ) - ) - .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) - .limit(count) - .offset(offset) - - return result.map((row: any) => { - return { - id: row.id, - balanceDate: new Date(row.balanceDate), - } - }) -} - -export async function loadDoubleLinkedTransactions( - db: MySql2Database, - offset: number, - rowsCount: number, -): Promise<{ id: number, balanceDate: Date }[]> { - const result = await db - .select({ - id: transactionsTable.id, - balanceDate: transactionsTable.balanceDate, - transactionLinkId: transactionsTable.transactionLinkId, - cnt: count(), - }) - .from(transactionsTable) - .where( - and( - eq(transactionsTable.typeId, TransactionTypeId.RECEIVE), - isNotNull(transactionsTable.transactionLinkId), - ) - ) - .groupBy(transactionsTable.transactionLinkId) - .having(gt(count(), 1)) - .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) - .limit(rowsCount) - .offset(offset) - - // logger.info(`loadDoubleLinkedTransactions ${result.length}: ${timeUsed.string()}`) - - return result.map((row: any) => { - return { - id: row.transactionLinkId, - balanceDate: new Date(row.balanceDate), - } - }) -} - -export async function loadContributionLinkTransactions( - db: MySql2Database, - offset: number, - count: number, -): Promise<{ transaction: TransactionSelect, user: UserSelect, contributionLinkId: number }[]> { - const result = await db - .select({ - transaction: transactionsTable, - user: usersTable, - contributionLinkId: contributionsTable.contributionLinkId, - }) - .from(contributionsTable) - .where( - and( - isNotNull(contributionsTable.contributionLinkId), - isNull(transactionsTable.linkedUserId) - ) - ) - .leftJoin(transactionsTable, eq(contributionsTable.transactionId, transactionsTable.id)) - .leftJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) - .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) - .limit(count) - .offset(offset) - - return result.map((row: any) => { - if (transactionIdSet.has(row.transaction.id)) { - throw new Error(`transaction ${row.transaction.id} already loaded`) - } - transactionIdSet.add(row.transaction.id) - return { - transaction: v.parse(transactionSelectSchema, row.transaction), - user: v.parse(userSelectSchema, row.user), - contributionLinkId: row.contributionLinkId, - } - }) -} - -export async function loadTransactionLinks( - db: MySql2Database, - offset: number, - count: number, -): Promise { - const result = await db - .select() - .from(transactionLinksTable) - .leftJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id)) - .orderBy(asc(transactionLinksTable.createdAt), asc(transactionLinksTable.id)) - .limit(count) - .offset(offset) - - return result.map((row: any) => { - return v.parse(transactionLinkDbSchema, { - ...row.transaction_links, - user: row.users, - }) - }) -} - -export async function loadDeletedTransactionLinks( - db: MySql2Database, - offset: number, - count: number, -): Promise { - const result = await db - .select() - .from(transactionLinksTable) - .leftJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id)) - .where(and( - isNotNull(transactionLinksTable.deletedAt), - lt(transactionLinksTable.deletedAt, transactionLinksTable.validUntil) - )) - .orderBy(asc(transactionLinksTable.deletedAt), asc(transactionLinksTable.id)) - .limit(count) - .offset(offset) - - return result.map((row: any) => { - return v.parse(transactionDbSchema, { - id: row.transaction_links.id, - typeId: TransactionTypeId.RECEIVE, - amount: row.transaction_links.amount, - balanceDate: new Date(row.transaction_links.deletedAt), - memo: row.transaction_links.memo, - transactionLinkCode: row.transaction_links.code, - user: row.users, - linkedUser: row.users, - }) - }) -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts index 3b7b2ec45..ac708c7ed 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts @@ -10,8 +10,6 @@ import { unique, varchar, } from 'drizzle-orm/mysql-core' -import { createSelectSchema } from 'drizzle-valibot' -import * as v from 'valibot' // use only fields needed in the migration, after update the rest of the project, import database instead export const communitiesTable = mysqlTable( @@ -62,9 +60,6 @@ export const usersTable = mysqlTable( (table) => [unique('uuid_key').on(table.gradidoId, table.communityUuid)], ) -export const userSelectSchema = createSelectSchema(usersTable) -export type UserSelect = v.InferOutput - export const userRolesTable = mysqlTable('user_roles', { id: int().autoincrement().notNull(), userId: int('user_id').notNull(), @@ -94,13 +89,11 @@ export const transactionsTable = mysqlTable( (table) => [index('user_id').on(table.userId)], ) -export const transactionSelectSchema = createSelectSchema(transactionsTable) -export type TransactionSelect = v.InferOutput - export const transactionLinksTable = mysqlTable('transaction_links', { id: int().autoincrement().notNull(), userId: int().notNull(), amount: decimal({ precision: 40, scale: 20 }).notNull(), + holdAvailableAmount: decimal("hold_available_amount", { precision: 40, scale: 20 }).notNull(), memo: varchar({ length: 255 }).notNull(), code: varchar({ length: 24 }).notNull(), createdAt: datetime({ mode: 'string' }).notNull(), diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts index 64b3a28e7..0c18102f6 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts @@ -26,6 +26,7 @@ export class DatabaseError extends Error { super(parts.join('\n\n')) this.name = 'DatabaseError' + this.cause = originalError } } @@ -40,7 +41,7 @@ export class BlockchainError extends Error { super(parts.join('\n\n')) this.name = 'BlockchainError' - this.stack = originalError.stack + this.cause = originalError } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts index 0d7f69805..b200e8769 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts @@ -1,11 +1,9 @@ -import * as v from 'valibot' import { onShutdown } from '../../../../shared/src/helper/onShutdown' -import { uuidv4Schema } from '../../schemas/typeGuard.schema' import { exportAllCommunities } from './binaryExport' import { bootstrap } from './bootstrap' import { syncDbWithBlockchainContext } from './interaction/syncDbWithBlockchain/syncDbWithBlockchain.context' -const BATCH_SIZE = 500 +const BATCH_SIZE = 1000 async function main() { // prepare in memory blockchains diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/AbstractBalances.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/AbstractBalances.role.ts deleted file mode 100644 index b2433240e..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/AbstractBalances.role.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { AccountBalances, Filter, InMemoryBlockchain, SearchDirection_DESC } from 'gradido-blockchain-js' -import { KeyPairIdentifierLogic } from '../../../../data/KeyPairIdentifier.logic' -import { ResolveKeyPair } from '../../../../interactions/resolveKeyPair/ResolveKeyPair.context' -import { IdentifierAccount } from '../../../../schemas/account.schema' -import { Transaction } from '../../../../schemas/transaction.schema' -import { Context } from '../../Context' -import { Balance } from '../../data/Balance' - -export abstract class AbstractBalancesRole { - public constructor(protected transaction: Transaction) {} - - abstract getAccountBalances(context: Context): Promise - - async getLastBalanceForUser(identifierAccount: IdentifierAccount, blockchain: InMemoryBlockchain): Promise { - const userKeyPair = await ResolveKeyPair( - new KeyPairIdentifierLogic(identifierAccount), - ) - const f = new Filter() - f.involvedPublicKey = userKeyPair.getPublicKey() - f.pagination.size = 1 - f.searchDirection = SearchDirection_DESC - const lastSenderTransaction = blockchain.findOne(f) - if (!lastSenderTransaction) { - throw new Error(`no last transaction found for user: ${JSON.stringify(identifierAccount, null, 2)}`) - } - const lastConfirmedTransaction = lastSenderTransaction.getConfirmedTransaction() - if (!lastConfirmedTransaction) { - throw new Error(`invalid transaction, getConfirmedTransaction call failed for transaction nr: ${lastSenderTransaction.getTransactionNr()}`) - } - const senderLastAccountBalance = lastConfirmedTransaction.getAccountBalance(userKeyPair.getPublicKey(), '') - if (!senderLastAccountBalance) { - throw new Error( - `no sender account balance found for transaction nr: ${lastSenderTransaction.getTransactionNr()} and public key: ${userKeyPair.getPublicKey()?.convertToHex()}` - ) - } - return Balance.fromAccountBalance(senderLastAccountBalance, lastConfirmedTransaction.getConfirmedAt().getDate()) - } -} \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/CreationBalances.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/CreationBalances.role.ts deleted file mode 100644 index 45d8e2308..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/CreationBalances.role.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { AccountBalances } from 'gradido-blockchain-js' -import { Transaction } from '../../../../schemas/transaction.schema' -import { Context } from '../../Context' -import { TransactionDb } from '../../valibot.schema' -import { AbstractBalancesRole } from './AbstractBalances.role' - -export class CreationBalancesRole extends AbstractBalancesRole { - - constructor(transaction: Transaction, protected dbTransaction: TransactionDb) { - super(transaction) - } - - async getAccountBalances(context: Context): Promise { - if (this.dbTransaction.linkedUser.communityUuid !== this.dbTransaction.user.communityUuid) { - throw new Error('creation: both recipient and signer must belong to same community') - } - - const accountBalances = new AccountBalances() - const communityContext = context.getCommunityContextByUuid(this.dbTransaction.user.communityUuid) - const balance = await this.getLastBalanceForUser(this.transaction.user, communityContext.blockchain) - - // calculate decay since last balance with legacy calculation method - balance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate) - communityContext.aufBalance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate) - communityContext.gmwBalance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate) - - accountBalances.add(balance.getAccountBalance()) - accountBalances.add(communityContext.aufBalance.getAccountBalance()) - accountBalances.add(communityContext.gmwBalance.getAccountBalance()) - - return accountBalances - } -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/DeferredTransferBalances.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/DeferredTransferBalances.role.ts deleted file mode 100644 index 077c20ca7..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/DeferredTransferBalances.role.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { AccountBalance, AccountBalances, GradidoUnit } from 'gradido-blockchain-js' -import { KeyPairIdentifierLogic } from '../../../../data/KeyPairIdentifier.logic' -import { ResolveKeyPair } from '../../../../interactions/resolveKeyPair/ResolveKeyPair.context' -import { Transaction } from '../../../../schemas/transaction.schema' -import { Context } from '../../Context' -import { TransactionLinkDb } from '../../valibot.schema' -import { AbstractBalancesRole } from './AbstractBalances.role' - - -export class DeferredTransferBalancesRole extends AbstractBalancesRole { - constructor(transaction: Transaction, protected dbTransactionLink: TransactionLinkDb) { - super(transaction) - } - - async getAccountBalances(context: Context): Promise { - const senderCommunityContext = context.getCommunityContextByUuid(this.dbTransactionLink.user.communityUuid) - const accountBalances = new AccountBalances() - - const seededIdentifier = new KeyPairIdentifierLogic(this.transaction.linkedUser!) - if (!seededIdentifier.isSeedKeyPair()) { - throw new Error(`linked user is not a seed: ${JSON.stringify(this.transaction, null, 2)}`) - } - const seedKeyPair = await ResolveKeyPair(seededIdentifier) - const senderAccountBalance = await this.getLastBalanceForUser(this.transaction.user, senderCommunityContext.blockchain) - - let amount = GradidoUnit.fromString(this.dbTransactionLink.amount.toString()) - amount = amount.calculateCompoundInterest((this.dbTransactionLink.validUntil.getTime() - this.dbTransactionLink.createdAt.getTime()) / 60000) - senderAccountBalance.updateLegacyDecay(amount.negated(), this.dbTransactionLink.createdAt) - accountBalances.add(senderAccountBalance.getAccountBalance()) - accountBalances.add(new AccountBalance(seedKeyPair.getPublicKey(), amount, '')) - return accountBalances - } -} \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RedeemDeferredTransferBalances.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RedeemDeferredTransferBalances.role.ts deleted file mode 100644 index 4024c3dd3..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RedeemDeferredTransferBalances.role.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { AccountBalances } from 'gradido-blockchain-js' -import { Transaction } from '../../../../schemas/transaction.schema' -import { Context } from '../../Context' -import { TransactionDb } from '../../valibot.schema' -import { AbstractBalancesRole } from './AbstractBalances.role' - -export class RedeemDeferredTransferBalancesRole extends AbstractBalancesRole { - constructor(transaction: Transaction, protected dbTransaction: TransactionDb) { - super(transaction) - } - - async getAccountBalances(context: Context): Promise { - // I use the receiving part of transaction pair, so the user is the recipient and the linked user the sender and amount is positive - const senderCommunityContext = context.getCommunityContextByUuid(this.dbTransaction.linkedUser.communityUuid) - const recipientCommunityContext = context.getCommunityContextByUuid(this.dbTransaction.user.communityUuid) - const accountBalances = new AccountBalances() - - context.cache.setHomeCommunityTopicId(senderCommunityContext.topicId) - const senderLastBalance = await this.getLastBalanceForUser(this.transaction.linkedUser!, senderCommunityContext.blockchain) - context.cache.setHomeCommunityTopicId(recipientCommunityContext.topicId) - const recipientLastBalance = await this.getLastBalanceForUser(this.transaction.user, recipientCommunityContext.blockchain) - - senderLastBalance.updateLegacyDecay(this.dbTransaction.amount.negate(), this.dbTransaction.balanceDate) - recipientLastBalance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate) - - accountBalances.add(senderLastBalance.getAccountBalance()) - accountBalances.add(recipientLastBalance.getAccountBalance()) - return accountBalances - } -} \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RegisterAddressBalances.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RegisterAddressBalances.role.ts deleted file mode 100644 index 0c4c53454..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RegisterAddressBalances.role.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { AccountBalance, AccountBalances, GradidoUnit } from 'gradido-blockchain-js' -import { KeyPairIdentifierLogic } from '../../../../data/KeyPairIdentifier.logic' -import { ResolveKeyPair } from '../../../../interactions/resolveKeyPair/ResolveKeyPair.context' -import { Context } from '../../Context' -import { AbstractBalancesRole } from './AbstractBalances.role' - - -export class RegisterAddressBalancesRole extends AbstractBalancesRole { - async getAccountBalances(_context: Context): Promise { - const accountBalances = new AccountBalances() - const recipientKeyPair = await ResolveKeyPair( - new KeyPairIdentifierLogic(this.transaction.user), - ) - accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), GradidoUnit.zero(), '')) - return accountBalances - } -} \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/TransferBalances.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/TransferBalances.role.ts deleted file mode 100644 index c5182627d..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/TransferBalances.role.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { AccountBalances } from 'gradido-blockchain-js' -import { Transaction } from '../../../../schemas/transaction.schema' -import { Context } from '../../Context' -import { TransactionDb } from '../../valibot.schema' -import { AbstractBalancesRole } from './AbstractBalances.role' - -export class TransferBalancesRole extends AbstractBalancesRole { - constructor(transaction: Transaction, protected dbTransaction: TransactionDb) { - super(transaction) - } - - async getAccountBalances(context: Context): Promise { - // I use the receiving part of transaction pair, so the user is the recipient and the linked user the sender and amount is positive - const senderCommunityContext = context.getCommunityContextByUuid(this.dbTransaction.linkedUser.communityUuid) - const recipientCommunityContext = context.getCommunityContextByUuid(this.dbTransaction.user.communityUuid) - const accountBalances = new AccountBalances() - - context.cache.setHomeCommunityTopicId(senderCommunityContext.topicId) - const senderLastBalance = await this.getLastBalanceForUser(this.transaction.linkedUser!, senderCommunityContext.blockchain) - context.cache.setHomeCommunityTopicId(recipientCommunityContext.topicId) - const recipientLastBalance = await this.getLastBalanceForUser(this.transaction.user, recipientCommunityContext.blockchain) - - senderLastBalance.updateLegacyDecay(this.dbTransaction.amount.negate(), this.dbTransaction.balanceDate) - recipientLastBalance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate) - - accountBalances.add(senderLastBalance.getAccountBalance()) - accountBalances.add(recipientLastBalance.getAccountBalance()) - return accountBalances - } -} \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/accountBalances.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/accountBalances.context.ts deleted file mode 100644 index 6a2f63604..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/accountBalances.context.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { AccountBalances } from 'gradido-blockchain-js' -import * as v from 'valibot' -import { InputTransactionType } from '../../../../data/InputTransactionType.enum' -import { Transaction } from '../../../../schemas/transaction.schema' -import { Context } from '../../Context' -import { TransactionDb, TransactionLinkDb, transactionDbSchema, transactionLinkDbSchema } from '../../valibot.schema' -import { AbstractBalancesRole } from './AbstractBalances.role' -import { CreationBalancesRole } from './CreationBalances.role' -import { DeferredTransferBalancesRole } from './DeferredTransferBalances.role' -import { RedeemDeferredTransferBalancesRole } from './RedeemDeferredTransferBalances.role' -import { RegisterAddressBalancesRole } from './RegisterAddressBalances.role' -import { TransferBalancesRole } from './TransferBalances.role' - -export async function accountBalancesContext(transaction: Transaction, item: TransactionDb | TransactionLinkDb, context: Context): Promise { - let role: AbstractBalancesRole | null = null - if (InputTransactionType.GRADIDO_CREATION === transaction.type) { - role = new CreationBalancesRole(transaction, v.parse(transactionDbSchema, item)) - } else if (InputTransactionType.GRADIDO_TRANSFER === transaction.type) { - role = new TransferBalancesRole(transaction, v.parse(transactionDbSchema, item)) - } else if (InputTransactionType.GRADIDO_DEFERRED_TRANSFER === transaction.type) { - role = new DeferredTransferBalancesRole(transaction, v.parse(transactionLinkDbSchema, item)) - } else if (InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER === transaction.type) { - role = new RedeemDeferredTransferBalancesRole(transaction, v.parse(transactionDbSchema, item)) - } else if (InputTransactionType.REGISTER_ADDRESS === transaction.type) { - role = new RegisterAddressBalancesRole(transaction) - } - if (!role) { - throw new Error(`No role found for transaction type ${transaction.type}`) - } - return await role.getAccountBalances(context) -} - diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts index 307b2933c..adb84e0c9 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts @@ -7,9 +7,14 @@ import { Context } from '../../Context' import { Balance } from '../../data/Balance' import { CommunityContext } from '../../valibot.schema' -export abstract class AbstractSyncRole { - private items: T[] = [] - private offset = 0 +export type IndexType = { + date: Date + id: number +} + +export abstract class AbstractSyncRole { + private items: ItemType[] = [] + protected lastIndex: IndexType = { date: new Date(0), id: 0 } protected logger: Logger constructor(protected readonly context: Context) { @@ -54,13 +59,15 @@ export abstract class AbstractSyncRole { const lastTransactions = blockchain.findAll(f) for (let i = lastTransactions.size() - 1; i >= 0; i--) { const tx = lastTransactions.get(i) - this.context.logger.debug(`${tx?.getConfirmedTransaction()!.toJson(true)}`) + this.context.logger.debug(`${i}: ${tx?.getConfirmedTransaction()!.toJson(true)}`) } } abstract getDate(): Date - abstract loadFromDb(offset: number, count: number): Promise - abstract pushToBlockchain(item: T): void + // for using seek rather than offset pagination approach + abstract getLastIndex(): IndexType + abstract loadFromDb(lastIndex: IndexType, count: number): Promise + abstract pushToBlockchain(item: ItemType): void abstract itemTypeName(): string // return count of new loaded items @@ -70,12 +77,14 @@ export abstract class AbstractSyncRole { if (this.logger.isDebugEnabled()) { timeUsed = new Profiler() } - this.items = await this.loadFromDb(this.offset, batchSize) - this.offset += this.items.length - if (timeUsed && this.items.length) { - this.logger.debug( - `${timeUsed.string()} for loading ${this.items.length} ${this.itemTypeName()} from db`, - ) + this.items = await this.loadFromDb(this.lastIndex, batchSize) + if (this.length > 0) { + this.lastIndex = this.getLastIndex() + if (timeUsed) { + this.logger.debug( + `${timeUsed.string()} for loading ${this.items.length} ${this.itemTypeName()} from db`, + ) + } } return this.items.length } @@ -89,14 +98,20 @@ export abstract class AbstractSyncRole { this.pushToBlockchain(this.shift()) } - peek(): T { + peek(): ItemType { if (this.isEmpty()) { throw new Error(`[peek] No items, please call this only if isEmpty returns false`) } return this.items[0] } + peekLast(): ItemType { + if (this.isEmpty()) { + throw new Error(`[peekLast] No items, please call this only if isEmpty returns false`) + } + return this.items[this.items.length - 1] + } - shift(): T { + shift(): ItemType { const item = this.items.shift() if (!item) { throw new Error(`[shift] No items, shift return undefined`) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts index 41f488d4b..299f4cec4 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts @@ -1,4 +1,4 @@ -import { and, asc, eq, isNotNull } from 'drizzle-orm' +import { and, asc, eq, gt, isNotNull, or } from 'drizzle-orm' import * as v from 'valibot' import { Context } from '../../Context' import { contributionLinkModerators } from '../../database' @@ -7,6 +7,8 @@ import { CreationsSyncRole } from './CreationsSync.role' import { contributionsTable, usersTable } from '../../drizzle.schema' import { ContributionStatus } from '../../data/ContributionStatus' import { DatabaseError } from '../../errors' +import { IndexType } from './AbstractSync.role' +import { toMysqlDateTime } from '../../utils' export class ContributionLinkTransactionSyncRole extends CreationsSyncRole { constructor(readonly context: Context) { @@ -16,7 +18,7 @@ export class ContributionLinkTransactionSyncRole extends CreationsSyncRole { return 'contributionLinkTransaction' } - async loadFromDb(offset: number, count: number): Promise { + async loadFromDb(lastIndex: IndexType, count: number): Promise { const result = await this.context.db .select({ contribution: contributionsTable, @@ -26,11 +28,17 @@ export class ContributionLinkTransactionSyncRole extends CreationsSyncRole { .where(and( isNotNull(contributionsTable.contributionLinkId), eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED), + or( + gt(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), + and( + eq(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), + gt(contributionsTable.transactionId, lastIndex.id) + ) + ) )) .innerJoin(usersTable, eq(contributionsTable.userId, usersTable.id)) .orderBy(asc(contributionsTable.confirmedAt), asc(contributionsTable.transactionId)) .limit(count) - .offset(offset) const verifiedCreationTransactions: CreationTransactionDb[] = [] for(const row of result) { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/CreationsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/CreationsSync.role.ts index ff23e1fcb..17d1b3119 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/CreationsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/CreationsSync.role.ts @@ -1,12 +1,15 @@ -import { and, asc, eq, isNull } from 'drizzle-orm' +import { and, asc, eq, isNull, gt, or } from 'drizzle-orm' import { alias } from 'drizzle-orm/mysql-core' import { AccountBalances, AuthenticatedEncryption, EncryptedMemo, + Filter, GradidoTransactionBuilder, KeyPairEd25519, MemoryBlockPtr, + SearchDirection_DESC, + TransactionType_CREATION, TransferAmount } from 'gradido-blockchain-js' import * as v from 'valibot' @@ -18,7 +21,8 @@ import { } from '../../drizzle.schema' import { BlockchainError, DatabaseError } from '../../errors' import { CommunityContext, CreationTransactionDb, creationTransactionDbSchema } from '../../valibot.schema' -import { AbstractSyncRole } from './AbstractSync.role' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' +import { toMysqlDateTime } from '../../utils' export class CreationsSyncRole extends AbstractSyncRole { @@ -26,11 +30,16 @@ export class CreationsSyncRole extends AbstractSyncRole { return this.peek().confirmedAt } + getLastIndex(): IndexType { + const lastItem = this.peekLast() + return { date: lastItem.confirmedAt, id: lastItem.transactionId } + } + itemTypeName(): string { return 'creationTransactions' } - async loadFromDb(offset: number, count: number): Promise { + async loadFromDb(lastIndex: IndexType, count: number): Promise { const confirmedByUsers = alias(usersTable, 'confirmedByUser') const result = await this.context.db .select({ @@ -42,12 +51,18 @@ export class CreationsSyncRole extends AbstractSyncRole { .where(and( isNull(contributionsTable.contributionLinkId), eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED), + or( + gt(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), + and( + eq(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), + gt(contributionsTable.transactionId, lastIndex.id) + ) + ) )) .innerJoin(usersTable, eq(contributionsTable.userId, usersTable.id)) .innerJoin(confirmedByUsers, eq(contributionsTable.confirmedBy, confirmedByUsers.id)) .orderBy(asc(contributionsTable.confirmedAt), asc(contributionsTable.transactionId)) .limit(count) - .offset(offset) return result.map((row) => { const item = { @@ -126,6 +141,14 @@ export class CreationsSyncRole extends AbstractSyncRole { this.calculateAccountBalances(item, communityContext, recipientPublicKey), ) } catch(e) { + const f= new Filter() + f.transactionType = TransactionType_CREATION + f.searchDirection = SearchDirection_DESC + f.pagination.size = 1 + const lastContribution = blockchain.findOne(f) + if (lastContribution) { + this.context.logger.warn(`last contribution: ${lastContribution.getConfirmedTransaction()?.toJson(true)}`) + } throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts index c1c12e724..0111234b7 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts @@ -1,7 +1,7 @@ import { CommunityContext, DeletedTransactionLinkDb, deletedTransactionLinKDbSchema } from '../../valibot.schema' -import { AbstractSyncRole } from './AbstractSync.role' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' import { transactionLinksTable, usersTable } from '../../drizzle.schema' -import { and, lt, asc, isNotNull, eq } from 'drizzle-orm' +import { and, lt, asc, isNotNull, eq, or, gt } from 'drizzle-orm' import * as v from 'valibot' import { AccountBalance, @@ -19,17 +19,23 @@ import { deriveFromCode } from '../../../../data/deriveKeyPair' import { addToBlockchain } from '../../blockchain' import { BlockchainError, DatabaseError } from '../../errors' import { Balance } from '../../data/Balance' +import { toMysqlDateTime } from '../../utils' export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { getDate(): Date { return this.peek().deletedAt } + getLastIndex(): IndexType { + const lastItem = this.peekLast() + return { date: lastItem.deletedAt, id: lastItem.id } + } + itemTypeName(): string { return 'deletedTransactionLinks' } - async loadFromDb(offset: number, count: number): Promise { + async loadFromDb(lastIndex: IndexType, count: number): Promise { const result = await this.context.db .select({ transactionLink: transactionLinksTable, @@ -39,13 +45,19 @@ export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { const item = { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DoubleLinkedTransactions.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DoubleLinkedTransactions.role.ts deleted file mode 100644 index 136155a2c..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DoubleLinkedTransactions.role.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Context } from '../../Context' -import { loadDoubleLinkedTransactions } from '../../database' -import { AbstractSyncRole } from './AbstractSync.role' - -export class DoubleLinkedTransactionsSyncRole extends AbstractSyncRole<{ id: number, balanceDate: Date }> { - static allTransactionIds: number[] = [] - constructor(readonly context: Context) { - super(context) - } - itemTypeName(): string { - return 'doubleLinkedTransaction' - } - - async loadFromDb(offset: number, count: number): Promise<{ id: number, balanceDate: Date }[]> { - const result = await loadDoubleLinkedTransactions(this.context.db, offset, count) - DoubleLinkedTransactionsSyncRole.allTransactionIds.push(...result.map((r) => r.id)) - return result - } - - getDate(): Date { - return this.peek().balanceDate - } - - async pushToBlockchain(item: { id: number, balanceDate: Date }): Promise { - this.logger.warn(`Double transaction_links ${item.id} found.`) - } -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/InvalidContributionTransactionSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/InvalidContributionTransactionSync.role.ts deleted file mode 100644 index 616dd1fe4..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/InvalidContributionTransactionSync.role.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Context } from '../../Context' -import { loadInvalidContributionTransactions } from '../../database' -import { AbstractSyncRole } from './AbstractSync.role' - -export class InvalidContributionTransactionSyncRole extends AbstractSyncRole<{ id: number, balanceDate: Date }> { - static allTransactionIds: number[] = [] - constructor(readonly context: Context) { - super(context) - } - itemTypeName(): string { - return 'invalidContributionTransaction' - } - - async loadFromDb(offset: number, count: number): Promise<{ id: number, balanceDate: Date }[]> { - const result = await loadInvalidContributionTransactions(this.context.db, offset, count) - InvalidContributionTransactionSyncRole.allTransactionIds.push(...result.map((r) => r.id)) - return result - } - - getDate(): Date { - return this.peek().balanceDate - } - - async pushToBlockchain(item: { id: number, balanceDate: Date }): Promise { - this.logger.warn(`Invalid contribution transaction ${item.id} found.`) - } -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts index 6476d886b..62ee0c832 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts @@ -1,4 +1,4 @@ -import { and, asc, eq, isNotNull, isNull } from 'drizzle-orm' +import { and, asc, eq, isNotNull, isNull, gt, or } from 'drizzle-orm' import { alias } from 'drizzle-orm/mysql-core' import { AccountBalances, @@ -17,7 +17,8 @@ import { TransactionTypeId } from '../../data/TransactionTypeId' import { transactionsTable, usersTable } from '../../drizzle.schema' import { BlockchainError, DatabaseError, NegativeBalanceError, NotEnoughGradidoBalanceError } from '../../errors' import { CommunityContext, TransactionDb, transactionDbSchema } from '../../valibot.schema' -import { AbstractSyncRole } from './AbstractSync.role' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' +import { toMysqlDateTime } from '../../utils' export class LocalTransactionsSyncRole extends AbstractSyncRole { @@ -25,11 +26,16 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { return this.peek().balanceDate } + getLastIndex(): IndexType { + const lastItem = this.peekLast() + return { date: lastItem.balanceDate, id: lastItem.id } + } + itemTypeName(): string { return 'localTransactions' } - async loadFromDb(offset: number, count: number): Promise { + async loadFromDb(lastIndex: IndexType, count: number): Promise { const linkedUsers = alias(usersTable, 'linkedUser') const result = await this.context.db .select({ @@ -45,13 +51,19 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { isNotNull(transactionsTable.linkedUserId), eq(usersTable.foreign, 0), eq(linkedUsers.foreign, 0), + or( + gt(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)), + and( + eq(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)), + gt(transactionsTable.id, lastIndex.id) + ) + ) ) ) .innerJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) .innerJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id)) .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) .limit(count) - .offset(offset) return result.map((row) => { const item = { @@ -74,8 +86,7 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { ): GradidoTransactionBuilder { return new GradidoTransactionBuilder() .setCreatedAt(item.balanceDate) - .addMemo( - new EncryptedMemo( + .addMemo(new EncryptedMemo( item.memo, new AuthenticatedEncryption(senderKeyPair), new AuthenticatedEncryption(recipientKeyPair), diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts index b2d0fd707..810fed60d 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts @@ -1,4 +1,4 @@ -import { and, asc, eq, isNotNull, isNull } from 'drizzle-orm' +import { and, asc, eq, isNotNull, isNull, or, gt } from 'drizzle-orm' import { AccountBalance, AccountBalances, @@ -18,20 +18,26 @@ import { addToBlockchain } from '../../blockchain' import { transactionLinksTable, usersTable } from '../../drizzle.schema' import { BlockchainError, DatabaseError } from '../../errors' import { CommunityContext, RedeemedTransactionLinkDb, redeemedTransactionLinkDbSchema } from '../../valibot.schema' -import { AbstractSyncRole } from './AbstractSync.role' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' import { deriveFromCode } from '../../../../data/deriveKeyPair' import { alias } from 'drizzle-orm/mysql-core' +import { toMysqlDateTime } from '../../utils' export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { getDate(): Date { return this.peek().redeemedAt } + getLastIndex(): IndexType { + const lastItem = this.peekLast() + return { date: lastItem.redeemedAt, id: lastItem.id } + } + itemTypeName(): string { return 'redeemTransactionLinks' } - async loadFromDb(offset: number, count: number): Promise { + async loadFromDb(lastIndex: IndexType, count: number): Promise { const redeemedByUser = alias(usersTable, 'redeemedByUser') const result = await this.context.db .select({ @@ -45,14 +51,20 @@ export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { const item = { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts index 574c02235..6ee368252 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts @@ -1,17 +1,15 @@ -import { asc, eq } from 'drizzle-orm' +import { asc, eq, or, gt, and, isNull } from 'drizzle-orm' import { AccountBalance, AccountBalances, AuthenticatedEncryption, DurationSeconds, EncryptedMemo, - Filter, GradidoTransactionBuilder, GradidoTransfer, GradidoUnit, KeyPairEd25519, MemoryBlockPtr, - SearchDirection_DESC, TransferAmount } from 'gradido-blockchain-js' import * as v from 'valibot' @@ -19,9 +17,9 @@ import { addToBlockchain } from '../../blockchain' import { transactionLinksTable, usersTable } from '../../drizzle.schema' import { BlockchainError, DatabaseError, NegativeBalanceError } from '../../errors' import { CommunityContext, TransactionLinkDb, transactionLinkDbSchema } from '../../valibot.schema' -import { AbstractSyncRole } from './AbstractSync.role' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' import { deriveFromCode } from '../../../../data/deriveKeyPair' -import { legacyCalculateDecay } from '../../utils' +import { calculateEffectiveSeconds, reverseLegacyDecay, toMysqlDateTime } from '../../utils' import Decimal from 'decimal.js-light' export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { @@ -29,19 +27,30 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { + async loadFromDb(lastIndex: IndexType, count: number): Promise { const result = await this.context.db .select() .from(transactionLinksTable) .innerJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id)) + .where(or( + gt(transactionLinksTable.createdAt, toMysqlDateTime(lastIndex.date)), + and( + eq(transactionLinksTable.createdAt, toMysqlDateTime(lastIndex.date)), + gt(transactionLinksTable.id, lastIndex.id) + ) + )) .orderBy(asc(transactionLinksTable.createdAt), asc(transactionLinksTable.id)) .limit(count) - .offset(offset) - + return result.map((row) => { const item = { ...row.transaction_links, @@ -90,7 +99,15 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole item.validUntil.getTime()) { + endDateTime = item.validUntil.getTime() + } + let duration = new DurationSeconds((endDateTime - item.createdAt.getTime()) / 1000) + const hourInSeconds = 60 * 60 + if (duration.getSeconds() < hourInSeconds) { + duration = new DurationSeconds(hourInSeconds) + } + let blockedAmount = GradidoUnit.fromString(reverseLegacyDecay(new Decimal(item.amount.toString()), duration.getSeconds()).toString()) + blockedAmount = blockedAmount.add(GradidoUnit.fromGradidoCent(1)) + // let blockedAmount = decayedAmount.calculateCompoundInterest(duration.getSeconds()) let accountBalances: AccountBalances try { accountBalances = this.calculateBalances(item, blockedAmount, communityContext, senderPublicKey, recipientPublicKey) @@ -119,9 +159,14 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { @@ -23,17 +24,28 @@ export class UsersSyncRole extends AbstractSyncRole { return this.peek().createdAt } + getLastIndex(): IndexType { + const lastItem = this.peekLast() + return { date: lastItem.createdAt, id: lastItem.id } + } + itemTypeName(): string { return 'users' } - async loadFromDb(offset: number, count: number): Promise { + async loadFromDb(lastIndex: IndexType, count: number): Promise { const result = await this.context.db .select() .from(usersTable) + .where(or( + gt(usersTable.createdAt, toMysqlDateTime(lastIndex.date)), + and( + eq(usersTable.createdAt, toMysqlDateTime(lastIndex.date)), + gt(usersTable.id, lastIndex.id) + ) + )) .orderBy(asc(usersTable.createdAt), asc(usersTable.id)) .limit(count) - .offset(offset) return result.map((row) => { try { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts index e3b2d059f..37ab78373 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts @@ -1,7 +1,6 @@ import { Profiler } from 'gradido-blockchain-js' import { Context } from '../../Context' import { CreationsSyncRole } from './CreationsSync.role' -import { InvalidContributionTransactionSyncRole } from './InvalidContributionTransactionSync.role' import { LocalTransactionsSyncRole } from './LocalTransactionsSync.role' import { UsersSyncRole } from './UsersSync.role' import { TransactionLinkFundingsSyncRole } from './TransactionLinkFundingsSync.role' @@ -24,13 +23,15 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n ] let transactionsCount = 0 let transactionsCountSinceLastLog = 0 + let transactionsCountSinceLastPrint = 0 let available = containers + const isDebug = context.logger.isDebugEnabled() while (true) { timeUsedDB.reset() const results = await Promise.all(available.map((c) => c.ensureFilled(batchSize))) const loadedItemsCount = results.reduce((acc, c) => acc + c, 0) // log only, if at least one new item was loaded - if (loadedItemsCount && context.logger.isDebugEnabled()) { + if (loadedItemsCount && isDebug) { context.logger.debug(`${loadedItemsCount} new items loaded from db in ${timeUsedDB.string()}`) } @@ -46,24 +47,24 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n available.sort((a, b) => a.getDate().getTime() - b.getDate().getTime()) // context.logger.debug(`sorted ${available.length} containers in ${sortTime.string()}`) } - available[0].toBlockchain() - process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`) + available[0].toBlockchain() transactionsCount++ - transactionsCountSinceLastLog++ - if (transactionsCountSinceLastLog >= batchSize) { - context.logger.debug(`${transactionsCountSinceLastLog} transactions added to blockchain in ${timeUsedBlockchain.string()}`) - timeUsedBlockchain.reset() - transactionsCountSinceLastLog = 0 + if (isDebug) { + process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`) + transactionsCountSinceLastLog++ + if (transactionsCountSinceLastLog >= batchSize) { + context.logger.debug(`${transactionsCountSinceLastLog} transactions added to blockchain in ${timeUsedBlockchain.string()}`) + timeUsedBlockchain.reset() + transactionsCountSinceLastLog = 0 + } + } else { + transactionsCountSinceLastPrint++ + if (transactionsCountSinceLastPrint >= 100) { + process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`) + transactionsCountSinceLastPrint = 0 + } } } - process.stdout.write(`\n`) - context.logger.info(`Synced ${transactionsCount} transactions to blockchain in ${(timeUsedAll.seconds() / 60).toFixed(2)} minutes`) - context.logger.info(`Invalid contribution transactions: ${InvalidContributionTransactionSyncRole.allTransactionIds.length}`) - if (context.logger.isDebugEnabled()) { - context.logger.debug(InvalidContributionTransactionSyncRole.allTransactionIds.join(', ')) - } - /*context.logger.info(`Double linked transactions: ${TransactionsSyncRole.doubleTransactionLinkCodes.length}`) - if (context.logger.isDebugEnabled()) { - context.logger.debug(TransactionsSyncRole.doubleTransactionLinkCodes.join(', ')) - }*/ + process.stdout.write(`successfully added to blockchain: ${transactionsCount}\n`) + context.logger.info(`Synced ${transactionsCount} transactions to blockchain in ${timeUsedAll.string()}`) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts index dc44af15f..1e7d0f0b2 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts @@ -9,6 +9,19 @@ export function bytesToKbyte(bytes: number): string { return (bytes / 1024).toFixed(0) } +export function bytesString(bytes: number): string { + if (bytes > (1024 * 1024)) { + return `${bytesToMbyte(bytes)} MB` + } else if (bytes > 1024) { + return `${bytesToKbyte(bytes)} KB` + } + return `${bytes.toFixed(0)} Bytes` +} + +export function toMysqlDateTime(date: Date): string { + return date.toISOString().slice(0, 23).replace('T', ' ') +} + export function calculateOneHashStep(hash: Buffer, data: Buffer): Buffer { const outputHash = Buffer.alloc(crypto_generichash_KEYBYTES, 0) crypto_generichash_batch(outputHash, [hash, data]) @@ -17,15 +30,22 @@ export function calculateOneHashStep(hash: Buffer, data: Buffer): Buffer { if (value.user && value.confirmedByUser && value.user.gradidoId === value.confirmedByUser.gradidoId) { throw new Error(`expect user to be different from confirmedByUser: ${JSON.stringify(value, null, 2)}`) @@ -77,6 +78,7 @@ export const transactionLinkDbSchema = v.object({ code: identifierSeedSchema, createdAt: dateSchema, validUntil: dateSchema, + holdAvailableAmount: gradidoAmountSchema, redeemedAt: v.nullish(dateSchema), deletedAt: v.nullish(dateSchema), }) From f27b438bfb0bec5d23fb0d427d698f6139d95c24 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 30 Dec 2025 14:52:08 +0100 Subject: [PATCH 14/50] use precise ledger anchor type --- dlt-connector/bun.lock | 1 + .../blockchain.ts | 32 ++----------------- .../db-v2.7.0_to_blockchain-v3.5/bootstrap.ts | 9 ++++-- .../CreationsSync.role.ts | 3 +- .../DeletedTransactionLinksSync.role.ts | 3 +- .../LocalTransactionsSync.role.ts | 3 +- .../RedeemTransactionLinksSync.role.ts | 3 +- .../TransactionLinkFundingsSync.role.ts | 3 +- .../syncDbWithBlockchain/UsersSync.role.ts | 16 ++++++---- 9 files changed, 31 insertions(+), 42 deletions(-) diff --git a/dlt-connector/bun.lock b/dlt-connector/bun.lock index 5337098a9..faaacaeb4 100644 --- a/dlt-connector/bun.lock +++ b/dlt-connector/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "dlt-connector", diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts index d689f22e1..5091a122b 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts @@ -4,29 +4,23 @@ import { GradidoTransactionBuilder, HieroAccountId, InMemoryBlockchain, + LedgerAnchor, } from 'gradido-blockchain-js' -import { getLogger } from 'log4js' -import { LOG4JS_BASE_CATEGORY } from '../../config/const' -import { CommunityRootTransactionRole } from '../../interactions/sendToHiero/CommunityRootTransaction.role' -import { Community } from '../../schemas/transaction.schema' import { NotEnoughGradidoBalanceError } from './errors' -const logger = getLogger( - `${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`, -) export const defaultHieroAccount = new HieroAccountId(0, 0, 2) export function addToBlockchain( builder: GradidoTransactionBuilder, blockchain: InMemoryBlockchain, - transactionId: number, + ledgerAnchor: LedgerAnchor, accountBalances: AccountBalances, ): boolean { const transaction = builder.build() try { const result = blockchain.createAndAddConfirmedTransactionExtern( transaction, - transactionId, + ledgerAnchor, accountBalances, ) return result @@ -44,23 +38,3 @@ export function addToBlockchain( } } -export async function addCommunityRootTransaction( - blockchain: InMemoryBlockchain, - community: Community, - accountBalances: AccountBalances -): Promise { - const communityRootTransactionRole = new CommunityRootTransactionRole(community) - if ( - addToBlockchain( - await communityRootTransactionRole.getGradidoTransactionBuilder(), - blockchain, - 0, - accountBalances, - ) - ) { - logger.info(`Community Root Transaction added`) - } else { - throw new Error(`Community Root Transaction not added`) - } -} - diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts index cd34bd8eb..9bc6f6a57 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts @@ -1,5 +1,5 @@ import { randomBytes } from 'node:crypto' -import { AccountBalances, GradidoTransactionBuilder, InMemoryBlockchainProvider } from 'gradido-blockchain-js' +import { AccountBalances, GradidoTransactionBuilder, InMemoryBlockchainProvider, LedgerAnchor } from 'gradido-blockchain-js' import * as v from 'valibot' import { CONFIG } from '../../config' import { deriveFromSeed } from '../../data/deriveKeyPair' @@ -80,7 +80,12 @@ async function bootstrapCommunities(context: Context): Promise { addToBlockchain( this.buildTransaction(item, communityContext, recipientKeyPair, signerKeyPair), blockchain, - item.id, + new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_CONTRIBUTION_ID), this.calculateAccountBalances(item, communityContext, recipientPublicKey), ) } catch(e) { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts index 0111234b7..e772509f9 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts @@ -12,6 +12,7 @@ import { GradidoTransfer, GradidoUnit, KeyPairEd25519, + LedgerAnchor, MemoryBlockPtr, TransferAmount } from 'gradido-blockchain-js' @@ -149,7 +150,7 @@ export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { addToBlockchain( this.buildTransaction(item, senderKeyPair, recipientKeyPair), blockchain, - item.id, + new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_ID), this.calculateBalances(item, communityContext, senderPublicKey, recipientPublicKey), ) } catch(e) { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts index 810fed60d..139be9259 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts @@ -10,6 +10,7 @@ import { GradidoTransfer, GradidoUnit, KeyPairEd25519, + LedgerAnchor, MemoryBlockPtr, TransferAmount } from 'gradido-blockchain-js' @@ -163,7 +164,7 @@ export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { const result = await this.context.db .select() .from(usersTable) - .where(or( - gt(usersTable.createdAt, toMysqlDateTime(lastIndex.date)), - and( - eq(usersTable.createdAt, toMysqlDateTime(lastIndex.date)), - gt(usersTable.id, lastIndex.id) + .where(and( + eq(usersTable.foreign, 0), + or( + gt(usersTable.createdAt, toMysqlDateTime(lastIndex.date)), + and( + eq(usersTable.createdAt, toMysqlDateTime(lastIndex.date)), + gt(usersTable.id, lastIndex.id) + ) ) )) .orderBy(asc(usersTable.createdAt), asc(usersTable.id)) @@ -94,7 +98,7 @@ export class UsersSyncRole extends AbstractSyncRole { addToBlockchain( this.buildTransaction(item, communityContext.keyPair, accountKeyPair, userKeyPair), communityContext.blockchain, - item.id, + new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_USER_ID), this.calculateAccountBalances(accountPublicKey), ) } catch (e) { From 170d4bcb785497a7fc64b78b70921f1421a9b16c Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 30 Dec 2025 14:53:22 +0100 Subject: [PATCH 15/50] lint database migration --- .../migrations/0097-fix_production_data_for_blockchain2.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts b/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts index 14ed3f826..f8f2fdd57 100644 --- a/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts +++ b/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts @@ -231,9 +231,9 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis * In the current DB version, there is no information about who confirmed contributions based on contribution_links. * Recently, functionality was added to create an 'ADMIN_CONTRIBUTION_LINK_CREATE' event * for newly created contribution_links, but existing contribution_links were not updated. - * + * * This query inserts an 'ADMIN_CONTRIBUTION_LINK_CREATE' event for every contribution_link - * that does not already have such an event. + * that does not already have such an event. * The acting_user_id is set to the first Admin, and affected_user_id is set to 0. */ await queryFn(` From d9666f3a8834de0dc5176920ef8a36b97779e359 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 30 Dec 2025 18:13:03 +0100 Subject: [PATCH 16/50] move transaction link validUntil correction into database migration --- ...097-fix_production_data_for_blockchain2.ts | 78 +++++++++++++++++++ .../TransactionLinkFundingsSync.role.ts | 31 +------- .../db-v2.7.0_to_blockchain-v3.5/utils.ts | 4 - shared/src/const/index.ts | 3 + shared/src/logic/decay.ts | 13 +++- 5 files changed, 93 insertions(+), 36 deletions(-) diff --git a/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts b/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts index f8f2fdd57..f7c99561d 100644 --- a/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts +++ b/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts @@ -1,3 +1,10 @@ +import Decimal from 'decimal.js-light' +import { DECAY_FACTOR, reverseLegacyDecay } from 'shared' + +function calculateEffectiveSeconds(holdOriginal: Decimal, holdCorrected: Decimal): Decimal { + return holdOriginal.div(holdCorrected).ln().div(DECAY_FACTOR.ln()) +} + export async function upgrade(queryFn: (query: string, values?: any[]) => Promise>) { /** * Migration: Correct historical inconsistencies in transactions, users, and contribution_links. @@ -34,6 +41,77 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis * ensuring blockchain consistency for contributions. */ + /** + * Fix 0: Update transaction links to match holdAvailableAmount with validUntil, because the old formula lead to incorrect values + */ + + let sumCount = 0 + let count = 0 + let lastProcessedId = 0 + const LIMIT = 200 + do { + const rows = await queryFn( + ` + SELECT id, amount, hold_available_amount, validUntil, createdAt, redeemedAt, deletedAt + FROM transaction_links + WHERE id > ? + ORDER BY id ASC + LIMIT ? + `, + [lastProcessedId, LIMIT], + ) + if (!rows.length) { + break + } + const updates: Array<{ id: number; newValidUntil: string }> = [] + for (const row of rows) { + const validUntil = new Date(row.validUntil) + const redeemedAt = row.redeemedAt ? new Date(row.redeemedAt) : null + const deletedAt = row.deletedAt ? new Date(row.deletedAt) : null + const createdAt = new Date(row.createdAt) + const amount = new Decimal(row.amount) + const duration = (validUntil.getTime() - createdAt.getTime()) / 1000 + const blockedAmountCorrected = reverseLegacyDecay(amount, duration) + // fix only if the difference is big enough to have an impact + if (blockedAmountCorrected.sub(amount).abs().lt(new Decimal('0.001'))) { + continue + } + const holdAvailableAmount = new Decimal(row.hold_available_amount) + const secondsDiff = calculateEffectiveSeconds( + new Decimal(holdAvailableAmount.toString()), + new Decimal(blockedAmountCorrected.toString()), + ) + const newValidUntil = new Date(validUntil.getTime() - secondsDiff.mul(1000).toNumber()) + if ( + (redeemedAt && redeemedAt.getTime() < validUntil.getTime()) || + (deletedAt && deletedAt.getTime() < validUntil.getTime()) + ) { + continue + } + updates.push({ + id: row.id, + newValidUntil: newValidUntil.toISOString().replace('T', ' ').replace('Z', ''), + }) + } + if (updates.length > 0) { + const caseStatements = updates.map((u) => `WHEN ${u.id} THEN '${u.newValidUntil}'`).join('\n') + + await queryFn( + ` + UPDATE transaction_links + SET validUntil = CASE id + ${caseStatements} + END + WHERE id IN (?) + `, + [updates.map((u) => u.id)], + ) + sumCount += updates.length + } + count = rows.length + lastProcessedId = rows[rows.length - 1].id + } while (count === LIMIT) + ///*/ /** * Fix 1: Remove self-signed contributions. * diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts index 6a28d4b7f..dc4032f68 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts @@ -127,32 +127,8 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole item.validUntil.getTime()) { - endDateTime = item.validUntil.getTime() - } - let duration = new DurationSeconds((endDateTime - item.createdAt.getTime()) / 1000) - const hourInSeconds = 60 * 60 - if (duration.getSeconds() < hourInSeconds) { - duration = new DurationSeconds(hourInSeconds) - } + let duration = new DurationSeconds((item.validUntil.getTime() - item.createdAt.getTime()) / 1000) let blockedAmount = GradidoUnit.fromString(reverseLegacyDecay(new Decimal(item.amount.toString()), duration.getSeconds()).toString()) - blockedAmount = blockedAmount.add(GradidoUnit.fromGradidoCent(1)) - // let blockedAmount = decayedAmount.calculateCompoundInterest(duration.getSeconds()) let accountBalances: AccountBalances try { accountBalances = this.calculateBalances(item, blockedAmount, communityContext, senderPublicKey, recipientPublicKey) @@ -171,11 +147,6 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole Date: Wed, 7 Jan 2026 17:21:44 +0100 Subject: [PATCH 17/50] update version --- dlt-connector/package.json | 2 +- .../Context.ts | 0 .../binaryExport.ts | 0 .../blockchain.ts | 0 .../bootstrap.ts | 0 .../data/Balance.ts | 0 .../data/ContributionStatus.ts | 0 .../data/TransactionTypeId.ts | 0 .../data/keyPair.ts | 0 .../database.ts | 0 .../drizzle.schema.ts | 0 .../errors.ts | 0 .../index.ts | 0 .../interaction/syncDbWithBlockchain/AbstractSync.role.ts | 0 .../ContributionLinkTransactionSync.role.ts | 0 .../interaction/syncDbWithBlockchain/CreationsSync.role.ts | 0 .../syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts | 0 .../syncDbWithBlockchain/LocalTransactionsSync.role.ts | 0 .../syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts | 0 .../syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts | 0 .../interaction/syncDbWithBlockchain/UsersSync.role.ts | 0 .../syncDbWithBlockchain/syncDbWithBlockchain.context.ts | 0 .../utils.ts | 0 .../valibot.schema.ts | 0 24 files changed, 1 insertion(+), 1 deletion(-) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/Context.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/binaryExport.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/blockchain.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/bootstrap.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/data/Balance.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/data/ContributionStatus.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/data/TransactionTypeId.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/data/keyPair.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/database.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/drizzle.schema.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/errors.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/index.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/interaction/syncDbWithBlockchain/AbstractSync.role.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/interaction/syncDbWithBlockchain/CreationsSync.role.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/interaction/syncDbWithBlockchain/UsersSync.role.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/utils.ts (100%) rename dlt-connector/src/migrations/{db-v2.7.0_to_blockchain-v3.5 => db-v2.7.0_to_blockchain-v3.7}/valibot.schema.ts (100%) diff --git a/dlt-connector/package.json b/dlt-connector/package.json index e173d6ede..84ed03df7 100644 --- a/dlt-connector/package.json +++ b/dlt-connector/package.json @@ -10,7 +10,7 @@ "start": "cross-env TZ=UTC bun run src/index.ts", "build": "bun build src/index.ts --outdir=build --target=bun --external=gradido-blockchain-js --minify", "dev": "cross-env TZ=UTC bun run --watch src/index.ts", - "migrate": "cross-env TZ=UTC bun src/migrations/db-v2.7.0_to_blockchain-v3.5", + "migrate": "cross-env TZ=UTC bun src/migrations/db-v2.7.0_to_blockchain-v3.7", "test": "cross-env TZ=UTC bun test", "test:debug": "cross-env TZ=UTC bun test --inspect-brk", "typecheck": "tsc --noEmit", diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/binaryExport.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/binaryExport.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/Balance.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/Balance.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/Balance.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/Balance.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/ContributionStatus.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/ContributionStatus.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/ContributionStatus.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/ContributionStatus.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/TransactionTypeId.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/TransactionTypeId.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/TransactionTypeId.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/TransactionTypeId.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/keyPair.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/keyPair.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/data/keyPair.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/keyPair.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/database.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/database.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/drizzle.schema.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/drizzle.schema.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/errors.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/errors.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/CreationsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/CreationsSync.role.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/UsersSync.role.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/UsersSync.role.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/utils.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/utils.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/valibot.schema.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/valibot.schema.ts From 473d8139e8c5a3c9c3142c82962c042730e0b276 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Fri, 23 Jan 2026 09:55:52 +0100 Subject: [PATCH 18/50] add remote transactions, fix some stuff --- ...097-fix_production_data_for_blockchain2.ts | 12 ++ .../sendToHiero/CreationTransaction.role.ts | 2 +- .../blockchain.ts | 8 +- .../db-v2.7.0_to_blockchain-v3.7/bootstrap.ts | 22 +- .../data/Balance.ts | 51 +++-- .../drizzle.schema.ts | 2 + .../syncDbWithBlockchain/AbstractSync.role.ts | 5 +- .../CreationsSync.role.ts | 7 +- .../DeletedTransactionLinksSync.role.ts | 17 +- .../LocalTransactionsSync.role.ts | 12 +- .../RedeemTransactionLinksSync.role.ts | 26 ++- .../RemoteTransactionsSync.role.ts | 193 ++++++++++++++++++ .../TransactionLinkFundingsSync.role.ts | 16 +- .../syncDbWithBlockchain/UsersSync.role.ts | 13 +- .../syncDbWithBlockchain.context.ts | 2 + .../valibot.schema.ts | 29 ++- 16 files changed, 345 insertions(+), 72 deletions(-) create mode 100644 dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts diff --git a/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts b/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts index f7c99561d..9f67e267a 100644 --- a/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts +++ b/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts @@ -251,6 +251,18 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis WHERE u.created_at >= t.first_date; ;`) + // linked user also, but we need to use gradido_id as index, because on cross group transactions linked_user_id is empty + await queryFn(` + UPDATE users u + LEFT JOIN ( + SELECT linked_user_gradido_id , MIN(balance_date) AS first_date + FROM transactions + GROUP BY linked_user_gradido_id + ) t ON t.linked_user_gradido_id = u.gradido_id + SET u.created_at = DATE_SUB(t.first_date, INTERVAL 1 SECOND) + WHERE u.created_at >= t.first_date; + ;`) + /** * Fix 4: Ensure all transaction memos meet the minimum length requirement. * diff --git a/dlt-connector/src/interactions/sendToHiero/CreationTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/CreationTransaction.role.ts index 4b0f7aefd..4f4d653a6 100644 --- a/dlt-connector/src/interactions/sendToHiero/CreationTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/CreationTransaction.role.ts @@ -36,7 +36,7 @@ export class CreationTransactionRole extends AbstractTransactionRole { } getRecipientCommunityTopicId(): HieroId { - throw new Error('creation: cannot be used as cross group transaction') + return this.creationTransaction.user.communityTopicId } public async getGradidoTransactionBuilder(): Promise { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts index 5091a122b..fd70dcca6 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts @@ -1,7 +1,7 @@ import { AccountBalances, Filter, - GradidoTransactionBuilder, + GradidoTransaction, HieroAccountId, InMemoryBlockchain, LedgerAnchor, @@ -11,12 +11,12 @@ import { NotEnoughGradidoBalanceError } from './errors' export const defaultHieroAccount = new HieroAccountId(0, 0, 2) export function addToBlockchain( - builder: GradidoTransactionBuilder, + transaction: GradidoTransaction, blockchain: InMemoryBlockchain, ledgerAnchor: LedgerAnchor, accountBalances: AccountBalances, ): boolean { - const transaction = builder.build() + try { const result = blockchain.createAndAddConfirmedTransactionExtern( transaction, @@ -33,7 +33,7 @@ export function addToBlockchain( throw new NotEnoughGradidoBalanceError(needed, exist) } } - const lastTransaction = blockchain.findOne(Filter.LAST_TRANSACTION) + const lastTransaction = blockchain.findOne(Filter.LAST_TRANSACTION) throw new Error(`Transaction ${transaction.toJson(true)} not added: ${error}, last transaction was: ${lastTransaction?.getConfirmedTransaction()?.toJson(true)}`) } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts index 9bc6f6a57..c1f5bc1ae 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts @@ -1,5 +1,5 @@ import { randomBytes } from 'node:crypto' -import { AccountBalances, GradidoTransactionBuilder, InMemoryBlockchainProvider, LedgerAnchor } from 'gradido-blockchain-js' +import { Abstract, AccountBalances, GradidoTransactionBuilder, InMemoryBlockchain, InMemoryBlockchainProvider, LedgerAnchor } from 'gradido-blockchain-js' import * as v from 'valibot' import { CONFIG } from '../../config' import { deriveFromSeed } from '../../data/deriveKeyPair' @@ -27,15 +27,13 @@ async function bootstrapCommunities(context: Context): Promise() for (const communityDb of communitiesDb) { - let alias = communityDb.name - if (communityNames.has(communityDb.name)) { + let alias = communityDb.name.toLowerCase() + if (communityNames.has(alias)) { alias = communityDb.communityUuid } else { - communityNames.add(communityDb.name) + communityNames.add(alias) } - const blockchain = InMemoryBlockchainProvider.getInstance().findBlockchain( - alias, - ) + const blockchain = InMemoryBlockchainProvider.getInstance().findBlockchain(alias) if (!blockchain) { throw new Error(`Couldn't create Blockchain for community ${alias}`) } @@ -66,22 +64,24 @@ async function bootstrapCommunities(context: Context): Promise [index('user_id').on(table.userId)], diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts index adb84e0c9..443c68b4a 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts @@ -29,7 +29,8 @@ export abstract class AbstractSyncRole { }) } - getLastBalanceForUser(publicKey: MemoryBlockPtr, blockchain: InMemoryBlockchain, communityId: string = ''): Balance { + getLastBalanceForUser(publicKey: MemoryBlockPtr, blockchain: InMemoryBlockchain, communityId: string + ): Balance { if (publicKey.isEmpty()) { throw new Error('publicKey is empty') } @@ -45,7 +46,7 @@ export abstract class AbstractSyncRole { if (!senderLastAccountBalance) { return new Balance(publicKey, communityId) } - return Balance.fromAccountBalance(senderLastAccountBalance, lastConfirmedTransaction.getConfirmedAt().getDate()) + return Balance.fromAccountBalance(senderLastAccountBalance, lastConfirmedTransaction.getConfirmedAt().getDate(), communityId) } logLastBalanceChangingTransactions(publicKey: MemoryBlockPtr, blockchain: InMemoryBlockchain, transactionCount: number = 5) { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts index a509a15ba..9546c07b9 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts @@ -95,9 +95,10 @@ export class CreationsSyncRole extends AbstractSyncRole { ), ) .setTransactionCreation( - new TransferAmount(recipientKeyPair.getPublicKey(), item.amount), + new TransferAmount(recipientKeyPair.getPublicKey(), item.amount, communityContext.communityId), item.contributionDate, ) + .setRecipientCommunity(communityContext.communityId) .sign(signerKeyPair) } @@ -107,7 +108,7 @@ export class CreationsSyncRole extends AbstractSyncRole { recipientPublicKey: MemoryBlockPtr ): AccountBalances { const accountBalances = new AccountBalances() - const balance = this.getLastBalanceForUser(recipientPublicKey, communityContext.blockchain) + const balance = this.getLastBalanceForUser(recipientPublicKey, communityContext.blockchain, communityContext.communityId) // calculate decay since last balance with legacy calculation method balance.updateLegacyDecay(item.amount, item.confirmedAt) @@ -136,7 +137,7 @@ export class CreationsSyncRole extends AbstractSyncRole { try { addToBlockchain( - this.buildTransaction(item, communityContext, recipientKeyPair, signerKeyPair), + this.buildTransaction(item, communityContext, recipientKeyPair, signerKeyPair).build(), blockchain, new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_CONTRIBUTION_ID), this.calculateAccountBalances(item, communityContext, recipientPublicKey), diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts index e772509f9..a1e6d046e 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts @@ -74,6 +74,7 @@ export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { } buildTransaction( + communityContext: CommunityContext, item: TransactionDb, senderKeyPair: KeyPairEd25519, recipientKeyPair: KeyPairEd25519, @@ -94,9 +93,10 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { ), ) .setTransactionTransfer( - new TransferAmount(senderKeyPair.getPublicKey(), item.amount), + new TransferAmount(senderKeyPair.getPublicKey(), item.amount, communityContext.communityId), recipientKeyPair.getPublicKey(), ) + .setSenderCommunity(communityContext.communityId) .sign(senderKeyPair) } @@ -108,8 +108,8 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { ): AccountBalances { const accountBalances = new AccountBalances() - const senderLastBalance = this.getLastBalanceForUser(senderPublicKey, communityContext.blockchain) - const recipientLastBalance = this.getLastBalanceForUser(recipientPublicKey, communityContext.blockchain) + const senderLastBalance = this.getLastBalanceForUser(senderPublicKey, communityContext.blockchain, communityContext.communityId) + const recipientLastBalance = this.getLastBalanceForUser(recipientPublicKey, communityContext.blockchain, communityContext.communityId) try { senderLastBalance.updateLegacyDecay(item.amount.negated(), item.balanceDate) @@ -145,7 +145,7 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { try { addToBlockchain( - this.buildTransaction(item, senderKeyPair, recipientKeyPair), + this.buildTransaction(communityContext, item, senderKeyPair, recipientKeyPair).build(), blockchain, new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_ID), this.calculateBalances(item, communityContext, senderPublicKey, recipientPublicKey), diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts index 139be9259..0f472a109 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts @@ -82,6 +82,7 @@ export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { + getDate(): Date { + return this.peek().balanceDate + } + + getLastIndex(): IndexType { + const lastItem = this.peekLast() + return { date: lastItem.balanceDate, id: lastItem.id } + } + + itemTypeName(): string { + return 'remoteTransactions' + } + + async loadFromDb(lastIndex: IndexType, count: number): Promise { + const linkedUsers = alias(usersTable, 'linkedUser') + const result = await this.context.db + .select({ + transaction: transactionsTable, + user: usersTable, + linkedUser: linkedUsers, + }) + .from(transactionsTable) + .where( + and( + inArray(transactionsTable.typeId, [TransactionTypeId.RECEIVE, TransactionTypeId.SEND]), + isNull(transactionsTable.transactionLinkId), + ne(usersTable.communityUuid, linkedUsers.communityUuid), + or( + gt(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)), + and( + eq(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)), + gt(transactionsTable.id, lastIndex.id) + ) + ) + ) + ) + .innerJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) + .innerJoin(linkedUsers, eq(transactionsTable.linkedUserGradidoId, linkedUsers.gradidoId)) + .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) + .limit(count) + + return result.map((row) => { + const item = { + ...row.transaction, + user: row.user, + linkedUser: row.linkedUser, + } + if (item.typeId === TransactionTypeId.SEND && item.amount) { + item.amount = new Decimal(item.amount).neg().toString() + } + try { + return v.parse(transactionDbSchema, item) + } catch (e) { + throw new DatabaseError('loadRemoteTransferTransactions', item, e as Error) + } + }) + } + + buildTransaction( + item: TransactionDb, + senderKeyPair: KeyPairEd25519, + recipientKeyPair: KeyPairEd25519, + senderCommunityId: string, + recipientCommunityId: string, + ): GradidoTransactionBuilder { + const builder = new GradidoTransactionBuilder() + .setCreatedAt(item.balanceDate) + .addMemo(new EncryptedMemo( + item.memo, + new AuthenticatedEncryption(senderKeyPair), + new AuthenticatedEncryption(recipientKeyPair), + ), + ) + .setTransactionTransfer( + new TransferAmount(senderKeyPair.getPublicKey(), item.amount, senderCommunityId), + recipientKeyPair.getPublicKey(), + ) + .setSenderCommunity(senderCommunityId) + .setRecipientCommunity(recipientCommunityId) + .sign(senderKeyPair) + return builder + } + + calculateBalances( + item: TransactionDb, + communityContext: CommunityContext, + amount: GradidoUnit, + publicKey: MemoryBlockPtr, + ): AccountBalances { + const accountBalances = new AccountBalances() + if (communityContext.foreign) { + accountBalances.add(new AccountBalance(publicKey, GradidoUnit.zero(), communityContext.communityId)) + return accountBalances + } else { + const lastBalance = this.getLastBalanceForUser(publicKey, communityContext.blockchain, communityContext.communityId) + + try { + lastBalance.updateLegacyDecay(amount, item.balanceDate) + } catch(e) { + if (e instanceof NegativeBalanceError) { + this.logLastBalanceChangingTransactions(publicKey, communityContext.blockchain, 10) + throw e + } + } + accountBalances.add(lastBalance.getAccountBalance()) + return accountBalances + } + } + + getUser(item: TransactionDb): { senderUser: UserDb, recipientUser: UserDb } { + return ( + item.typeId === TransactionTypeId.RECEIVE + ? { senderUser: item.linkedUser, recipientUser: item.user } + : { senderUser: item.user, recipientUser: item.linkedUser } + ) + } + + pushToBlockchain(item: TransactionDb): void { + const { senderUser, recipientUser } = this.getUser(item) + const ledgerAnchor = new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_ID) + + if (senderUser.communityUuid === recipientUser.communityUuid) { + throw new Error(`transfer between user from same community: ${JSON.stringify(item, null, 2)}, check db query`) + } + const senderCommunityContext = this.context.getCommunityContextByUuid(senderUser.communityUuid) + const recipientCommunityContext = this.context.getCommunityContextByUuid(recipientUser.communityUuid) + const senderBlockchain = senderCommunityContext.blockchain + const recipientBlockchain = recipientCommunityContext.blockchain + + // I use the received transaction so user and linked user are swapped and user is recipient and linkedUser ist sender + const senderKeyPair = this.getAccountKeyPair(senderCommunityContext, senderUser.gradidoId) + const senderPublicKey = senderKeyPair.getPublicKey() + const recipientKeyPair = this.getAccountKeyPair(recipientCommunityContext, recipientUser.gradidoId) + const recipientPublicKey = recipientKeyPair.getPublicKey() + + if (!senderKeyPair || !senderPublicKey || !recipientKeyPair || !recipientPublicKey) { + throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`) + } + const transactionBuilder = this.buildTransaction( + item, + senderKeyPair, + recipientKeyPair, + senderCommunityContext.communityId, + recipientCommunityContext.communityId + ) + const outboundTransaction = transactionBuilder.buildOutbound() + + try { + addToBlockchain( + outboundTransaction, + senderBlockchain, + ledgerAnchor, + this.calculateBalances(item, senderCommunityContext, item.amount.negated(), senderPublicKey), + ) + } catch(e) { + if (e instanceof NotEnoughGradidoBalanceError) { + this.logLastBalanceChangingTransactions(senderPublicKey, senderBlockchain) + } + throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) + } + transactionBuilder.setParentLedgerAnchor(ledgerAnchor) + const inboundTransaction = transactionBuilder.buildInbound() + try { + addToBlockchain( + inboundTransaction, + recipientBlockchain, + ledgerAnchor, + this.calculateBalances(item, recipientCommunityContext, item.amount, recipientPublicKey), + ) + } catch(e) { + if (e instanceof NotEnoughGradidoBalanceError) { + this.logLastBalanceChangingTransactions(recipientPublicKey, recipientBlockchain) + } + throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) + } + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts index dc4032f68..bc568df61 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts @@ -1,4 +1,4 @@ -import { asc, eq, or, gt, and, isNull } from 'drizzle-orm' +import { asc, eq, or, gt, and } from 'drizzle-orm' import { AccountBalance, AccountBalances, @@ -20,7 +20,7 @@ import { BlockchainError, DatabaseError, NegativeBalanceError } from '../../erro import { CommunityContext, TransactionLinkDb, transactionLinkDbSchema } from '../../valibot.schema' import { AbstractSyncRole, IndexType } from './AbstractSync.role' import { deriveFromCode } from '../../../../data/deriveKeyPair' -import { calculateEffectiveSeconds, reverseLegacyDecay, toMysqlDateTime } from '../../utils' +import { reverseLegacyDecay, toMysqlDateTime } from '../../utils' import Decimal from 'decimal.js-light' export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { @@ -66,6 +66,7 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { .select() .from(usersTable) .where(and( - eq(usersTable.foreign, 0), or( gt(usersTable.createdAt, toMysqlDateTime(lastIndex.date)), and( @@ -61,6 +60,7 @@ export class UsersSyncRole extends AbstractSyncRole { } buildTransaction( + communityContext: CommunityContext, item: UserDb, communityKeyPair: KeyPairEd25519, accountKeyPair: KeyPairEd25519, @@ -74,14 +74,15 @@ export class UsersSyncRole extends AbstractSyncRole { new Uuidv4Hash(item.gradidoId).getAsMemoryBlock(), accountKeyPair.getPublicKey(), ) + .setSenderCommunity(communityContext.communityId) .sign(communityKeyPair) .sign(accountKeyPair) .sign(userKeyPair) } - calculateAccountBalances(accountPublicKey: MemoryBlockPtr): AccountBalances { + calculateAccountBalances(accountPublicKey: MemoryBlockPtr, communityContext: CommunityContext,): AccountBalances { const accountBalances = new AccountBalances() - accountBalances.add(new AccountBalance(accountPublicKey, GradidoUnit.zero(), '')) + accountBalances.add(new AccountBalance(accountPublicKey, GradidoUnit.zero(), communityContext.communityId)) return accountBalances } @@ -96,10 +97,10 @@ export class UsersSyncRole extends AbstractSyncRole { try { addToBlockchain( - this.buildTransaction(item, communityContext.keyPair, accountKeyPair, userKeyPair), + this.buildTransaction(communityContext, item, communityContext.keyPair, accountKeyPair, userKeyPair).build(), communityContext.blockchain, new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_USER_ID), - this.calculateAccountBalances(accountPublicKey), + this.calculateAccountBalances(accountPublicKey, communityContext), ) } catch (e) { throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts index 37ab78373..8bc3b5a0b 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts @@ -7,6 +7,7 @@ import { TransactionLinkFundingsSyncRole } from './TransactionLinkFundingsSync.r import { RedeemTransactionLinksSyncRole } from './RedeemTransactionLinksSync.role' import { ContributionLinkTransactionSyncRole } from './ContributionLinkTransactionSync.role' import { DeletedTransactionLinksSyncRole } from './DeletedTransactionLinksSync.role' +import { RemoteTransactionsSyncRole } from './RemoteTransactionsSync.role' export async function syncDbWithBlockchainContext(context: Context, batchSize: number) { const timeUsedDB = new Profiler() @@ -20,6 +21,7 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n new RedeemTransactionLinksSyncRole(context), new ContributionLinkTransactionSyncRole(context), new DeletedTransactionLinksSyncRole(context), + new RemoteTransactionsSyncRole(context), ] let transactionsCount = 0 let transactionsCountSinceLastLog = 0 diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/valibot.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/valibot.schema.ts index 207b401ed..904be605d 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/valibot.schema.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/valibot.schema.ts @@ -1,4 +1,4 @@ -import { InMemoryBlockchain, KeyPairEd25519 } from 'gradido-blockchain-js' +import { GradidoUnit, InMemoryBlockchain, KeyPairEd25519 } from 'gradido-blockchain-js' import * as v from 'valibot' import { booleanSchema, dateSchema } from '../../schemas/typeConverter.schema' import { @@ -9,6 +9,7 @@ import { } from '../../schemas/typeGuard.schema' import { Balance } from './data/Balance' import { TransactionTypeId } from './data/TransactionTypeId' +import Decimal from 'decimal.js-light' const positiveNumberSchema = v.pipe(v.number(), v.minValue(1)) @@ -18,7 +19,31 @@ export const userDbSchema = v.object({ communityUuid: uuidv4Schema, createdAt: dateSchema, }) +/* +declare const validLegacyAmount: unique symbol +export type LegacyAmount = string & { [validLegacyAmount]: true } +export const legacyAmountSchema = v.pipe( + v.string(), + v.regex(/^-?[0-9]+(\.[0-9]+)?$/), + v.transform((input: string) => input as LegacyAmount), +) + +declare const validGradidoAmount: unique symbol +export type GradidoAmount = GradidoUnit & { [validGradidoAmount]: true } + +export const gradidoAmountSchema = v.pipe( + v.union([legacyAmountSchema, v.instance(GradidoUnit, 'expect GradidoUnit type')]), + v.transform((input: LegacyAmount | GradidoUnit) => { + if (input instanceof GradidoUnit) { + return input as GradidoAmount + } + // round floor with decimal js beforehand + const rounded = new Decimal(input).toDecimalPlaces(4, Decimal.ROUND_FLOOR).toString() + return GradidoUnit.fromString(rounded) as GradidoAmount + }), +) +*/ export const transactionBaseSchema = v.object({ id: positiveNumberSchema, amount: gradidoAmountSchema, @@ -26,6 +51,7 @@ export const transactionBaseSchema = v.object({ user: userDbSchema, }) + export const transactionDbSchema = v.pipe(v.object({ ...transactionBaseSchema.entries, typeId: v.enum(TransactionTypeId), @@ -107,6 +133,7 @@ export const communityDbSchema = v.object({ export const communityContextSchema = v.object({ communityId: v.string(), + foreign: booleanSchema, blockchain: v.instance(InMemoryBlockchain, 'expect InMemoryBlockchain type'), keyPair: v.instance(KeyPairEd25519), folder: v.pipe( From 7fe7f4cb11a87187d607f67ee295a4ebcdadab8f Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 25 Jan 2026 12:43:36 +0100 Subject: [PATCH 19/50] add more profiling --- .../db-v2.7.0_to_blockchain-v3.7/blockchain.ts | 5 +++++ .../syncDbWithBlockchain/AbstractSync.role.ts | 9 +++++++-- .../syncDbWithBlockchain/syncDbWithBlockchain.context.ts | 7 +++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts index fd70dcca6..ffff61f1a 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts @@ -5,10 +5,13 @@ import { HieroAccountId, InMemoryBlockchain, LedgerAnchor, + Profiler, } from 'gradido-blockchain-js' import { NotEnoughGradidoBalanceError } from './errors' export const defaultHieroAccount = new HieroAccountId(0, 0, 2) +export let callTime: number = 0 +const timeUsed = new Profiler export function addToBlockchain( transaction: GradidoTransaction, @@ -18,11 +21,13 @@ export function addToBlockchain( ): boolean { try { + timeUsed.reset() const result = blockchain.createAndAddConfirmedTransactionExtern( transaction, ledgerAnchor, accountBalances, ) + callTime += timeUsed.nanos() return result } catch (error) { if (error instanceof Error) { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts index 443c68b4a..bcdc133b5 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts @@ -11,6 +11,8 @@ export type IndexType = { date: Date id: number } +export let nanosBalanceForUser = 0 +const lastBalanceOfUserTimeUsed = new Profiler export abstract class AbstractSyncRole { private items: ItemType[] = [] @@ -31,9 +33,10 @@ export abstract class AbstractSyncRole { getLastBalanceForUser(publicKey: MemoryBlockPtr, blockchain: InMemoryBlockchain, communityId: string ): Balance { + lastBalanceOfUserTimeUsed.reset() if (publicKey.isEmpty()) { throw new Error('publicKey is empty') - } + } const lastSenderTransaction = blockchain.findOne(Filter.lastBalanceFor(publicKey)) if (!lastSenderTransaction) { return new Balance(publicKey, communityId) @@ -46,7 +49,9 @@ export abstract class AbstractSyncRole { if (!senderLastAccountBalance) { return new Balance(publicKey, communityId) } - return Balance.fromAccountBalance(senderLastAccountBalance, lastConfirmedTransaction.getConfirmedAt().getDate(), communityId) + const result = Balance.fromAccountBalance(senderLastAccountBalance, lastConfirmedTransaction.getConfirmedAt().getDate(), communityId) + nanosBalanceForUser += lastBalanceOfUserTimeUsed.nanos() + return result } logLastBalanceChangingTransactions(publicKey: MemoryBlockPtr, blockchain: InMemoryBlockchain, transactionCount: number = 5) { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts index 8bc3b5a0b..6446e2600 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts @@ -8,6 +8,8 @@ import { RedeemTransactionLinksSyncRole } from './RedeemTransactionLinksSync.rol import { ContributionLinkTransactionSyncRole } from './ContributionLinkTransactionSync.role' import { DeletedTransactionLinksSyncRole } from './DeletedTransactionLinksSync.role' import { RemoteTransactionsSyncRole } from './RemoteTransactionsSync.role' +import { callTime } from '../../blockchain' +import { nanosBalanceForUser } from './AbstractSync.role' export async function syncDbWithBlockchainContext(context: Context, batchSize: number) { const timeUsedDB = new Profiler() @@ -28,6 +30,7 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n let transactionsCountSinceLastPrint = 0 let available = containers const isDebug = context.logger.isDebugEnabled() + let lastPrintedCallTime = 0 while (true) { timeUsedDB.reset() const results = await Promise.all(available.map((c) => c.ensureFilled(batchSize))) @@ -56,6 +59,8 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n transactionsCountSinceLastLog++ if (transactionsCountSinceLastLog >= batchSize) { context.logger.debug(`${transactionsCountSinceLastLog} transactions added to blockchain in ${timeUsedBlockchain.string()}`) + context.logger.info(`Time for createAndConfirm: ${((callTime - lastPrintedCallTime) / 1000 / 1000).toFixed(2)} milliseconds`) + lastPrintedCallTime = callTime timeUsedBlockchain.reset() transactionsCountSinceLastLog = 0 } @@ -69,4 +74,6 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n } process.stdout.write(`successfully added to blockchain: ${transactionsCount}\n`) context.logger.info(`Synced ${transactionsCount} transactions to blockchain in ${timeUsedAll.string()}`) + context.logger.info(`Time for createAndConfirm: ${(callTime / 1000 / 1000 / 1000).toFixed(2)} seconds`) + context.logger.info(`Time for call lastBalance of user: ${(nanosBalanceForUser / 1000 / 1000 / 1000).toFixed(2)} seconds`) } From 015481cee19c5efc684f488074cc0de878ba976c Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sun, 25 Jan 2026 17:36:24 +0100 Subject: [PATCH 20/50] implement compact confirmed transaction struct in C --- dlt-connector/.gitignore | 2 + dlt-connector/bun-zigar.toml | 8 ++ dlt-connector/bun.lock | 11 +- dlt-connector/bunfig.toml | 1 + dlt-connector/c/grdc/grdc_account_balance.h | 23 ++++ dlt-connector/c/grdc/grdc_confirmed_tx.h | 104 +++++++++++++++ dlt-connector/c/grdc/grdc_public_key_index.h | 21 +++ dlt-connector/c/grdc/grdc_tx_id.h | 25 ++++ dlt-connector/c/grdc/grdc_tx_timestamps.h | 42 ++++++ dlt-connector/c/grdd/grdd_address_type.h | 23 ++++ .../c/grdd/grdd_balance_derivation_type.h | 26 ++++ dlt-connector/c/grdd/grdd_cross_group_type.h | 20 +++ dlt-connector/c/grdd/grdd_timestamp.h | 38 ++++++ dlt-connector/c/grdd/grdd_transaction_type.h | 56 ++++++++ dlt-connector/c/grdl/grdl_unit.c | 123 ++++++++++++++++++ dlt-connector/c/grdl/grdl_unit.h | 84 ++++++++++++ dlt-connector/package.json | 1 + dlt-connector/zig/hello.zig | 5 + 18 files changed, 612 insertions(+), 1 deletion(-) create mode 100644 dlt-connector/bun-zigar.toml create mode 100644 dlt-connector/bunfig.toml create mode 100644 dlt-connector/c/grdc/grdc_account_balance.h create mode 100644 dlt-connector/c/grdc/grdc_confirmed_tx.h create mode 100644 dlt-connector/c/grdc/grdc_public_key_index.h create mode 100644 dlt-connector/c/grdc/grdc_tx_id.h create mode 100644 dlt-connector/c/grdc/grdc_tx_timestamps.h create mode 100644 dlt-connector/c/grdd/grdd_address_type.h create mode 100644 dlt-connector/c/grdd/grdd_balance_derivation_type.h create mode 100644 dlt-connector/c/grdd/grdd_cross_group_type.h create mode 100644 dlt-connector/c/grdd/grdd_timestamp.h create mode 100644 dlt-connector/c/grdd/grdd_transaction_type.h create mode 100644 dlt-connector/c/grdl/grdl_unit.c create mode 100644 dlt-connector/c/grdl/grdl_unit.h create mode 100644 dlt-connector/zig/hello.zig diff --git a/dlt-connector/.gitignore b/dlt-connector/.gitignore index 4c6422640..c1c4be080 100644 --- a/dlt-connector/.gitignore +++ b/dlt-connector/.gitignore @@ -3,6 +3,8 @@ /.env.bak /build/ /locales/ +lib +.zigar-cache package-json.lock coverage # emacs diff --git a/dlt-connector/bun-zigar.toml b/dlt-connector/bun-zigar.toml new file mode 100644 index 000000000..5a2dc3f56 --- /dev/null +++ b/dlt-connector/bun-zigar.toml @@ -0,0 +1,8 @@ +optimize = "ReleaseSmall" + +[modules."lib/hello.zigar"] +source = "zig/hello.zig" + +[[targets]] +arch = "x64" +platform = "linux" diff --git a/dlt-connector/bun.lock b/dlt-connector/bun.lock index faaacaeb4..1bbaedc8c 100644 --- a/dlt-connector/bun.lock +++ b/dlt-connector/bun.lock @@ -5,6 +5,7 @@ "": { "name": "dlt-connector", "dependencies": { + "bun-zigar": "^0.15.2", "cross-env": "^7.0.3", "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#f265dbb1780a912cf8b0418dfe3eaf5cdc5b51cf", }, @@ -392,6 +393,8 @@ "bun-types": ["bun-types@1.3.1", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-NMrcy7smratanWJ2mMXdpatalovtxVggkj11bScuWuiOoXTiKIu2eVS1/7qbyI/4yHedtsn175n4Sm4JcdHLXw=="], + "bun-zigar": ["bun-zigar@0.15.2", "", { "dependencies": { "node-zigar-addon": "0.15.2", "zigar-compiler": "^0.15.2" }, "bin": { "zigar": "bin/cli.js", "bun-zigar": "bin/cli.js" } }, "sha512-slEHTEapQEIqB86OeiToPuuFXe39DCIYISTPzbIMBTZL34vRzCIa5wFn5ATudauHFFwl5/y5JYv8tluk2QL9Eg=="], + "call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="], "camelcase": ["camelcase@6.3.0", "", {}, "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA=="], @@ -784,7 +787,7 @@ "node-addon-api": ["node-addon-api@7.1.1", "", {}, "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ=="], - "node-api-headers": ["node-api-headers@1.6.0", "", {}, "sha512-81T99+mWLZnxX0LlZPYuafyFlxVVaWKQ0BDAbSrOqLO+v+gzCzu0GTAVNeVK8lucqjqo9L/1UcK9cpkem8Py4Q=="], + "node-api-headers": ["node-api-headers@1.8.0", "", {}, "sha512-jfnmiKWjRAGbdD1yQS28bknFM1tbHC1oucyuMPjmkEs+kpiu76aRs40WlTmBmyEgzDM76ge1DQ7XJ3R5deiVjQ=="], "node-gyp-build": ["node-gyp-build@4.8.4", "", { "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", "node-gyp-build-test": "build-test.js" } }, "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ=="], @@ -792,6 +795,8 @@ "node-releases": ["node-releases@2.0.26", "", {}, "sha512-S2M9YimhSjBSvYnlr5/+umAnPHE++ODwt5e2Ij6FoX45HA/s4vHdkDx1eax2pAPeAOqu4s9b7ppahsyEFdVqQA=="], + "node-zigar-addon": ["node-zigar-addon@0.15.2", "", { "dependencies": { "node-api-headers": "^1.7.0" } }, "sha512-QjJcPRtUZLkULaFXapAvTzLKKRddgaupr7wQqgDUQo541FMCXAhgWdZJtNcIgCNykJG0bG0Fza5VTKBdSvyavQ=="], + "normalize-path": ["normalize-path@3.0.0", "", {}, "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="], "npm-path": ["npm-path@2.0.4", "", { "dependencies": { "which": "^1.2.10" }, "bin": { "npm-path": "bin/npm-path" } }, "sha512-IFsj0R9C7ZdR5cP+ET342q77uSRdtWOlWpih5eC+lu29tIDbNEgDbzgVJ5UFvYHWhxDZ5TFkJafFioO0pPQjCw=="], @@ -1060,6 +1065,8 @@ "yoctocolors-cjs": ["yoctocolors-cjs@2.1.3", "", {}, "sha512-U/PBtDf35ff0D8X8D0jfdzHYEPFxAI7jJlxZXwCSez5M3190m+QobIfh+sWDWSHMCWWJN2AWamkegn6vr6YBTw=="], + "zigar-compiler": ["zigar-compiler@0.15.2", "", {}, "sha512-zlJ8kUwndwrLl4iRlIWEcidC2rcSsfeWM0jvbSoxUVf+SEKd4bVik3z4YDivuyX3SUiUjpCMNyp65etD6BKRmQ=="], + "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], "@babel/core/debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], @@ -1104,6 +1111,8 @@ "cmake-js/fs-extra": ["fs-extra@11.3.2", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" } }, "sha512-Xr9F6z6up6Ws+NjzMCZc6WXg2YFRlrLP9NQDO3VQrWrfiojdhS56TzueT88ze0uBdCTwEIhQ3ptnmKeWGFAe0A=="], + "cmake-js/node-api-headers": ["node-api-headers@1.6.0", "", {}, "sha512-81T99+mWLZnxX0LlZPYuafyFlxVVaWKQ0BDAbSrOqLO+v+gzCzu0GTAVNeVK8lucqjqo9L/1UcK9cpkem8Py4Q=="], + "connect/debug": ["debug@2.6.9", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="], "elliptic/bn.js": ["bn.js@4.12.2", "", {}, "sha512-n4DSx829VRTRByMRGdjQ9iqsN0Bh4OolPsFnaZBLcbi8iXcB+kJ9s7EnRt4wILZNV3kPLHkRVfOc/HvhC3ovDw=="], diff --git a/dlt-connector/bunfig.toml b/dlt-connector/bunfig.toml new file mode 100644 index 000000000..4299d9c62 --- /dev/null +++ b/dlt-connector/bunfig.toml @@ -0,0 +1 @@ +preload = ["bun-zigar"] diff --git a/dlt-connector/c/grdc/grdc_account_balance.h b/dlt-connector/c/grdc/grdc_account_balance.h new file mode 100644 index 000000000..fc38b1926 --- /dev/null +++ b/dlt-connector/c/grdc/grdc_account_balance.h @@ -0,0 +1,23 @@ +#ifndef __GRADIDO_BLOCKCHAIN_C_COMPACT_ACCOUNT_BALANCE_H +#define __GRADIDO_BLOCKCHAIN_C_COMPACT_ACCOUNT_BALANCE_H + +#include +#include "grdc_public_key_index.h" +#include "../grdl/grdl_unit.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct grdc_account_balance +{ + grdl_unit balance; + grdc_public_key_index publicKeyIndex; +} grdc_account_balance; + +#ifdef __cplusplus +} +#endif + + +#endif // __GRADIDO_BLOCKCHAIN_C_COMPACT_ACCOUNT_BALANCE_H \ No newline at end of file diff --git a/dlt-connector/c/grdc/grdc_confirmed_tx.h b/dlt-connector/c/grdc/grdc_confirmed_tx.h new file mode 100644 index 000000000..999e4633d --- /dev/null +++ b/dlt-connector/c/grdc/grdc_confirmed_tx.h @@ -0,0 +1,104 @@ +#ifndef __GRADIDO_BLOCKCHAIN_C_COMPACT_CONFIRMED_TRANSACTION_COMPACT_H +#define __GRADIDO_BLOCKCHAIN_C_COMPACT_CONFIRMED_TRANSACTION_COMPACT_H + +#include +#include "grdc_account_balance.h" +#include "grdc_public_key_index.h" +#include "grdc_tx_id.h" +#include "grdc_tx_timestamps.h" +#include "../grdd/grdd_address_type.h" +#include "../grdd/grdd_balance_derivation_type.h" +#include "../grdd/grdd_cross_group_type.h" +#include "../grdd/grdd_transaction_type.h" +#include "../grdl/grdl_unit.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Gradido Compact Confirmed Transaction +struct grdc_confirmed_tx +{ + grdc_tx_timestamps timestamps; + + // inline Timestamp getConfirmedAt() const { return timestamps.getConfirmedAt(); } + // inline Timestamp getCreatedAt() const { return timestamps.getCreatedAt(); } + + // txId and pairingTxId packed to save 8 Bytes padding + uint64_t txNrs[2]; + uint32_t communityIdIndex[2]; + // inline TxId getTxId() const { return TxId(txNrs[0], communityIdIndex[0]); } + // for cross group transactions, else empty + // inline TxId getPairingTxId() const { return TxId(txNrs[1], communityIdIndex[1]); } + + // account balances and memos via txId in separate list/manager/thingy + + // enums, usually uint8_t + uint8_t crossGroupType; // grdd_cross_group_type + uint8_t transactionType; // grdd_transaction_type + uint8_t balanceDerivationType; // grdd_balance_derivation_type + uint8_t accountBalanceCount; + + grdc_account_balance accountBalances[2]; + + // common fields for most transactions + union { // 24 Bytes + struct { + grdl_unit amount; // 8 Bytes + grdc_public_key_index recipientPublicKeyIndex; // 8 Bytes + uint64_t targetDate; // 8 Bytes + } creation; + struct { + grdl_unit amount; // 8 Bytes + grdc_public_key_index senderPublicKeyIndex; // 8 Bytes + grdc_public_key_index recipientPublicKeyIndex; // 8 Bytes + } transfer; // also used for redeem deferred transfer, and deferredTransferTransactionNr is stored in extra dictionary + struct { + grdl_unit amount; // 8 Bytes + // work only on local, take communityIdIndex from txId + uint32_t senderPublicKeyIndex; // 4 Bytes + uint32_t recipientPublicKeyIndex; // 4 Bytes + uint32_t timeoutDuration; // 4 Bytes + } deferredTransfer; // fund deferred transfer address only on your own community + struct { + grdc_tx_id deferredTransferTransactionNr; // 16 Bytes, contain 4 Bytes padding + } timeoutDeferredTransfer; + struct { + uint8_t addressType; // grdd_address_type // 1 Byte + uint16_t derivationIndex; // 2 Byte (for the time beeing, update if more than 65535 are needed) + uint32_t nameHashIndex; // 4 Bytes + grdc_public_key_index userPublicKeyIndex; // 8 Bytes + grdc_public_key_index accountPublicKeyIndex; // 8 Bytes + } registerAddress; + struct { + grdc_public_key_index communityPublicKeyIndex; // 8 Bytes + grdc_public_key_index communityAufPublicKeyIndex; // 8 Bytes + grdc_public_key_index communityGmwPublicKeyIndex; // 8 Bytes + } communityRoot; + }; +}; + +inline grdd_timestamp grdc_confirmed_tx_get_confirmed_at(const grdc_confirmed_tx* c) { + return grdc_tx_timestamps_get_confirmed_at(&c->timestamps); +} + +inline grdd_timestamp grdc_confirmed_tx_get_created_at(const grdc_confirmed_tx* c) { + return grdc_tx_timestamps_get_created_at(&c->timestamps); +} + +inline grdc_tx_id grdc_confirmed_tx_get_tx_id(const grdc_confirmed_tx* c) { + grdc_tx_id txId{.nr = c->txNrs[0], .communityIdIndex = c->communityIdIndex[0]}; + return txId; +} + +inline grdc_tx_id grdc_confirmed_tx_get_pairing_tx_id(const grdc_confirmed_tx* c) { + grdc_tx_id pairingTxId{.nr = c->txNrs[1], .communityIdIndex = c->communityIdIndex[1]}; + return pairingTxId; +} + +#ifdef __cplusplus +} +#endif + + +#endif //__GRADIDO_BLOCKCHAIN_C_COMPACT_CONFIRMED_TRANSACTION_COMPACT_H \ No newline at end of file diff --git a/dlt-connector/c/grdc/grdc_public_key_index.h b/dlt-connector/c/grdc/grdc_public_key_index.h new file mode 100644 index 000000000..6ab88fcbf --- /dev/null +++ b/dlt-connector/c/grdc/grdc_public_key_index.h @@ -0,0 +1,21 @@ +#ifndef __GRADIDO_BLOCKCHAIN_C_COMPACT_PUBLIC_KEY_INDEX_H +#define __GRADIDO_BLOCKCHAIN_C_COMPACT_PUBLIC_KEY_INDEX_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct grdc_public_key_index +{ + uint32_t publicKeyIndex; + uint32_t communityIdIndex; +} grdc_public_key_index; + +#ifdef __cplusplus +} +#endif + + +#endif // __GRADIDO_BLOCKCHAIN_C_COMPACT_PUBLIC_KEY_INDEX_H \ No newline at end of file diff --git a/dlt-connector/c/grdc/grdc_tx_id.h b/dlt-connector/c/grdc/grdc_tx_id.h new file mode 100644 index 000000000..b8a1ed12c --- /dev/null +++ b/dlt-connector/c/grdc/grdc_tx_id.h @@ -0,0 +1,25 @@ +#ifndef __GRADIDO_BLOCKCHAIN_C_COMPACT_TRANSACTION_ID +#define __GRADIDO_BLOCKCHAIN_C_COMPACT_TRANSACTION_ID + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct grdc_tx_id +{ + uint64_t nr; + uint32_t communityIdIndex; +} grdc_tx_id; + +inline bool grdc_tx_id_empty(const grdc_tx_id* id) { + return !id->nr && !id->communityIdIndex; +} + +#ifdef __cplusplus +} +#endif + +#endif // __GRADIDO_BLOCKCHAIN_C_COMPACT_TRANSACTION_ID diff --git a/dlt-connector/c/grdc/grdc_tx_timestamps.h b/dlt-connector/c/grdc/grdc_tx_timestamps.h new file mode 100644 index 000000000..ab2e331d6 --- /dev/null +++ b/dlt-connector/c/grdc/grdc_tx_timestamps.h @@ -0,0 +1,42 @@ +#ifndef __GRADIDO_BLOCKCHAIN_C_COMPACT_TRANSACTION_TIMESTAMPS_H +#define __GRADIDO_BLOCKCHAIN_C_COMPACT_TRANSACTION_TIMESTAMPS_H + +#include +#include "../grdd/grdd_timestamp.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct grdc_tx_timestamps +{ + uint64_t confirmedSeconds; + uint32_t confirmedNanos; + int32_t deltaMs; // add to confirmed second and nanos to get created_at +} grdc_tx_timestamps; + +inline grdd_timestamp grdc_tx_timestamps_get_confirmed_at(const grdc_tx_timestamps* in) { + grdd_timestamp t; + t.seconds = in->confirmedSeconds; + t.nanos = in->confirmedNanos; + return t; +} + +inline grdd_timestamp grdc_tx_timestamps_get_created_at(const grdc_tx_timestamps* in) { + grdd_timestamp t; + uint64_t sec = (in->confirmedSeconds + in->deltaMs) / 1000; + uint32_t ns = (in->confirmedNanos + (in->deltaMs % 1000)) * 1000000; + if (ns >= 1000000000) { + sec += 1; + ns -= 1000000000; + } + t.seconds = sec; + t.nanos = ns; + return t; +} + +#ifdef __cplusplus +} +#endif + +#endif //__GRADIDO_BLOCKCHAIN_C_COMPACT_TRANSACTION_TIMESTAMPS_H \ No newline at end of file diff --git a/dlt-connector/c/grdd/grdd_address_type.h b/dlt-connector/c/grdd/grdd_address_type.h new file mode 100644 index 000000000..c49593b00 --- /dev/null +++ b/dlt-connector/c/grdd/grdd_address_type.h @@ -0,0 +1,23 @@ +#ifndef __GRADIDO_BLOCKCHAIN_C_DATA_ADDRESS_TYPE_H +#define __GRADIDO_BLOCKCHAIN_C_DATA_ADDRESS_TYPE_H + +#ifdef __cplusplus +extern "C" { +#endif + +enum grdd_address_type { + GRDD_ADDRESS_TYPE_NONE = 0, // if no address was found + GRDD_ADDRESS_TYPE_COMMUNITY_HUMAN = 1, // creation account for human + GRDD_ADDRESS_TYPE_COMMUNITY_GMW = 2, // community public budget account + GRDD_ADDRESS_TYPE_COMMUNITY_AUF = 3, // community compensation and environment founds account + GRDD_ADDRESS_TYPE_COMMUNITY_PROJECT = 4, // no creations allowed + GRDD_ADDRESS_TYPE_SUBACCOUNT = 5, // no creations allowed + GRDD_ADDRESS_TYPE_CRYPTO_ACCOUNT = 6, // user control his keys, no creations + GRDD_ADDRESS_TYPE_DEFERRED_TRANSFER = 7 // special type, no need for register address +}; + +#ifdef __cplusplus +} +#endif + +#endif //__GRADIDO_BLOCKCHAIN_C_DATA_ADDRESS_TYPE_H \ No newline at end of file diff --git a/dlt-connector/c/grdd/grdd_balance_derivation_type.h b/dlt-connector/c/grdd/grdd_balance_derivation_type.h new file mode 100644 index 000000000..1a24131ac --- /dev/null +++ b/dlt-connector/c/grdd/grdd_balance_derivation_type.h @@ -0,0 +1,26 @@ +#ifndef GRADIDO_BLOCKCHAIN_C_DATA_BALANCE_DERIVATION_TYPE_H +#define GRADIDO_BLOCKCHAIN_C_DATA_BALANCE_DERIVATION_TYPE_H + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @enum grdd_balance_derivation_type + * + * Flag to decide if Node calculates balance and decay + * or trusts external data. + */ +enum grdd_balance_derivation_type { + GRDD_BALANCE_DERIVATION_UNSPECIFIED = 0, + /* Balances & decay can be recalculated deterministically */ + GRDD_BALANCE_DERIVATION_NODE = 1, + /* Balances are accepted as-is from external / legacy system */ + GRDD_BALANCE_DERIVATION_EXTERN = 2 +}; + +#ifdef __cplusplus +} +#endif + +#endif /* GRADIDO_BLOCKCHAIN_C_DATA_BALANCE_DERIVATION_TYPE_H */ diff --git a/dlt-connector/c/grdd/grdd_cross_group_type.h b/dlt-connector/c/grdd/grdd_cross_group_type.h new file mode 100644 index 000000000..2a0c4a72a --- /dev/null +++ b/dlt-connector/c/grdd/grdd_cross_group_type.h @@ -0,0 +1,20 @@ +#ifndef __GRADIDO_BLOCKCHAIN_C_DATA_CROSS_GROUP_TYPE_H +#define __GRADIDO_BLOCKCHAIN_C_DATA_CROSS_GROUP_TYPE_H + +#ifdef __cplusplus +extern "C" { +#endif + +enum grdd_cross_group_type +{ + GRDD_CROSS_GROUP_TYPE_LOCAL = 0, + GRDD_CROSS_GROUP_TYPE_INBOUND = 1, + GRDD_CROSS_GROUP_TYPE_OUTBOUND = 2, + GRDD_CROSS_GROUP_TYPE_CROSS = 3 +}; + +#ifdef __cplusplus +} +#endif + +#endif // __GRADIDO_BLOCKCHAIN_C_DATA_CROSS_GROUP_TYPE_H \ No newline at end of file diff --git a/dlt-connector/c/grdd/grdd_timestamp.h b/dlt-connector/c/grdd/grdd_timestamp.h new file mode 100644 index 000000000..2bf54de67 --- /dev/null +++ b/dlt-connector/c/grdd/grdd_timestamp.h @@ -0,0 +1,38 @@ +#ifndef __GRADIDO_BLOCKCHAIN_C_DATA_TIMESTAMP_H +#define __GRADIDO_BLOCKCHAIN_C_DATA_TIMESTAMP_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct grdd_timestamp +{ + int64_t seconds; + int32_t nanos; +} grdd_timestamp; + +inline bool grdd_timestamp_empty(const grdd_timestamp* timestamp) { + return !timestamp->seconds && !timestamp->nanos; +} +inline bool grdd_timestamp_eq(const grdd_timestamp* t1, grdd_timestamp* t2) { + return t1->seconds == t2->seconds && t1->nanos == t2->nanos; +} +inline bool grdd_timestamp_gt(const grdd_timestamp* t1, const grdd_timestamp* t2) { + return t1->seconds > t2->seconds || t1->seconds == t2->seconds && t1->nanos > t2->nanos; +} +inline bool grdd_timestamp_lt(const grdd_timestamp* t1, const grdd_timestamp* t2) { + return t1->seconds < t2->seconds || t1->seconds == t2->seconds && t1->nanos < t2->nanos; +} +inline grdd_timestamp grdd_timestamp_sub(const grdd_timestamp* t1, grdd_timestamp* t2) { + grdd_timestamp result{.seconds = t1->seconds - t2->seconds, .nanos = t1->nanos - t2->nanos}; + return result; +} + +#ifdef __cplusplus +} +#endif + +#endif // __GRADIDO_BLOCKCHAIN_C_DATA_TIMESTAMP_H \ No newline at end of file diff --git a/dlt-connector/c/grdd/grdd_transaction_type.h b/dlt-connector/c/grdd/grdd_transaction_type.h new file mode 100644 index 000000000..83a0540ea --- /dev/null +++ b/dlt-connector/c/grdd/grdd_transaction_type.h @@ -0,0 +1,56 @@ +#ifndef GRADIDO_BLOCKCHAIN_C_DATA_TRANSACTION_TYPE_H +#define GRADIDO_BLOCKCHAIN_C_DATA_TRANSACTION_TYPE_H + +#ifdef __cplusplus +extern "C" { +#endif + +/*! + * \addtogroup enums + * @{ + */ + +/*! + * \enum grdd_transaction_type + * Enum for different transaction types + * !!! don't change order + */ +enum grdd_transaction_type { + //! Invalid or Empty Transaction + GRDD_TRANSACTION_TYPE_NONE = 0, + + //! Creation Transaction, creates new Gradidos + GRDD_TRANSACTION_TYPE_CREATION, + + //! Transfer Transaction, move Gradidos from one account to another + GRDD_TRANSACTION_TYPE_TRANSFER, + + //! Group Friends Update Transaction, update relationship between groups + GRDD_TRANSACTION_TYPE_COMMUNITY_FRIENDS_UPDATE, + + //! Register new address or sub address to group or move address to another group + GRDD_TRANSACTION_TYPE_REGISTER_ADDRESS, + + //! Special Transfer Transaction with timeout used for Gradido Link + GRDD_TRANSACTION_TYPE_DEFERRED_TRANSFER, + + //! First Transaction in Blockchain + GRDD_TRANSACTION_TYPE_COMMUNITY_ROOT, + + //! Redeeming deferred transfer + GRDD_TRANSACTION_TYPE_REDEEM_DEFERRED_TRANSFER, + + //! Timeout deferred transfer, send back locked gdds + GRDD_TRANSACTION_TYPE_TIMEOUT_DEFERRED_TRANSFER, + + //! Technical type for using it in for loops, as max index + GRDD_TRANSACTION_TYPE_MAX_VALUE +}; + +/*! @} */ + +#ifdef __cplusplus +} +#endif + +#endif /* GRADIDO_BLOCKCHAIN_C_DATA_TRANSACTION_TYPE_H */ \ No newline at end of file diff --git a/dlt-connector/c/grdl/grdl_unit.c b/dlt-connector/c/grdl/grdl_unit.c new file mode 100644 index 000000000..90f68ad38 --- /dev/null +++ b/dlt-connector/c/grdl/grdl_unit.c @@ -0,0 +1,123 @@ +#include "grdl_unit.h" + +#include +#include +#include + +const double SECONDS_PER_YEAR = 31556952.0; // seconds in a year in gregorian calender +const grdd_timestamp DECAY_START_TIME = (grdd_timestamp){.seconds = 1620927991, .nanos = 0}; + +double roundToPrecision(double gdd, uint8_t precision) +{ + double factor = pow(10.0, precision); + return round(gdd * factor) / factor; +} + +grdl_unit grdl_unit_from_decimal(double gdd) +{ + grdl_unit gradidoUnit = { + (int64_t)(roundToPrecision(gdd, 4) * 10000.0) + }; + return gradidoUnit; +} + +grdl_unit grdl_unit_from_string(const char* gdd_string) +{ + if (!gdd_string) return (grdl_unit){0}; + char* end; + double gdd_double = strtod(gdd_string, &end); + if (end == gdd_string || *end != '\0') { + // invalid string + return (grdl_unit){0}; + } + return grdl_unit_from_decimal(gdd_double); +} + +int grdl_unit_to_string(const grdl_unit* u, char* buffer, size_t bufferSize, uint8_t precision) +{ + if (precision > 4) return 1; // C hat keine Exceptions + + // Convert to double + double decimal = (double)(u->gradidoCent) / 10000.0; + + // Round down like Node.js if precision < 4 + if (precision < 4) { + double factor = pow(10.0, precision); + decimal = round(decimal * factor) / factor; + } + + // Write to buffer + int written = snprintf(buffer, bufferSize, "%.*f", precision, decimal); + + // snprintf returns number of chars that would have been written (excluding null) + // snprintf return negative value on encoding error + if (written < 0) return 2; + if ((size_t)written < bufferSize) { + return 0; + } + return bufferSize - written; +} + +grdl_unit grdl_unit_calculate_decay(const grdl_unit* u, int64_t seconds) +{ + if (seconds == 0) return (grdl_unit){u->gradidoCent}; + + // decay for one year is 50% + /* + * while (seconds >= SECONDS_PER_YEAR) { + mGradidoCent *= 0.5; + seconds -= SECONDS_PER_YEAR; + } + */ + int64_t gradidoCent = u->gradidoCent; + // optimize version from above + if (seconds >= SECONDS_PER_YEAR) { + uint64_t times = (uint64_t)(seconds / SECONDS_PER_YEAR); + seconds = seconds - times * SECONDS_PER_YEAR; + gradidoCent = u->gradidoCent >> times; + if (!seconds) return (grdl_unit){gradidoCent}; + } +// */ + /*! + * calculate decay factor with compound interest formula converted to q
+ * n = (lg Kn - lg K0) / lg q =>
+ * lg q = (lg Kn - lg K0) / n =>
+ * q = e^((lg Kn - lg K0) / n)
+ *
+ * with: + *
    + *
  • q = decay_factor
    • + *
  • n = days_per_year * 60 * 60 * 24 = seconds per year
    • + *
  • Kn = 50 (capital after a year)
    • + *
  • K0 = 100 (capital at start)
    • + *
+ * further simplified: + * lg 50 - lg 100 = lg 2 => + * q = e^(lg 2 / n) = 2^(x/n) + * with x as seconds in which decay occured + */ + // https://www.wolframalpha.com/input?i=%28e%5E%28lg%282%29+%2F+31556952%29%29%5Ex&assumption=%7B%22FunClash%22%2C+%22lg%22%7D+-%3E+%7B%22Log%22%7D + // from wolframalpha, based on the interest rate formula + return (grdl_unit){((int64_t)((double)(gradidoCent) * pow(2.0, (double)((double)(-seconds) / SECONDS_PER_YEAR))))}; +} + +bool grdl_unit_calculate_duration_seconds( + const grdd_timestamp* startTime, + const grdd_timestamp* endTime, + int64_t* outSeconds +) { + if (!outSeconds) { + return false; + } + if(grdd_timestamp_gt(startTime, endTime) { + return false; + } + grdd_timestamp start = grdd_timestamp_gt(startTime, &DECAY_START_TIME) ? *startTime : DECAY_START_TIME; + grdd_timestamp end = grdd_timestamp_gt(endTime, &DECAY_START_TIME) ? *endTime : DECAY_START_TIME; + if (grdd_timestamp_eq(&start, &end)) { + *outSeconds = 0; + return true; + } + *outSeconds = grdd_timestamp_sub(&end, &start).seconds; + return true; +} diff --git a/dlt-connector/c/grdl/grdl_unit.h b/dlt-connector/c/grdl/grdl_unit.h new file mode 100644 index 000000000..66a658e1b --- /dev/null +++ b/dlt-connector/c/grdl/grdl_unit.h @@ -0,0 +1,84 @@ +#ifndef __GRADIDO_BLOCKCHAIN_C_LOGIC_UNIT_H +#define __GRADIDO_BLOCKCHAIN_C_LOGIC_UNIT_H + +#include "../grdd/grdd_timestamp.h" + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct grdl_unit +{ + int64_t gradidoCent; +} grdl_unit; + +grdl_unit grdl_unit_from_decimal(double gdd); +grdl_unit grdl_unit_from_string(const char* gdd_string); + +//! \return 0 for ok, 1 for invalid precision, 2 for printf encoding error +// and -x if string buffer wasn't big enough where x is the number of missing bytes +int grdl_unit_to_string(const grdl_unit* u, char* buffer, size_t bufferSize, uint8_t precision); + +inline grdl_unit grdl_unit_negated(const grdl_unit* u) +{ + grdl_unit gradidoUnit = {u->gradidoCent}; + gradidoUnit.gradidoCent *= -1; + return gradidoUnit; +} + +inline void grdl_unit_negate(grdl_unit* u) +{ + if (u) u->gradidoCent = -u->gradidoCent; +} + +static inline grdl_unit grdl_unit_zero() +{ + return (grdl_unit){0}; +} +//! return false if startTime > endTime +bool grdl_unit_calculate_duration_seconds( + const grdd_timestamp* startTime, + const grdd_timestamp* endTime, + int64_t* outSeconds +); + +grdl_unit grdl_unit_calculate_decay(const grdl_unit* u, int64_t seconds); + +inline grdl_unit grdl_unit_calculate_decay_timestamp( + const grdl_unit* u, + const grdd_timestamp* startTime, + const grdd_timestamp* endTime +) { + int64_t seconds = 0; + if(!grdl_unit_calculate_duration_seconds(startTime, endTime, &seconds)) { + return (grdl_unit){0}; + } + return grdl_unit_calculate_decay(u, seconds); +} + +inline grdl_unit grld_unit_calculate_compound_interest(const grdl_unit* u, int64_t seconds) { + return grdl_unit_calculate_decay(u, -seconds); +} + +inline grdl_unit grld_unit_calculate_compound_interest_timestamp( + const grdl_unit* u, + const grdd_timestamp* startTime, + const grdd_timestamp* endTime +) { + int64_t seconds = 0; + if(!grdl_unit_calculate_duration_seconds(startTime, endTime, &seconds)) { + return (grdl_unit){0}; + } + return grdl_unit_calculate_decay(u, -seconds); +} + +#ifdef __cplusplus +} +#endif + + +#endif // __GRADIDO_BLOCKCHAIN_C_LOGIC_UNIT_H \ No newline at end of file diff --git a/dlt-connector/package.json b/dlt-connector/package.json index 84ed03df7..518e17418 100644 --- a/dlt-connector/package.json +++ b/dlt-connector/package.json @@ -18,6 +18,7 @@ "lint:fix": "biome check --error-on-warnings . --write" }, "dependencies": { + "bun-zigar": "^0.15.2", "cross-env": "^7.0.3", "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#f265dbb1780a912cf8b0418dfe3eaf5cdc5b51cf" }, diff --git a/dlt-connector/zig/hello.zig b/dlt-connector/zig/hello.zig new file mode 100644 index 000000000..36bc9b556 --- /dev/null +++ b/dlt-connector/zig/hello.zig @@ -0,0 +1,5 @@ +const std = @import("std"); + +pub fn hello() void { + std.debug.print("Hello world!", .{}); +} From be48f6aaa2bf75864a1c6e63ba505f915b8fc50a Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Mon, 9 Feb 2026 09:24:08 +0100 Subject: [PATCH 21/50] reduce new calls every tx --- .../binaryExport.ts | 6 +++++- .../db-v2.7.0_to_blockchain-v3.7/index.ts | 3 +++ .../syncDbWithBlockchain/AbstractSync.role.ts | 15 ++++++++++++++- .../CreationsSync.role.ts | 18 ++++++++++++------ .../DeletedTransactionLinksSync.role.ts | 16 +++++++++++----- .../LocalTransactionsSync.role.ts | 17 +++++++++++------ .../RedeemTransactionLinksSync.role.ts | 18 ++++++++++++------ .../RemoteTransactionsSync.role.ts | 19 ++++++++++++------- .../TransactionLinkFundingsSync.role.ts | 15 ++++++++++----- .../syncDbWithBlockchain/UsersSync.role.ts | 14 +++++++++----- .../syncDbWithBlockchain.context.ts | 10 +++++++--- 11 files changed, 106 insertions(+), 45 deletions(-) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts index 9d34f5588..8ec631201 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts @@ -27,6 +27,7 @@ export function exportCommunity( batchSize: number, ) { const timeUsed = new Profiler() + const timeSinceLastPrint = new Profiler() // write as binary file for GradidoNode const f = new Filter() f.pagination.size = batchSize @@ -64,7 +65,10 @@ export function exportCommunity( count++ } if (isDebug) { - printConsole() + if (timeSinceLastPrint.millis() > 100) { + printConsole() + timeSinceLastPrint.reset() + } } else { printCount++ if (printCount >= 100) { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts index b200e8769..cb8e99af3 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts @@ -2,10 +2,13 @@ import { onShutdown } from '../../../../shared/src/helper/onShutdown' import { exportAllCommunities } from './binaryExport' import { bootstrap } from './bootstrap' import { syncDbWithBlockchainContext } from './interaction/syncDbWithBlockchain/syncDbWithBlockchain.context' +// import { hello } from '../../../zig/hello.zig' const BATCH_SIZE = 1000 async function main() { + // hello() + // return // prepare in memory blockchains const context = await bootstrap() onShutdown(async (reason, error) => { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts index bcdc133b5..b2d532c44 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts @@ -1,4 +1,13 @@ -import { Filter, InMemoryBlockchain, KeyPairEd25519, MemoryBlockPtr, Profiler, SearchDirection_DESC } from 'gradido-blockchain-js' +import { + AccountBalances, + Filter, + InMemoryBlockchain, + KeyPairEd25519, + MemoryBlockPtr, + Profiler, + SearchDirection_DESC, + GradidoTransactionBuilder +} from 'gradido-blockchain-js' import { getLogger, Logger } from 'log4js' import { LOG4JS_BASE_CATEGORY } from '../../../../config/const' import { deriveFromKeyPairAndIndex, deriveFromKeyPairAndUuid } from '../../../../data/deriveKeyPair' @@ -18,11 +27,15 @@ export abstract class AbstractSyncRole { private items: ItemType[] = [] protected lastIndex: IndexType = { date: new Date(0), id: 0 } protected logger: Logger + protected transactionBuilder: GradidoTransactionBuilder + protected accountBalances: AccountBalances constructor(protected readonly context: Context) { this.logger = getLogger( `${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5.interaction.syncDbWithBlockchain`, ) + this.transactionBuilder = new GradidoTransactionBuilder() + this.accountBalances = new AccountBalances() } getAccountKeyPair(communityContext: CommunityContext, gradidoId: Uuidv4): KeyPairEd25519 { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts index 9546c07b9..589c65423 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts @@ -16,6 +16,7 @@ import { import * as v from 'valibot' import { addToBlockchain } from '../../blockchain' import { ContributionStatus } from '../../data/ContributionStatus' +import { Context } from '../../Context' import { contributionsTable, usersTable @@ -27,6 +28,11 @@ import { toMysqlDateTime } from '../../utils' export class CreationsSyncRole extends AbstractSyncRole { + constructor(context: Context) { + super(context) + this.accountBalances.reserve(3) + } + getDate(): Date { return this.peek().confirmedAt } @@ -85,7 +91,7 @@ export class CreationsSyncRole extends AbstractSyncRole { recipientKeyPair: KeyPairEd25519, signerKeyPair: KeyPairEd25519 ): GradidoTransactionBuilder { - return new GradidoTransactionBuilder() + return this.transactionBuilder .setCreatedAt(item.confirmedAt) .addMemo( new EncryptedMemo( @@ -107,7 +113,7 @@ export class CreationsSyncRole extends AbstractSyncRole { communityContext: CommunityContext, recipientPublicKey: MemoryBlockPtr ): AccountBalances { - const accountBalances = new AccountBalances() + this.accountBalances.clear() const balance = this.getLastBalanceForUser(recipientPublicKey, communityContext.blockchain, communityContext.communityId) // calculate decay since last balance with legacy calculation method @@ -115,10 +121,10 @@ export class CreationsSyncRole extends AbstractSyncRole { communityContext.aufBalance.updateLegacyDecay(item.amount, item.confirmedAt) communityContext.gmwBalance.updateLegacyDecay(item.amount, item.confirmedAt) - accountBalances.add(balance.getAccountBalance()) - accountBalances.add(communityContext.aufBalance.getAccountBalance()) - accountBalances.add(communityContext.gmwBalance.getAccountBalance()) - return accountBalances + this.accountBalances.add(balance.getAccountBalance()) + this.accountBalances.add(communityContext.aufBalance.getAccountBalance()) + this.accountBalances.add(communityContext.gmwBalance.getAccountBalance()) + return this.accountBalances } pushToBlockchain(item: CreationTransactionDb): void { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts index a1e6d046e..4eb338d37 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts @@ -21,8 +21,14 @@ import { addToBlockchain } from '../../blockchain' import { BlockchainError, DatabaseError } from '../../errors' import { Balance } from '../../data/Balance' import { toMysqlDateTime } from '../../utils' +import { Context } from '../../Context' export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { + constructor(context: Context) { + super(context) + this.accountBalances.reserve(2) + } + getDate(): Date { return this.peek().deletedAt } @@ -81,7 +87,7 @@ export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { - + constructor(context: Context) { + super(context) + this.accountBalances.reserve(2) + } + getDate(): Date { return this.peek().balanceDate } @@ -84,7 +89,7 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { senderKeyPair: KeyPairEd25519, recipientKeyPair: KeyPairEd25519, ): GradidoTransactionBuilder { - return new GradidoTransactionBuilder() + return this.transactionBuilder .setCreatedAt(item.balanceDate) .addMemo(new EncryptedMemo( item.memo, @@ -106,7 +111,7 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { senderPublicKey: MemoryBlockPtr, recipientPublicKey: MemoryBlockPtr, ): AccountBalances { - const accountBalances = new AccountBalances() + this.accountBalances.clear() const senderLastBalance = this.getLastBalanceForUser(senderPublicKey, communityContext.blockchain, communityContext.communityId) const recipientLastBalance = this.getLastBalanceForUser(recipientPublicKey, communityContext.blockchain, communityContext.communityId) @@ -121,9 +126,9 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { } recipientLastBalance.updateLegacyDecay(item.amount, item.balanceDate) - accountBalances.add(senderLastBalance.getAccountBalance()) - accountBalances.add(recipientLastBalance.getAccountBalance()) - return accountBalances + this.accountBalances.add(senderLastBalance.getAccountBalance()) + this.accountBalances.add(recipientLastBalance.getAccountBalance()) + return this.accountBalances } pushToBlockchain(item: TransactionDb): void { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts index 0f472a109..0134c7b6d 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts @@ -23,8 +23,14 @@ import { AbstractSyncRole, IndexType } from './AbstractSync.role' import { deriveFromCode } from '../../../../data/deriveKeyPair' import { alias } from 'drizzle-orm/mysql-core' import { toMysqlDateTime } from '../../utils' +import { Context } from '../../Context' export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { + constructor(context: Context) { + super(context) + this.accountBalances.reserve(3) + } + getDate(): Date { return this.peek().redeemedAt } @@ -88,7 +94,7 @@ export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { + constructor(context: Context) { + super(context) + this.accountBalances.reserve(1) + } + getDate(): Date { return this.peek().balanceDate } @@ -79,7 +85,7 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole senderCommunityId: string, recipientCommunityId: string, ): GradidoTransactionBuilder { - const builder = new GradidoTransactionBuilder() + return this.transactionBuilder .setCreatedAt(item.balanceDate) .addMemo(new EncryptedMemo( item.memo, @@ -94,7 +100,6 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole .setSenderCommunity(senderCommunityId) .setRecipientCommunity(recipientCommunityId) .sign(senderKeyPair) - return builder } calculateBalances( @@ -103,10 +108,10 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole amount: GradidoUnit, publicKey: MemoryBlockPtr, ): AccountBalances { - const accountBalances = new AccountBalances() + this.accountBalances.clear() if (communityContext.foreign) { - accountBalances.add(new AccountBalance(publicKey, GradidoUnit.zero(), communityContext.communityId)) - return accountBalances + this.accountBalances.add(new AccountBalance(publicKey, GradidoUnit.zero(), communityContext.communityId)) + return this.accountBalances } else { const lastBalance = this.getLastBalanceForUser(publicKey, communityContext.blockchain, communityContext.communityId) @@ -118,8 +123,8 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole throw e } } - accountBalances.add(lastBalance.getAccountBalance()) - return accountBalances + this.accountBalances.add(lastBalance.getAccountBalance()) + return this.accountBalances } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts index bc568df61..167de11ae 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts @@ -22,8 +22,13 @@ import { AbstractSyncRole, IndexType } from './AbstractSync.role' import { deriveFromCode } from '../../../../data/deriveKeyPair' import { reverseLegacyDecay, toMysqlDateTime } from '../../utils' import Decimal from 'decimal.js-light' +import { Context } from '../../Context' export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { + constructor(context: Context) { + super(context) + this.accountBalances.reserve(2) + } getDate(): Date { return this.peek().createdAt } @@ -73,7 +78,7 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { - + constructor(context: Context) { + super(context) + this.accountBalances.reserve(1) + } getDate(): Date { return this.peek().createdAt } @@ -66,7 +70,7 @@ export class UsersSyncRole extends AbstractSyncRole { accountKeyPair: KeyPairEd25519, userKeyPair: KeyPairEd25519 ): GradidoTransactionBuilder { - return new GradidoTransactionBuilder() + return this.transactionBuilder .setCreatedAt(item.createdAt) .setRegisterAddress( userKeyPair.getPublicKey(), @@ -81,9 +85,9 @@ export class UsersSyncRole extends AbstractSyncRole { } calculateAccountBalances(accountPublicKey: MemoryBlockPtr, communityContext: CommunityContext,): AccountBalances { - const accountBalances = new AccountBalances() - accountBalances.add(new AccountBalance(accountPublicKey, GradidoUnit.zero(), communityContext.communityId)) - return accountBalances + this.accountBalances.clear() + this.accountBalances.add(new AccountBalance(accountPublicKey, GradidoUnit.zero(), communityContext.communityId)) + return this.accountBalances } pushToBlockchain(item: UserDb): void { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts index 6446e2600..024836aa0 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts @@ -15,6 +15,7 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n const timeUsedDB = new Profiler() const timeUsedBlockchain = new Profiler() const timeUsedAll = new Profiler() + const timeBetweenPrints = new Profiler() const containers = [ new UsersSyncRole(context), new CreationsSyncRole(context), @@ -23,7 +24,7 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n new RedeemTransactionLinksSyncRole(context), new ContributionLinkTransactionSyncRole(context), new DeletedTransactionLinksSyncRole(context), - new RemoteTransactionsSyncRole(context), + // new RemoteTransactionsSyncRole(context), ] let transactionsCount = 0 let transactionsCountSinceLastLog = 0 @@ -54,8 +55,11 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n } available[0].toBlockchain() transactionsCount++ - if (isDebug) { - process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`) + if (isDebug) { + if (timeBetweenPrints.millis() > 100) { + process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`) + timeBetweenPrints.reset() + } transactionsCountSinceLastLog++ if (transactionsCountSinceLastLog >= batchSize) { context.logger.debug(`${transactionsCountSinceLastLog} transactions added to blockchain in ${timeUsedBlockchain.string()}`) From 4dde30ea4bc7d585a02eeb7fd02771c43f5f40b3 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Fri, 13 Feb 2026 08:38:20 +0100 Subject: [PATCH 22/50] fix order of builder commands, debug logs --- .../db-v2.7.0_to_blockchain-v3.7/blockchain.ts | 5 +++++ .../db-v2.7.0_to_blockchain-v3.7/bootstrap.ts | 8 ++++---- .../syncDbWithBlockchain/CreationsSync.role.ts | 2 +- .../DeletedTransactionLinksSync.role.ts | 2 +- .../LocalTransactionsSync.role.ts | 2 +- .../RedeemTransactionLinksSync.role.ts | 2 +- .../RemoteTransactionsSync.role.ts | 11 ++++++++--- .../TransactionLinkFundingsSync.role.ts | 4 ++-- .../syncDbWithBlockchain/UsersSync.role.ts | 2 +- .../syncDbWithBlockchain.context.ts | 2 +- .../src/schemas/typeConverter.schema.test.ts | 1 - 11 files changed, 25 insertions(+), 16 deletions(-) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts index ffff61f1a..fe7dbd62c 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts @@ -4,8 +4,10 @@ import { GradidoTransaction, HieroAccountId, InMemoryBlockchain, + InMemoryBlockchainProvider, LedgerAnchor, Profiler, + TransactionEntry, } from 'gradido-blockchain-js' import { NotEnoughGradidoBalanceError } from './errors' @@ -38,6 +40,9 @@ export function addToBlockchain( throw new NotEnoughGradidoBalanceError(needed, exist) } } + // const wekingheim = InMemoryBlockchainProvider.getInstance().getBlockchain('wekingheim') + // const lastTransactionw = wekingheim?.findOne(Filter.LAST_TRANSACTION) + const lastTransaction = blockchain.findOne(Filter.LAST_TRANSACTION) throw new Error(`Transaction ${transaction.toJson(true)} not added: ${error}, last transaction was: ${lastTransaction?.getConfirmedTransaction()?.toJson(true)}`) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts index c1f5bc1ae..3ffad58c8 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts @@ -1,5 +1,5 @@ import { randomBytes } from 'node:crypto' -import { Abstract, AccountBalances, GradidoTransactionBuilder, InMemoryBlockchain, InMemoryBlockchainProvider, LedgerAnchor } from 'gradido-blockchain-js' +import { AccountBalances, GradidoTransactionBuilder, InMemoryBlockchainProvider, LedgerAnchor } from 'gradido-blockchain-js' import * as v from 'valibot' import { CONFIG } from '../../config' import { deriveFromSeed } from '../../data/deriveKeyPair' @@ -33,7 +33,7 @@ async function bootstrapCommunities(context: Context): Promise { ): GradidoTransactionBuilder { return this.transactionBuilder .setCreatedAt(item.confirmedAt) + .setRecipientCommunity(communityContext.communityId) .addMemo( new EncryptedMemo( item.memo, @@ -104,7 +105,6 @@ export class CreationsSyncRole extends AbstractSyncRole { new TransferAmount(recipientKeyPair.getPublicKey(), item.amount, communityContext.communityId), item.contributionDate, ) - .setRecipientCommunity(communityContext.communityId) .sign(signerKeyPair) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts index 4eb338d37..7e609ff2c 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts @@ -89,6 +89,7 @@ export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { new AuthenticatedEncryption(recipientKeyPair), ), ) + .setSenderCommunity(communityContext.communityId) .setTransactionTransfer( new TransferAmount(senderKeyPair.getPublicKey(), item.amount, communityContext.communityId), recipientKeyPair.getPublicKey(), ) - .setSenderCommunity(communityContext.communityId) .sign(senderKeyPair) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts index 0134c7b6d..e6d9d0c75 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts @@ -103,6 +103,7 @@ export class RedeemTransactionLinksSyncRole extends AbstractSyncRole new AuthenticatedEncryption(recipientKeyPair), ), ) + .setSenderCommunity(senderCommunityId) + .setRecipientCommunity(recipientCommunityId) .setTransactionTransfer( new TransferAmount(senderKeyPair.getPublicKey(), item.amount, senderCommunityId), recipientKeyPair.getPublicKey(), - ) - .setSenderCommunity(senderCommunityId) - .setRecipientCommunity(recipientCommunityId) + ) .sign(senderKeyPair) } @@ -165,6 +165,9 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole recipientCommunityContext.communityId ) const outboundTransaction = transactionBuilder.buildOutbound() + console.log(ledgerAnchor.toJson(true)) + console.log(outboundTransaction.toJson(true)) + console.log("outbound coin color: %d", outboundTransaction.getTransactionBody()?.getTransferAmount().getCoinCommunityIdIndex()) try { addToBlockchain( @@ -181,6 +184,8 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole } transactionBuilder.setParentLedgerAnchor(ledgerAnchor) const inboundTransaction = transactionBuilder.buildInbound() + console.log(inboundTransaction.toJson(true)) + console.log("inbound coin color: %d", inboundTransaction.getTransactionBody()?.getTransferAmount().getCoinCommunityIdIndex()) try { addToBlockchain( inboundTransaction, diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts index 167de11ae..ddbddd9d4 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts @@ -87,14 +87,14 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { ): GradidoTransactionBuilder { return this.transactionBuilder .setCreatedAt(item.createdAt) + .setSenderCommunity(communityContext.communityId) .setRegisterAddress( userKeyPair.getPublicKey(), AddressType_COMMUNITY_HUMAN, new Uuidv4Hash(item.gradidoId).getAsMemoryBlock(), accountKeyPair.getPublicKey(), ) - .setSenderCommunity(communityContext.communityId) .sign(communityKeyPair) .sign(accountKeyPair) .sign(userKeyPair) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts index 024836aa0..833b7d6f8 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts @@ -24,7 +24,7 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n new RedeemTransactionLinksSyncRole(context), new ContributionLinkTransactionSyncRole(context), new DeletedTransactionLinksSyncRole(context), - // new RemoteTransactionsSyncRole(context), + new RemoteTransactionsSyncRole(context), ] let transactionsCount = 0 let transactionsCountSinceLastLog = 0 diff --git a/dlt-connector/src/schemas/typeConverter.schema.test.ts b/dlt-connector/src/schemas/typeConverter.schema.test.ts index 5420de106..a46341599 100644 --- a/dlt-connector/src/schemas/typeConverter.schema.test.ts +++ b/dlt-connector/src/schemas/typeConverter.schema.test.ts @@ -102,6 +102,5 @@ describe('basic.schema', () => { ) expect(confirmedTransaction.getId()).toBe(7) expect(confirmedTransaction.getConfirmedAt().getSeconds()).toBe(1609464130) - expect(confirmedTransaction.getVersionNumber()).toBe('3.5') }) }) From 521e678e1f3884c20736713ce77b6d7bb21544d3 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 14 Feb 2026 17:23:07 +0100 Subject: [PATCH 23/50] call verify separatly as multithreaded version --- .../db-v2.7.0_to_blockchain-v3.7/blockchain.ts | 5 +---- .../db-v2.7.0_to_blockchain-v3.7/index.ts | 14 +++++++++++++- .../RemoteTransactionsSync.role.ts | 5 ----- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts index fe7dbd62c..770122f92 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts @@ -4,10 +4,8 @@ import { GradidoTransaction, HieroAccountId, InMemoryBlockchain, - InMemoryBlockchainProvider, LedgerAnchor, Profiler, - TransactionEntry, } from 'gradido-blockchain-js' import { NotEnoughGradidoBalanceError } from './errors' @@ -21,10 +19,9 @@ export function addToBlockchain( ledgerAnchor: LedgerAnchor, accountBalances: AccountBalances, ): boolean { - try { timeUsed.reset() - const result = blockchain.createAndAddConfirmedTransactionExtern( + const result = blockchain.createAndAddConfirmedTransactionExternFast( transaction, ledgerAnchor, accountBalances, diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts index cb8e99af3..aa418625d 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts @@ -2,7 +2,8 @@ import { onShutdown } from '../../../../shared/src/helper/onShutdown' import { exportAllCommunities } from './binaryExport' import { bootstrap } from './bootstrap' import { syncDbWithBlockchainContext } from './interaction/syncDbWithBlockchain/syncDbWithBlockchain.context' -// import { hello } from '../../../zig/hello.zig' +import { Filter, Profiler, ThreadingPolicy_Half, verifySignatures } from 'gradido-blockchain-js' + // import { hello } from '../../../zig/hello.zig' const BATCH_SIZE = 1000 @@ -26,6 +27,17 @@ async function main() { //context.logBlogchain(v.parse(uuidv4Schema, 'e70da33e-5976-4767-bade-aa4e4fa1c01a')) } + const timeUsed = new Profiler() + // bulk verify transaction signatures + for (const communityContext of context.communities.values()) { + // verifySignatures(Filter.ALL_TRANSACTIONS, ThreadingPolicy_Half) + const result = verifySignatures(Filter.ALL_TRANSACTIONS, communityContext.communityId, ThreadingPolicy_Half) + if(!result.isEmpty()){ + throw new Error(`Verification of signatures failed for community ${communityContext.communityId}`) + } + } + context.logger.info(`verified in ${timeUsed.string()}`) + // write as binary file for GradidoNode exportAllCommunities(context, BATCH_SIZE) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts index d2d01925a..712119378 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts @@ -165,9 +165,6 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole recipientCommunityContext.communityId ) const outboundTransaction = transactionBuilder.buildOutbound() - console.log(ledgerAnchor.toJson(true)) - console.log(outboundTransaction.toJson(true)) - console.log("outbound coin color: %d", outboundTransaction.getTransactionBody()?.getTransferAmount().getCoinCommunityIdIndex()) try { addToBlockchain( @@ -184,8 +181,6 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole } transactionBuilder.setParentLedgerAnchor(ledgerAnchor) const inboundTransaction = transactionBuilder.buildInbound() - console.log(inboundTransaction.toJson(true)) - console.log("inbound coin color: %d", inboundTransaction.getTransactionBody()?.getTransferAmount().getCoinCommunityIdIndex()) try { addToBlockchain( inboundTransaction, From fcb701efb8586579ce06eb85b55e3148b7960a7c Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 18 Feb 2026 14:07:39 +0100 Subject: [PATCH 24/50] set correct coin color values --- .../syncDbWithBlockchain/AbstractSync.role.ts | 4 +++- .../syncDbWithBlockchain/RemoteTransactionsSync.role.ts | 9 +++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts index b2d532c44..a5cfb710c 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts @@ -50,7 +50,9 @@ export abstract class AbstractSyncRole { if (publicKey.isEmpty()) { throw new Error('publicKey is empty') } - const lastSenderTransaction = blockchain.findOne(Filter.lastBalanceFor(publicKey)) + const f = Filter.lastBalanceFor(publicKey) + f.coinCommunityId = communityId + const lastSenderTransaction = blockchain.findOne(f) if (!lastSenderTransaction) { return new Balance(publicKey, communityId) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts index 712119378..567a7ee79 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts @@ -105,15 +105,16 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole calculateBalances( item: TransactionDb, communityContext: CommunityContext, + coinCommunityId: string, amount: GradidoUnit, publicKey: MemoryBlockPtr, ): AccountBalances { this.accountBalances.clear() if (communityContext.foreign) { - this.accountBalances.add(new AccountBalance(publicKey, GradidoUnit.zero(), communityContext.communityId)) + this.accountBalances.add(new AccountBalance(publicKey, GradidoUnit.zero(), coinCommunityId)) return this.accountBalances } else { - const lastBalance = this.getLastBalanceForUser(publicKey, communityContext.blockchain, communityContext.communityId) + const lastBalance = this.getLastBalanceForUser(publicKey, communityContext.blockchain, coinCommunityId) try { lastBalance.updateLegacyDecay(amount, item.balanceDate) @@ -171,7 +172,7 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole outboundTransaction, senderBlockchain, ledgerAnchor, - this.calculateBalances(item, senderCommunityContext, item.amount.negated(), senderPublicKey), + this.calculateBalances(item, senderCommunityContext, senderCommunityContext.communityId, item.amount.negated(), senderPublicKey), ) } catch(e) { if (e instanceof NotEnoughGradidoBalanceError) { @@ -186,7 +187,7 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole inboundTransaction, recipientBlockchain, ledgerAnchor, - this.calculateBalances(item, recipientCommunityContext, item.amount, recipientPublicKey), + this.calculateBalances(item, recipientCommunityContext, senderCommunityContext.communityId, item.amount, recipientPublicKey), ) } catch(e) { if (e instanceof NotEnoughGradidoBalanceError) { From b023fd98b642817888dcb6038a2f2973aab3d360 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 18 Feb 2026 15:23:24 +0100 Subject: [PATCH 25/50] fix balance for cross group transactions --- .../syncDbWithBlockchain/AbstractSync.role.ts | 3 ++- .../RemoteTransactionsSync.role.ts | 12 ++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts index a5cfb710c..d6a6fc23a 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts @@ -51,7 +51,7 @@ export abstract class AbstractSyncRole { throw new Error('publicKey is empty') } const f = Filter.lastBalanceFor(publicKey) - f.coinCommunityId = communityId + f.setCommunityId(communityId) const lastSenderTransaction = blockchain.findOne(f) if (!lastSenderTransaction) { return new Balance(publicKey, communityId) @@ -59,6 +59,7 @@ export abstract class AbstractSyncRole { const lastConfirmedTransaction = lastSenderTransaction.getConfirmedTransaction() if (!lastConfirmedTransaction) { throw new Error(`invalid transaction, getConfirmedTransaction call failed for transaction nr: ${lastSenderTransaction.getTransactionNr()}`) + } const senderLastAccountBalance = lastConfirmedTransaction.getAccountBalance(publicKey, communityId) if (!senderLastAccountBalance) { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts index 567a7ee79..19a33189b 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts @@ -114,13 +114,19 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole this.accountBalances.add(new AccountBalance(publicKey, GradidoUnit.zero(), coinCommunityId)) return this.accountBalances } else { - const lastBalance = this.getLastBalanceForUser(publicKey, communityContext.blockchain, coinCommunityId) + // try to use same coins from this community + let lastBalance = this.getLastBalanceForUser(publicKey, communityContext.blockchain, coinCommunityId) + if (lastBalance.getBalance().equal(GradidoUnit.zero()) || lastBalance.getBalance().calculateDecay(lastBalance.getDate(), item.balanceDate).lt(amount)) { + // don't work, so we use or own coins + lastBalance = this.getLastBalanceForUser(publicKey, communityContext.blockchain, communityContext.communityId) + } try { lastBalance.updateLegacyDecay(amount, item.balanceDate) } catch(e) { if (e instanceof NegativeBalanceError) { - this.logLastBalanceChangingTransactions(publicKey, communityContext.blockchain, 10) + console.log(`coin community id: ${coinCommunityId}, context community id: ${communityContext.communityId}`) + this.logLastBalanceChangingTransactions(publicKey, communityContext.blockchain, 1) throw e } } @@ -195,5 +201,7 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole } throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) } + // this.logLastBalanceChangingTransactions(senderPublicKey, senderCommunityContext.blockchain, 1) + // this.logLastBalanceChangingTransactions(recipientPublicKey, recipientCommunityContext.blockchain, 1) } } From 14f7e91fdc16412e83092c71763e853701e0019c Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 25 Feb 2026 12:12:27 +0100 Subject: [PATCH 26/50] add call for getting authorized communities --- .../src/graphql/resolver/CommunityResolver.ts | 13 ++++++++++++- database/src/queries/communities.ts | 16 +++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/backend/src/graphql/resolver/CommunityResolver.ts b/backend/src/graphql/resolver/CommunityResolver.ts index 0209c284d..d037fffb4 100644 --- a/backend/src/graphql/resolver/CommunityResolver.ts +++ b/backend/src/graphql/resolver/CommunityResolver.ts @@ -2,7 +2,7 @@ import { Paginated } from '@arg/Paginated' import { EditCommunityInput } from '@input/EditCommunityInput' import { AdminCommunityView } from '@model/AdminCommunityView' import { Community } from '@model/Community' -import { Community as DbCommunity, getHomeCommunity, getReachableCommunities } from 'database' +import { Community as DbCommunity, getAuthorizedCommunities, getHomeCommunity, getReachableCommunities } from 'database' import { updateAllDefinedAndChanged } from 'shared' import { Arg, Args, Authorized, Mutation, Query, Resolver } from 'type-graphql' import { RIGHTS } from '@/auth/RIGHTS' @@ -34,6 +34,17 @@ export class CommunityResolver { return dbCommunities.map((dbCom: DbCommunity) => new Community(dbCom)) } + @Authorized([RIGHTS.COMMUNITIES]) + @Query(() => [Community]) + async authorizedCommunities(): Promise { + const dbCommunities: DbCommunity[] = await getAuthorizedCommunities({ + // order by + foreign: 'ASC', // home community first + name: 'ASC', + }) + return dbCommunities.map((dbCom: DbCommunity) => new Community(dbCom)) + } + @Authorized([RIGHTS.COMMUNITIES]) @Query(() => Community) async communityByIdentifier( diff --git a/database/src/queries/communities.ts b/database/src/queries/communities.ts index 83e8933d0..37fba27ff 100644 --- a/database/src/queries/communities.ts +++ b/database/src/queries/communities.ts @@ -84,7 +84,7 @@ export async function getReachableCommunities( federatedCommunities: { verifiedAt: MoreThanOrEqual(new Date(Date.now() - authenticationTimeoutMs)), }, - }, + }, // or { foreign: false }, ], order, @@ -99,3 +99,17 @@ export async function getNotReachableCommunities( order, }) } + +// return the home community and all communities which had at least once make it through the first handshake +export async function getAuthorizedCommunities( + order?: FindOptionsOrder, +): Promise +{ + return await DbCommunity.find({ + where: [ + { authenticatedAt: Not(IsNull()) }, // or + { foreign: false } + ], + order + }) +} \ No newline at end of file From 3ef4c25ae8f51682ef56c4fa372f44c85df510f8 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 25 Feb 2026 12:12:58 +0100 Subject: [PATCH 27/50] adapt to GradidoNode Changes --- dlt-connector/src/bootstrap/init.ts | 2 +- .../src/client/GradidoNode/communities.ts | 21 +++++---------- .../src/client/GradidoNode/input.schema.ts | 6 ++--- .../src/client/backend/BackendClient.ts | 14 ++++++++++ dlt-connector/src/client/backend/graphql.ts | 11 ++++++++ dlt-connector/src/config/const.ts | 2 +- .../db-v2.7.0_to_blockchain-v3.7/bootstrap.ts | 27 ++++++++----------- .../db-v2.7.0_to_blockchain-v3.7/database.ts | 7 ----- dlt-connector/src/utils/filesystem.ts | 4 +++ 9 files changed, 52 insertions(+), 42 deletions(-) diff --git a/dlt-connector/src/bootstrap/init.ts b/dlt-connector/src/bootstrap/init.ts index 13b77783c..a1c599cea 100644 --- a/dlt-connector/src/bootstrap/init.ts +++ b/dlt-connector/src/bootstrap/init.ts @@ -83,7 +83,7 @@ export async function checkGradidoNode( if ( !(await clients.gradidoNode.getTransaction({ transactionId: 1, - topic: homeCommunity.hieroTopicId, + communityId: homeCommunity.uuid, })) ) { // if not exist, create community root transaction diff --git a/dlt-connector/src/client/GradidoNode/communities.ts b/dlt-connector/src/client/GradidoNode/communities.ts index a418255f7..b285eed92 100644 --- a/dlt-connector/src/client/GradidoNode/communities.ts +++ b/dlt-connector/src/client/GradidoNode/communities.ts @@ -5,7 +5,7 @@ import { getLogger } from 'log4js' import { CONFIG } from '../../config' import { GRADIDO_NODE_HOME_FOLDER_NAME, LOG4JS_BASE_CATEGORY } from '../../config/const' import { HieroId } from '../../schemas/typeGuard.schema' -import { checkFileExist, checkPathExist } from '../../utils/filesystem' +import { checkFileExist, checkPathExist, toFolderName } from '../../utils/filesystem' import { BackendClient } from '../backend/BackendClient' import { GradidoNodeProcess } from './GradidoNodeProcess' @@ -15,7 +15,7 @@ const ensureCommunitiesAvailableMutex: Mutex = new Mutex() // prototype, later add api call to gradido dlt node server for adding/updating communities type CommunityForDltNodeServer = { communityId: string - hieroTopicId: string + hieroTopicId?: string | null alias: string folder: string } @@ -38,28 +38,21 @@ export async function ensureCommunitiesAvailable(communityTopicIds: HieroId[]): } export async function exportCommunities(homeFolder: string, client: BackendClient): Promise { - const communities = await client.getReachableCommunities() + const communities = await client.getAuthorizedCommunities() + console.log(`communities: ${JSON.stringify(communities, null, 2)}`) const communitiesPath = path.join(homeFolder, 'communities.json') checkPathExist(path.dirname(communitiesPath), true) - // make sure communityName is unique - const communityName = new Set() const communitiesForDltNodeServer: CommunityForDltNodeServer[] = [] for (const com of communities) { - if (!com.uuid || !com.hieroTopicId) { + if (!com.uuid) { continue } - // use name as alias if not empty and unique, otherwise use uuid - let alias = com.name - if (!alias || communityName.has(alias)) { - alias = com.uuid - } - communityName.add(alias) communitiesForDltNodeServer.push({ communityId: com.uuid, hieroTopicId: com.hieroTopicId, - alias, + alias: com.name, // use only alpha-numeric chars for folder name - folder: alias.replace(/[^a-zA-Z0-9]/g, '_'), + folder: toFolderName(com.uuid), }) } fs.writeFileSync(communitiesPath, JSON.stringify(communitiesForDltNodeServer, null, 2)) diff --git a/dlt-connector/src/client/GradidoNode/input.schema.ts b/dlt-connector/src/client/GradidoNode/input.schema.ts index 16a8643f5..f8a21f562 100644 --- a/dlt-connector/src/client/GradidoNode/input.schema.ts +++ b/dlt-connector/src/client/GradidoNode/input.schema.ts @@ -1,12 +1,12 @@ import * as v from 'valibot' -import { hieroIdSchema, hieroTransactionIdStringSchema } from '../../schemas/typeGuard.schema' +import { uuidv4Schema, hieroTransactionIdStringSchema } from '../../schemas/typeGuard.schema' export const transactionsRangeSchema = v.object({ // default value is 1, from first transactions fromTransactionId: v.nullish(v.pipe(v.number(), v.minValue(1, 'expect number >= 1')), 1), // default value is 100, max 100 transactions maxResultCount: v.nullish(v.pipe(v.number(), v.minValue(1, 'expect number >= 1')), 100), - topic: hieroIdSchema, + communityId: uuidv4Schema, }) export type TransactionsRangeInput = v.InferInput @@ -19,7 +19,7 @@ export const transactionIdentifierSchema = v.pipe( undefined, ), hieroTransactionId: v.nullish(hieroTransactionIdStringSchema, undefined), - topic: hieroIdSchema, + communityId: uuidv4Schema, }), v.custom((value: any) => { const setFieldsCount = diff --git a/dlt-connector/src/client/backend/BackendClient.ts b/dlt-connector/src/client/backend/BackendClient.ts index ce9660e36..7829e0e25 100644 --- a/dlt-connector/src/client/backend/BackendClient.ts +++ b/dlt-connector/src/client/backend/BackendClient.ts @@ -6,6 +6,7 @@ import { CONFIG } from '../../config' import { LOG4JS_BASE_CATEGORY } from '../../config/const' import { HieroId, Uuidv4 } from '../../schemas/typeGuard.schema' import { + getAuthorizedCommunities, getReachableCommunities, homeCommunityGraphqlQuery, setHomeCommunityTopicId, @@ -101,6 +102,19 @@ export class BackendClient { return v.parse(v.array(communitySchema), data.reachableCommunities) } + public async getAuthorizedCommunities(): Promise { + this.logger.info('get authorized communities on backend') + const { data, errors } = await this.client.rawRequest<{ authorizedCommunities: Community[] }>( + getAuthorizedCommunities, + {}, + await this.getRequestHeader(), + ) + if (errors) { + throw errors[0] + } + return v.parse(v.array(communitySchema), data.authorizedCommunities) + } + private async getRequestHeader(): Promise<{ authorization: string }> { diff --git a/dlt-connector/src/client/backend/graphql.ts b/dlt-connector/src/client/backend/graphql.ts index fafd77fcd..2f1fbec74 100644 --- a/dlt-connector/src/client/backend/graphql.ts +++ b/dlt-connector/src/client/backend/graphql.ts @@ -44,3 +44,14 @@ export const getReachableCommunities = gql` } ${communityFragment} ` + +export const getAuthorizedCommunities = gql` + query { + authorizedCommunities { + ...Community_common + } + } + ${communityFragment} +` + + diff --git a/dlt-connector/src/config/const.ts b/dlt-connector/src/config/const.ts index 5657f65ee..5dcd3d293 100644 --- a/dlt-connector/src/config/const.ts +++ b/dlt-connector/src/config/const.ts @@ -17,6 +17,6 @@ export const GRADIDO_NODE_RUNTIME_PATH = path.join( // if last start was less than this time, do not restart export const GRADIDO_NODE_MIN_RUNTIME_BEFORE_RESTART_MILLISECONDS = 1000 * 30 export const GRADIDO_NODE_MIN_RUNTIME_BEFORE_EXIT_MILLISECONDS = 1000 * 2 -export const GRADIDO_NODE_KILL_TIMEOUT_MILLISECONDS = 10000 +export const GRADIDO_NODE_KILL_TIMEOUT_MILLISECONDS = 60 * 1000 // currently hard coded in gradido node, update in future export const GRADIDO_NODE_HOME_FOLDER_NAME = '.gradido' diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts index 3ffad58c8..3ccb24f1d 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts @@ -10,6 +10,7 @@ import { Context } from './Context' import { Balance } from './data/Balance' import { loadAdminUsersCache, loadCommunities, loadContributionLinkModeratorCache } from './database' import { CommunityContext } from './valibot.schema' +import { toFolderName } from '../../utils/filesystem' export async function bootstrap(): Promise { const context = await Context.create() @@ -24,20 +25,14 @@ export async function bootstrap(): Promise { async function bootstrapCommunities(context: Context): Promise> { const communities = new Map() const communitiesDb = await loadCommunities(context.db) - const communityNames = new Set() for (const communityDb of communitiesDb) { - let alias = communityDb.name.toLowerCase() - if (communityNames.has(alias)) { - alias = communityDb.communityUuid - } else { - communityNames.add(alias) - } - const blockchain = InMemoryBlockchainProvider.getInstance().getBlockchain(alias) + const communityId = communityDb.communityUuid + const blockchain = InMemoryBlockchainProvider.getInstance().getBlockchain(communityId) if (!blockchain) { - throw new Error(`Couldn't create Blockchain for community ${alias}`) + throw new Error(`Couldn't create Blockchain for community ${communityId}`) } - context.logger.info(`Blockchain for community '${alias}' created`) + context.logger.info(`Blockchain for community '${communityId}' created`) let seed: Hex32 if (!communityDb.foreign) { seed = v.parse(hex32Schema, CONFIG.HOME_COMMUNITY_SEED.convertToHex()) @@ -59,7 +54,7 @@ async function bootstrapCommunities(context: Context): Promise() export const adminUsers = new Map() -const transactionIdSet = new Set() export async function loadContributionLinkModeratorCache(db: MySql2Database): Promise { const result = await db diff --git a/dlt-connector/src/utils/filesystem.ts b/dlt-connector/src/utils/filesystem.ts index ea2d64e39..f01008a93 100644 --- a/dlt-connector/src/utils/filesystem.ts +++ b/dlt-connector/src/utils/filesystem.ts @@ -28,3 +28,7 @@ export function checkPathExist(path: string, createIfMissing: boolean = false): } return false } + +export function toFolderName(name: string): string { + return name.toLowerCase().replace(/[^a-z0-9]/g, '_') +} \ No newline at end of file From f45705e714625eba9741ad75f69c08fa61b51f7e Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 25 Feb 2026 16:04:25 +0100 Subject: [PATCH 28/50] give communitId with txs for blockchain --- .../apis/dltConnector/model/AccountIdentifier.ts | 4 +++- .../apis/dltConnector/model/TransactionDraft.ts | 15 +++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/backend/src/apis/dltConnector/model/AccountIdentifier.ts b/backend/src/apis/dltConnector/model/AccountIdentifier.ts index 3882e8584..1d324a70f 100644 --- a/backend/src/apis/dltConnector/model/AccountIdentifier.ts +++ b/backend/src/apis/dltConnector/model/AccountIdentifier.ts @@ -1,15 +1,17 @@ import { CommunityAccountIdentifier } from './CommunityAccountIdentifier' export class AccountIdentifier { communityTopicId: string + communityId: string account?: CommunityAccountIdentifier seed?: string // used for deferred transfers - constructor(communityTopicId: string, input: CommunityAccountIdentifier | string) { + constructor(communityTopicId: string, communityUuid: string, input: CommunityAccountIdentifier | string) { if (input instanceof CommunityAccountIdentifier) { this.account = input } else { this.seed = input } this.communityTopicId = communityTopicId + this.communityId = communityUuid } } diff --git a/backend/src/apis/dltConnector/model/TransactionDraft.ts b/backend/src/apis/dltConnector/model/TransactionDraft.ts index 1727c7fd8..5352bf841 100755 --- a/backend/src/apis/dltConnector/model/TransactionDraft.ts +++ b/backend/src/apis/dltConnector/model/TransactionDraft.ts @@ -36,6 +36,7 @@ export class TransactionDraft { const draft = new TransactionDraft() draft.user = new AccountIdentifier( community.hieroTopicId, + community.communityUuid!, new CommunityAccountIdentifier(user.gradidoID), ) draft.type = TransactionType.REGISTER_ADDRESS @@ -58,10 +59,12 @@ export class TransactionDraft { const draft = new TransactionDraft() draft.user = new AccountIdentifier( community.hieroTopicId, + community.communityUuid!, new CommunityAccountIdentifier(contribution.user.gradidoID), ) draft.linkedUser = new AccountIdentifier( community.hieroTopicId, + community.communityUuid!, new CommunityAccountIdentifier(signingUser.gradidoID), ) draft.type = TransactionType.GRADIDO_CREATION @@ -96,10 +99,12 @@ export class TransactionDraft { const draft = new TransactionDraft() draft.user = new AccountIdentifier( senderUserTopic, + sendingUser.community.communityUuid!, new CommunityAccountIdentifier(sendingUser.gradidoID), ) draft.linkedUser = new AccountIdentifier( receiverUserTopic, + receivingUser.community.communityUuid!, new CommunityAccountIdentifier(receivingUser.gradidoID), ) draft.type = TransactionType.GRADIDO_TRANSFER @@ -125,9 +130,10 @@ export class TransactionDraft { const draft = new TransactionDraft() draft.user = new AccountIdentifier( senderUserTopic, + sendingUser.community.communityUuid!, new CommunityAccountIdentifier(sendingUser.gradidoID), ) - draft.linkedUser = new AccountIdentifier(senderUserTopic, transactionLink.code) + draft.linkedUser = new AccountIdentifier(senderUserTopic, sendingUser.community.communityUuid!, transactionLink.code) draft.type = TransactionType.GRADIDO_DEFERRED_TRANSFER draft.createdAt = createdAtOnlySeconds.toISOString() draft.amount = transactionLink.amount.toString() @@ -159,9 +165,14 @@ export class TransactionDraft { const createdAtOnlySeconds = createdAt createdAtOnlySeconds.setMilliseconds(0) const draft = new TransactionDraft() - draft.user = new AccountIdentifier(senderUserTopic, transactionLink.code) + draft.user = new AccountIdentifier( + senderUserTopic, + transactionLink.user.community.communityUuid!, + transactionLink.code, + ) draft.linkedUser = new AccountIdentifier( recipientUserTopic, + recipientUser.community.communityUuid!, new CommunityAccountIdentifier(recipientUser.gradidoID), ) draft.type = TransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER From 10334f5a46c38c8dd601e63b95948c7dfc54351d Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 25 Feb 2026 16:05:14 +0100 Subject: [PATCH 29/50] add option, fix type errors because of changes regarding need of communityId on some places --- dlt-connector/src/bootstrap/init.ts | 3 +- .../src/bootstrap/initGradidoNode.ts | 3 +- .../client/GradidoNode/GradidoNodeClient.ts | 32 +++++++++---------- .../client/GradidoNode/GradidoNodeProcess.ts | 16 ++++++++++ .../client/GradidoNode/input.schema.test.ts | 22 +++++++------ dlt-connector/src/client/hiero/HieroClient.ts | 5 +-- dlt-connector/src/config/schema.ts | 7 ++++ .../src/data/KeyPairIdentifier.logic.ts | 7 ++-- .../AbstractRemoteKeyPair.role.ts | 8 ++--- .../ForeignCommunityKeyPair.role.ts | 2 +- .../RemoteAccountKeyPair.role.ts | 4 +-- .../resolveKeyPair/ResolveKeyPair.context.ts | 2 +- .../CommunityRootTransaction.role.ts | 5 ++- .../sendToHiero/CreationTransaction.role.ts | 7 +++- .../DeferredTransferTransaction.role.ts | 2 ++ .../RedeemDeferredTransferTransaction.role.ts | 1 + .../RegisterAddressTransaction.role.ts | 5 ++- .../sendToHiero/SendToHiero.context.ts | 15 +++++++-- .../sendToHiero/TransferTransaction.role.ts | 6 +++- .../data/keyPair.ts | 4 ++- dlt-connector/src/schemas/account.schema.ts | 1 + .../src/schemas/transaction.schema.test.ts | 19 +++++++++++ .../src/schemas/transaction.schema.ts | 2 ++ .../src/schemas/typeConverter.schema.ts | 21 ++++++++---- dlt-connector/src/server/index.ts | 12 ++++--- dlt-connector/src/server/input.schema.ts | 2 ++ dlt-connector/src/utils/filesystem.ts | 2 +- dlt-connector/src/utils/typeConverter.ts | 5 +-- 28 files changed, 156 insertions(+), 64 deletions(-) diff --git a/dlt-connector/src/bootstrap/init.ts b/dlt-connector/src/bootstrap/init.ts index a1c599cea..97a3a03ef 100644 --- a/dlt-connector/src/bootstrap/init.ts +++ b/dlt-connector/src/bootstrap/init.ts @@ -1,5 +1,5 @@ import { readFileSync } from 'node:fs' -import { loadCryptoKeys, MemoryBlock } from 'gradido-blockchain-js' +import { InMemoryBlockchainProvider, loadCryptoKeys, MemoryBlock } from 'gradido-blockchain-js' import { configure, getLogger, Logger } from 'log4js' import * as v from 'valibot' import { CONFIG } from '../config' @@ -80,6 +80,7 @@ export async function checkGradidoNode( // ask gradido node if community blockchain was created try { + InMemoryBlockchainProvider.getInstance().getBlockchain(homeCommunity.uuid) if ( !(await clients.gradidoNode.getTransaction({ transactionId: 1, diff --git a/dlt-connector/src/bootstrap/initGradidoNode.ts b/dlt-connector/src/bootstrap/initGradidoNode.ts index 7c652a667..dfc8d0c10 100644 --- a/dlt-connector/src/bootstrap/initGradidoNode.ts +++ b/dlt-connector/src/bootstrap/initGradidoNode.ts @@ -9,7 +9,6 @@ import { HieroClient } from '../client/hiero/HieroClient' import { CONFIG } from '../config' import { GRADIDO_NODE_HOME_FOLDER_NAME, - GRADIDO_NODE_RUNTIME_PATH, LOG4JS_BASE_CATEGORY, } from '../config/const' import { checkFileExist, checkPathExist } from '../utils/filesystem' @@ -37,7 +36,7 @@ export async function initGradidoNode(clients: AppContextClients): Promise // write Hedera Address Book exportHederaAddressbooks(gradidoNodeHomeFolder, clients.hiero), // check GradidoNode Runtime, download when missing - ensureGradidoNodeRuntimeAvailable(GRADIDO_NODE_RUNTIME_PATH), + ensureGradidoNodeRuntimeAvailable(GradidoNodeProcess.getRuntimePathFileName()), // export communities to GradidoNode Folder exportCommunities(gradidoNodeHomeFolder, clients.backend), ]) diff --git a/dlt-connector/src/client/GradidoNode/GradidoNodeClient.ts b/dlt-connector/src/client/GradidoNode/GradidoNodeClient.ts index d66a58887..0a3d0c7ae 100644 --- a/dlt-connector/src/client/GradidoNode/GradidoNodeClient.ts +++ b/dlt-connector/src/client/GradidoNode/GradidoNodeClient.ts @@ -8,7 +8,7 @@ import { LOG4JS_BASE_CATEGORY } from '../../config/const' import { AddressType } from '../../data/AddressType.enum' import { Uuidv4Hash } from '../../data/Uuidv4Hash' import { addressTypeSchema, confirmedTransactionSchema } from '../../schemas/typeConverter.schema' -import { Hex32, Hex32Input, HieroId, hex32Schema } from '../../schemas/typeGuard.schema' +import { Hex32, Hex32Input, Uuidv4, hex32Schema } from '../../schemas/typeGuard.schema' import { isPortOpenRetry } from '../../utils/network' import { GradidoNodeErrorCodes } from './GradidoNodeErrorCodes' import { @@ -75,7 +75,7 @@ export class GradidoNodeClient { const response = await this.rpcCall<{ transaction: string }>('getTransaction', parameter) if (response.isSuccess()) { // this.logger.debug('result: ', response.result.transaction) - return v.parse(confirmedTransactionSchema, response.result.transaction) + return v.parse(confirmedTransactionSchema, { base64: response.result.transaction, communityId: parameter.communityId }) } if (response.isError()) { if (response.error.code === GradidoNodeErrorCodes.TRANSACTION_NOT_FOUND) { @@ -88,19 +88,19 @@ export class GradidoNodeClient { /** * getLastTransaction * get the last confirmed transaction from a specific community - * @param hieroTopic the community hiero topic id + * @param communityId the community id * @returns the last confirmed transaction or undefined if blockchain for community is empty or not found * @throws GradidoNodeRequestError */ - public async getLastTransaction(hieroTopic: HieroId): Promise { + public async getLastTransaction(communityId: Uuidv4): Promise { const parameter = { format: 'base64', - topic: hieroTopic, + communityId, } const response = await this.rpcCall<{ transaction: string }>('getLastTransaction', parameter) if (response.isSuccess()) { - return v.parse(confirmedTransactionSchema, response.result.transaction) + return v.parse(confirmedTransactionSchema, { base64: response.result.transaction, communityId: parameter.communityId }) } if (response.isError()) { if (response.error.code === GradidoNodeErrorCodes.GRADIDO_NODE_ERROR) { @@ -115,7 +115,7 @@ export class GradidoNodeClient { * get list of confirmed transactions from a specific community * @param input fromTransactionId is the id of the first transaction to return * @param input maxResultCount is the max number of transactions to return - * @param input topic is the community hiero topic id + * @param input communityId is the community id * @returns list of confirmed transactions * @throws GradidoNodeRequestError * @example @@ -123,7 +123,7 @@ export class GradidoNodeClient { * const transactions = await getTransactions({ * fromTransactionId: 1, * maxResultCount: 100, - * topic: communityUuid, + * communityId: communityUuid, * }) * ``` */ @@ -137,7 +137,7 @@ export class GradidoNodeClient { parameter, ) return result.transactions.map((transactionBase64) => - v.parse(confirmedTransactionSchema, transactionBase64), + v.parse(confirmedTransactionSchema, { base64: transactionBase64, communityId: parameter.communityId }), ) } @@ -163,7 +163,7 @@ export class GradidoNodeClient { parameter, ) return response.transactions.map((transactionBase64) => - v.parse(confirmedTransactionSchema, transactionBase64), + v.parse(confirmedTransactionSchema, { base64: transactionBase64, communityId: parameter.communityId }), ) } @@ -173,15 +173,15 @@ export class GradidoNodeClient { * can be used to check if user/account exists on blockchain * look also for gmw, auf and deferred transfer accounts * @param pubkey the public key of the user or account - * @param hieroTopic the community hiero topic id + * @param communityId the community id * @returns the address type of the user/account, AddressType.NONE if not found * @throws GradidoNodeRequestError */ - public async getAddressType(pubkey: Hex32Input, hieroTopic: HieroId): Promise { + public async getAddressType(pubkey: Hex32Input, communityId: Uuidv4): Promise { const parameter = { pubkey: v.parse(hex32Schema, pubkey), - topic: hieroTopic, + communityId, } const response = await this.rpcCallResolved<{ addressType: string }>( 'getAddressType', @@ -194,17 +194,17 @@ export class GradidoNodeClient { * findUserByNameHash * find a user by name hash * @param nameHash the name hash of the user - * @param hieroTopic the community hiero topic id + * @param communityId the community id * @returns the public key of the user as hex32 string or undefined if user is not found * @throws GradidoNodeRequestError */ public async findUserByNameHash( nameHash: Uuidv4Hash, - hieroTopic: HieroId, + communityId: Uuidv4, ): Promise { const parameter = { nameHash: nameHash.getAsHexString(), - topic: hieroTopic, + communityId, } const response = await this.rpcCall<{ pubkey: string; timeUsed: string }>( 'findUserByNameHash', diff --git a/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts b/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts index 0eae2f37e..526fd99be 100644 --- a/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts +++ b/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts @@ -10,6 +10,7 @@ import { LOG4JS_BASE_CATEGORY, } from '../../config/const' import { delay } from '../../utils/time' +import path from 'node:path' /** * A Singleton class defines the `getInstance` method that lets clients access * the unique singleton instance. @@ -43,6 +44,20 @@ export class GradidoNodeProcess { return GradidoNodeProcess.instance } + public static getRuntimePathFileName(): string { + const isWindows = process.platform === 'win32' + const binaryName = isWindows ? 'GradidoNode.exe' : 'GradidoNode' + + return path.join( + __dirname, + '..', + '..', + 'gradido_node', + 'bin', + binaryName, + ) + } + public start() { if (this.proc) { this.logger.warn('GradidoNodeProcess already running.') @@ -57,6 +72,7 @@ export class GradidoNodeProcess { SERVER_JSON_RPC_PORT: CONFIG.DLT_NODE_SERVER_PORT.toString(), USERPROFILE: CONFIG.DLT_GRADIDO_NODE_SERVER_HOME_FOLDER, HOME: CONFIG.DLT_GRADIDO_NODE_SERVER_HOME_FOLDER, + UNSECURE_ALLOW_CORS_ALL: CONFIG.DLT_GRADIDO_NODE_SERVER_ALLOW_CORS ? '1' : '0', }, onExit(_proc, exitCode, signalCode, error) { logger.warn(`GradidoNodeProcess exited with code ${exitCode} and signalCode ${signalCode}`) diff --git a/dlt-connector/src/client/GradidoNode/input.schema.test.ts b/dlt-connector/src/client/GradidoNode/input.schema.test.ts index fbe63dabb..550437100 100644 --- a/dlt-connector/src/client/GradidoNode/input.schema.test.ts +++ b/dlt-connector/src/client/GradidoNode/input.schema.test.ts @@ -1,18 +1,20 @@ import { beforeAll, describe, expect, it } from 'bun:test' import * as v from 'valibot' import { - HieroId, HieroTransactionIdString, + Uuidv4, hieroIdSchema, hieroTransactionIdStringSchema, + uuidv4Schema, } from '../../schemas/typeGuard.schema' import { transactionIdentifierSchema } from './input.schema' +import { v4 as uuidv4 } from 'uuid' -let topic: HieroId -const topicString = '0.0.261' +let communityId: Uuidv4 +const uuidv4String = uuidv4() let hieroTransactionId: HieroTransactionIdString beforeAll(() => { - topic = v.parse(hieroIdSchema, topicString) + communityId = v.parse(uuidv4Schema, uuidv4String) hieroTransactionId = v.parse(hieroTransactionIdStringSchema, '0.0.261-1755348116-1281621') }) @@ -21,26 +23,26 @@ describe('transactionIdentifierSchema ', () => { expect( v.parse(transactionIdentifierSchema, { transactionId: 1, - topic: topicString, + communityId, }), ).toEqual({ transactionId: 1, hieroTransactionId: undefined, - topic, + communityId, }) }) it('valid, transaction identified by hieroTransactionId and topic', () => { expect( v.parse(transactionIdentifierSchema, { hieroTransactionId: '0.0.261-1755348116-1281621', - topic: topicString, + communityId, }), ).toEqual({ hieroTransactionId, - topic, + communityId, }) }) - it('invalid, missing topic', () => { + it('invalid, missing communityId', () => { expect(() => v.parse(transactionIdentifierSchema, { transactionId: 1, @@ -53,7 +55,7 @@ describe('transactionIdentifierSchema ', () => { v.parse(transactionIdentifierSchema, { transactionId: 1, hieroTransactionId: '0.0.261-1755348116-1281621', - topic, + communityId, }), ).toThrowError(new Error('expect transactionNr or hieroTransactionId not both')) }) diff --git a/dlt-connector/src/client/hiero/HieroClient.ts b/dlt-connector/src/client/hiero/HieroClient.ts index c32c4cbd7..f82139854 100644 --- a/dlt-connector/src/client/hiero/HieroClient.ts +++ b/dlt-connector/src/client/hiero/HieroClient.ts @@ -20,7 +20,7 @@ import { getLogger, Logger } from 'log4js' import * as v from 'valibot' import { CONFIG } from '../../config' import { LOG4JS_BASE_CATEGORY } from '../../config/const' -import { HieroId, hieroIdSchema } from '../../schemas/typeGuard.schema' +import { HieroId, hieroIdSchema, Uuidv4 } from '../../schemas/typeGuard.schema' import { durationInMinutesFromDates, printTimeDuration } from '../../utils/time' import { GradidoNodeClient } from '../GradidoNode/GradidoNodeClient' import { GradidoNodeProcess } from '../GradidoNode/GradidoNodeProcess' @@ -72,6 +72,7 @@ export class HieroClient { public async sendMessage( topicId: HieroId, + communityId: Uuidv4, transaction: GradidoTransaction, ): Promise { const timeUsed = new Profiler() @@ -102,7 +103,7 @@ export class HieroClient { // after 10 seconds, else restart GradidoNode setTimeout(async () => { const transaction = await GradidoNodeClient.getInstance().getTransaction({ - topic: topicId, + communityId, hieroTransactionId: sendResponse.transactionId.toString(), }) if (!transaction) { diff --git a/dlt-connector/src/config/schema.ts b/dlt-connector/src/config/schema.ts index ef4558545..753da05c5 100644 --- a/dlt-connector/src/config/schema.ts +++ b/dlt-connector/src/config/schema.ts @@ -90,6 +90,13 @@ export const configSchema = v.object({ v.string('The home folder for the gradido dlt node server'), path.join(__dirname, '..', '..', 'gradido_node'), ), + DLT_GRADIDO_NODE_SERVER_ALLOW_CORS: v.optional( + v.pipe( + v.string('Whether to allow CORS for the gradido dlt node server'), + v.transform((input: string) => input === 'true'), + ), + 'false', + ), BACKEND_PORT: v.optional( v.pipe( v.string('A valid port on which the backend server is running'), diff --git a/dlt-connector/src/data/KeyPairIdentifier.logic.ts b/dlt-connector/src/data/KeyPairIdentifier.logic.ts index 3b6b71c6e..0ee541572 100644 --- a/dlt-connector/src/data/KeyPairIdentifier.logic.ts +++ b/dlt-connector/src/data/KeyPairIdentifier.logic.ts @@ -54,6 +54,9 @@ export class KeyPairIdentifierLogic { return this.identifier.seed } + getCommunityId(): Uuidv4 { + return this.identifier.communityId + } getCommunityTopicId(): HieroId { return this.identifier.communityTopicId } @@ -76,7 +79,7 @@ export class KeyPairIdentifierLogic { return this.getSeed() } getCommunityKey(): string { - return this.getCommunityTopicId() + return this.getCommunityId() } getCommunityUserKey(): string { return this.deriveCommunityUserHash(0) @@ -107,7 +110,7 @@ export class KeyPairIdentifierLogic { ) } const resultString = - this.identifier.communityTopicId + + this.identifier.communityId + this.identifier.account.userUuid.replace(/-/g, '') + accountNr.toString() return new MemoryBlock(resultString).calculateHash().convertToHex() diff --git a/dlt-connector/src/interactions/resolveKeyPair/AbstractRemoteKeyPair.role.ts b/dlt-connector/src/interactions/resolveKeyPair/AbstractRemoteKeyPair.role.ts index 30b824d86..5b99a6c19 100644 --- a/dlt-connector/src/interactions/resolveKeyPair/AbstractRemoteKeyPair.role.ts +++ b/dlt-connector/src/interactions/resolveKeyPair/AbstractRemoteKeyPair.role.ts @@ -1,10 +1,10 @@ import { KeyPairEd25519 } from 'gradido-blockchain-js' -import { HieroId } from '../../schemas/typeGuard.schema' +import { Uuidv4 } from '../../schemas/typeGuard.schema' export abstract class AbstractRemoteKeyPairRole { - protected topic: HieroId - public constructor(communityTopicId: HieroId) { - this.topic = communityTopicId + protected communityId: Uuidv4 + public constructor(communityId: Uuidv4) { + this.communityId = communityId } public abstract retrieveKeyPair(): Promise } diff --git a/dlt-connector/src/interactions/resolveKeyPair/ForeignCommunityKeyPair.role.ts b/dlt-connector/src/interactions/resolveKeyPair/ForeignCommunityKeyPair.role.ts index 8911b8851..88db253a7 100644 --- a/dlt-connector/src/interactions/resolveKeyPair/ForeignCommunityKeyPair.role.ts +++ b/dlt-connector/src/interactions/resolveKeyPair/ForeignCommunityKeyPair.role.ts @@ -10,7 +10,7 @@ export class ForeignCommunityKeyPairRole extends AbstractRemoteKeyPairRole { public async retrieveKeyPair(): Promise { const transactionIdentifier = { transactionId: 1, - topic: this.topic, + communityId: this.communityId, } const firstTransaction = await GradidoNodeClient.getInstance().getTransaction(transactionIdentifier) diff --git a/dlt-connector/src/interactions/resolveKeyPair/RemoteAccountKeyPair.role.ts b/dlt-connector/src/interactions/resolveKeyPair/RemoteAccountKeyPair.role.ts index 524c3dc1a..43e1fa5f6 100644 --- a/dlt-connector/src/interactions/resolveKeyPair/RemoteAccountKeyPair.role.ts +++ b/dlt-connector/src/interactions/resolveKeyPair/RemoteAccountKeyPair.role.ts @@ -7,7 +7,7 @@ import { AbstractRemoteKeyPairRole } from './AbstractRemoteKeyPair.role' export class RemoteAccountKeyPairRole extends AbstractRemoteKeyPairRole { public constructor(private identifier: IdentifierAccount) { - super(identifier.communityTopicId) + super(identifier.communityId) } public async retrieveKeyPair(): Promise { @@ -17,7 +17,7 @@ export class RemoteAccountKeyPairRole extends AbstractRemoteKeyPairRole { const accountPublicKey = await GradidoNodeClient.getInstance().findUserByNameHash( new Uuidv4Hash(this.identifier.account.userUuid), - this.topic, + this.communityId, ) if (accountPublicKey) { return new KeyPairEd25519(MemoryBlock.createPtr(MemoryBlock.fromHex(accountPublicKey))) diff --git a/dlt-connector/src/interactions/resolveKeyPair/ResolveKeyPair.context.ts b/dlt-connector/src/interactions/resolveKeyPair/ResolveKeyPair.context.ts index 406463c4c..551997e9d 100644 --- a/dlt-connector/src/interactions/resolveKeyPair/ResolveKeyPair.context.ts +++ b/dlt-connector/src/interactions/resolveKeyPair/ResolveKeyPair.context.ts @@ -45,7 +45,7 @@ export async function ResolveKeyPair(input: KeyPairIdentifierLogic): Promise { const builder = new GradidoTransactionBuilder() const communityKeyPair = await ResolveKeyPair( - new KeyPairIdentifierLogic({ communityTopicId: this.community.hieroTopicId }), + new KeyPairIdentifierLogic({ + communityTopicId: this.community.hieroTopicId, + communityId: this.community.uuid, + }), ) const gmwKeyPair = communityKeyPair.deriveChild( hardenDerivationIndex(GMW_ACCOUNT_DERIVATION_INDEX), diff --git a/dlt-connector/src/interactions/sendToHiero/CreationTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/CreationTransaction.role.ts index 4f4d653a6..9bd7a7e8c 100644 --- a/dlt-connector/src/interactions/sendToHiero/CreationTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/CreationTransaction.role.ts @@ -52,6 +52,7 @@ export class CreationTransactionRole extends AbstractTransactionRole { const homeCommunityKeyPair = await ResolveKeyPair( new KeyPairIdentifierLogic({ communityTopicId: this.homeCommunityTopicId, + communityId: this.creationTransaction.user.communityId, }), ) // Memo: encrypted, home community and recipient can decrypt it @@ -65,7 +66,11 @@ export class CreationTransactionRole extends AbstractTransactionRole { ), ) .setTransactionCreation( - new TransferAmount(recipientKeyPair.getPublicKey(), this.creationTransaction.amount), + new TransferAmount( + recipientKeyPair.getPublicKey(), + this.creationTransaction.amount, + this.creationTransaction.user.communityId, + ), this.creationTransaction.targetDate, ) .sign(signerKeyPair) diff --git a/dlt-connector/src/interactions/sendToHiero/DeferredTransferTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/DeferredTransferTransaction.role.ts index 95459a8b6..3ac72c0ad 100644 --- a/dlt-connector/src/interactions/sendToHiero/DeferredTransferTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/DeferredTransferTransaction.role.ts @@ -41,6 +41,7 @@ export class DeferredTransferTransactionRole extends AbstractTransactionRole { const recipientKeyPair = await ResolveKeyPair( new KeyPairIdentifierLogic({ communityTopicId: this.deferredTransferTransaction.linkedUser.communityTopicId, + communityId: this.deferredTransferTransaction.linkedUser.communityId, seed: this.seed, }), ) @@ -61,6 +62,7 @@ export class DeferredTransferTransactionRole extends AbstractTransactionRole { this.deferredTransferTransaction.amount.calculateCompoundInterest( this.deferredTransferTransaction.timeoutDuration.getSeconds(), ), + this.deferredTransferTransaction.user.communityId, ), recipientKeyPair.getPublicKey(), ), diff --git a/dlt-connector/src/interactions/sendToHiero/RedeemDeferredTransferTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/RedeemDeferredTransferTransaction.role.ts index 4615a4707..0c0a3c64e 100644 --- a/dlt-connector/src/interactions/sendToHiero/RedeemDeferredTransferTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/RedeemDeferredTransferTransaction.role.ts @@ -65,6 +65,7 @@ export class RedeemDeferredTransferTransactionRole extends AbstractTransactionRo new TransferAmount( senderKeyPair.getPublicKey(), this.redeemDeferredTransferTransaction.amount, + this.redeemDeferredTransferTransaction.user.communityId, ), recipientKeyPair.getPublicKey(), ), diff --git a/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.ts index acb40975c..98ddc01f3 100644 --- a/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.ts @@ -35,7 +35,10 @@ export class RegisterAddressTransactionRole extends AbstractTransactionRole { public async getGradidoTransactionBuilder(): Promise { const builder = new GradidoTransactionBuilder() const communityTopicId = this.registerAddressTransaction.user.communityTopicId - const communityKeyPair = await ResolveKeyPair(new KeyPairIdentifierLogic({ communityTopicId })) + const communityKeyPair = await ResolveKeyPair(new KeyPairIdentifierLogic({ + communityTopicId, + communityId: this.registerAddressTransaction.user.communityId, + })) const keyPairIdentifier = this.registerAddressTransaction.user // when accountNr is 0 it is the user account keyPairIdentifier.account.accountNr = 0 diff --git a/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts b/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts index f96bfe402..9bb6f5c74 100644 --- a/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts +++ b/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts @@ -23,6 +23,8 @@ import { HieroTransactionIdString, hieroTransactionIdStringSchema, identifierSeedSchema, + Uuidv4, + uuidv4Schema, } from '../../schemas/typeGuard.schema' import { isTopicStillOpen } from '../../utils/hiero' import { LinkedTransactionKeyPairRole } from '../resolveKeyPair/LinkedTransactionKeyPair.role' @@ -59,6 +61,7 @@ export async function SendToHieroContext( const outboundHieroTransactionIdString = await sendViaHiero( outboundTransaction, role.getSenderCommunityTopicId(), + v.parse(uuidv4Schema, outboundTransaction.getCommunityId()), ) // attach Hiero transaction ID to the builder for the inbound transaction @@ -69,7 +72,11 @@ export async function SendToHieroContext( validate(inboundTransaction) // send inbound transaction to hiero - await sendViaHiero(inboundTransaction, role.getRecipientCommunityTopicId()) + await sendViaHiero( + inboundTransaction, + role.getRecipientCommunityTopicId(), + v.parse(uuidv4Schema, inboundTransaction.getCommunityId()), + ) return outboundHieroTransactionIdString } else { // build and validate local transaction @@ -80,6 +87,7 @@ export async function SendToHieroContext( const hieroTransactionIdString = await sendViaHiero( transaction, role.getSenderCommunityTopicId(), + v.parse(uuidv4Schema, transaction.getCommunityId()), ) return hieroTransactionIdString } @@ -95,9 +103,10 @@ function validate(transaction: GradidoTransaction): void { async function sendViaHiero( gradidoTransaction: GradidoTransaction, topic: HieroId, + communityId: Uuidv4 ): Promise { const client = HieroClient.getInstance() - const transactionId = await client.sendMessage(topic, gradidoTransaction) + const transactionId = await client.sendMessage(topic, communityId, gradidoTransaction) if (!transactionId) { throw new Error('missing transaction id from hiero') } @@ -153,7 +162,7 @@ async function chooseCorrectRole( throw new Error("redeem deferred transfer: couldn't generate seed public key") } const transactions = await GradidoNodeClient.getInstance().getTransactionsForAccount( - { maxResultCount: 2, topic: transaction.user.communityTopicId }, + { maxResultCount: 2, communityId: transaction.user.communityId }, seedPublicKey.convertToHex(), ) if (!transactions || transactions.length !== 1) { diff --git a/dlt-connector/src/interactions/sendToHiero/TransferTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/TransferTransaction.role.ts index f0a1314cb..fe9258d74 100644 --- a/dlt-connector/src/interactions/sendToHiero/TransferTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/TransferTransaction.role.ts @@ -51,7 +51,11 @@ export class TransferTransactionRole extends AbstractTransactionRole { ), ) .setTransactionTransfer( - new TransferAmount(senderKeyPair.getPublicKey(), this.transferTransaction.amount), + new TransferAmount( + senderKeyPair.getPublicKey(), + this.transferTransaction.amount, + this.transferTransaction.user.communityId, + ), recipientKeyPair.getPublicKey(), ) const senderCommunity = this.transferTransaction.user.communityTopicId diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/keyPair.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/keyPair.ts index cbd365f75..64ba7345d 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/keyPair.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/keyPair.ts @@ -30,7 +30,7 @@ export function generateKeyPairCommunity( if (!keyPair) { throw new Error(`Couldn't create key pair for community ${community.communityUuid}`) } - const communityKeyPairKey = new KeyPairIdentifierLogic({ communityTopicId: topicId }).getKey() + const communityKeyPairKey = new KeyPairIdentifierLogic({ communityTopicId: topicId, communityId: community.communityUuid }).getKey() cache.addKeyPair(communityKeyPairKey, keyPair) logger.info(`Community Key Pair added with key: ${communityKeyPairKey}`) } @@ -44,6 +44,7 @@ export async function generateKeyPairUserAccount( const userKeyPairRole = new UserKeyPairRole(user.gradidoId, communityKeyPair) const userKeyPairKey = new KeyPairIdentifierLogic({ communityTopicId: communityTopicId, + communityId: user.communityUuid, account: { userUuid: user.gradidoId, accountNr: 0, @@ -56,6 +57,7 @@ export async function generateKeyPairUserAccount( const accountKeyPairRole = new AccountKeyPairRole(1, userKeyPair) const accountKeyPairKey = new KeyPairIdentifierLogic({ communityTopicId: communityTopicId, + communityId: user.communityUuid, account: { userUuid: user.gradidoId, accountNr: 1, diff --git a/dlt-connector/src/schemas/account.schema.ts b/dlt-connector/src/schemas/account.schema.ts index e328a2f90..c36655bcb 100644 --- a/dlt-connector/src/schemas/account.schema.ts +++ b/dlt-connector/src/schemas/account.schema.ts @@ -11,6 +11,7 @@ export type IdentifierCommunityAccount = v.InferOutput { } let topic: HieroId const topicString = '0.0.261' +let communityUuid: Uuidv4 +const communityUuidString = '123e4567-e89b-12d3-a456-426614174000' + beforeAll(() => { topic = v.parse(hieroIdSchema, topicString) + communityUuid = v.parse(uuidv4Schema, communityUuidString) }) describe('transaction schemas', () => { @@ -55,6 +59,7 @@ describe('transaction schemas', () => { registerAddress = { user: { communityTopicId: topicString, + communityId: communityUuidString, account: { userUuid: userUuidString }, }, type: InputTransactionType.REGISTER_ADDRESS, @@ -66,6 +71,7 @@ describe('transaction schemas', () => { expect(v.parse(transactionSchema, registerAddress)).toEqual({ user: { communityTopicId: topic, + communityId: communityUuid, account: { userUuid, accountNr: 0, @@ -80,6 +86,7 @@ describe('transaction schemas', () => { expect(v.parse(registerAddressTransactionSchema, registerAddress)).toEqual({ user: { communityTopicId: topic, + communityId: communityUuid, account: { userUuid, accountNr: 0, @@ -101,10 +108,12 @@ describe('transaction schemas', () => { const gradidoTransfer: TransactionInput = { user: { communityTopicId: topicString, + communityId: communityUuidString, account: { userUuid: userUuidString }, }, linkedUser: { communityTopicId: topicString, + communityId: communityUuidString, account: { userUuid: userUuidString }, }, amount: '100', @@ -115,6 +124,7 @@ describe('transaction schemas', () => { expect(v.parse(transactionSchema, gradidoTransfer)).toEqual({ user: { communityTopicId: topic, + communityId: communityUuid, account: { userUuid, accountNr: 0, @@ -122,6 +132,7 @@ describe('transaction schemas', () => { }, linkedUser: { communityTopicId: topic, + communityId: communityUuid, account: { userUuid, accountNr: 0, @@ -138,10 +149,12 @@ describe('transaction schemas', () => { const gradidoCreation: TransactionInput = { user: { communityTopicId: topicString, + communityId: communityUuidString, account: { userUuid: userUuidString }, }, linkedUser: { communityTopicId: topicString, + communityId: communityUuidString, account: { userUuid: userUuidString }, }, amount: '1000', @@ -153,10 +166,12 @@ describe('transaction schemas', () => { expect(v.parse(transactionSchema, gradidoCreation)).toEqual({ user: { communityTopicId: topic, + communityId: communityUuid, account: { userUuid, accountNr: 0 }, }, linkedUser: { communityTopicId: topic, + communityId: communityUuid, account: { userUuid, accountNr: 0 }, }, amount: v.parse(gradidoAmountSchema, gradidoCreation.amount!), @@ -172,12 +187,14 @@ describe('transaction schemas', () => { const gradidoTransactionLink: TransactionInput = { user: { communityTopicId: topicString, + communityId: communityUuidString, account: { userUuid: userUuidString, }, }, linkedUser: { communityTopicId: topicString, + communityId: communityUuidString, seed, }, amount: '100', @@ -189,6 +206,7 @@ describe('transaction schemas', () => { expect(v.parse(transactionSchema, gradidoTransactionLink)).toEqual({ user: { communityTopicId: topic, + communityId: communityUuid, account: { userUuid, accountNr: 0, @@ -196,6 +214,7 @@ describe('transaction schemas', () => { }, linkedUser: { communityTopicId: topic, + communityId: communityUuid, seed: seedParsed, }, amount: v.parse(gradidoAmountSchema, gradidoTransactionLink.amount!), diff --git a/dlt-connector/src/schemas/transaction.schema.ts b/dlt-connector/src/schemas/transaction.schema.ts index b018a8e86..c1e16c050 100644 --- a/dlt-connector/src/schemas/transaction.schema.ts +++ b/dlt-connector/src/schemas/transaction.schema.ts @@ -43,12 +43,14 @@ export type Transaction = v.InferOutput // if the account is identified by seed export const seedAccountSchema = v.object({ communityTopicId: hieroIdSchema, + communityId: uuidv4Schema, seed: identifierSeedSchema, }) // if the account is identified by userUuid and accountNr export const userAccountSchema = v.object({ communityTopicId: hieroIdSchema, + communityId: uuidv4Schema, account: identifierCommunityAccountSchema, }) diff --git a/dlt-connector/src/schemas/typeConverter.schema.ts b/dlt-connector/src/schemas/typeConverter.schema.ts index 8774bd6c2..572593f0c 100644 --- a/dlt-connector/src/schemas/typeConverter.schema.ts +++ b/dlt-connector/src/schemas/typeConverter.schema.ts @@ -8,6 +8,7 @@ import { toAccountType, toAddressType, } from '../utils/typeConverter' +import { Uuidv4, uuidv4Schema } from './typeGuard.schema' /** * dateSchema for creating a date from string or Date object @@ -72,17 +73,23 @@ export const accountTypeSchema = v.pipe( export const confirmedTransactionSchema = v.pipe( v.union([ v.instance(ConfirmedTransaction, 'expect ConfirmedTransaction'), - v.pipe( - v.string('expect confirmed Transaction base64 as string type'), - v.base64('expect to be valid base64'), - ), + v.object({ + base64: v.pipe( + v.string('expect confirmed Transaction base64 as string type'), + v.base64('expect to be valid base64'), + ), + communityId: uuidv4Schema, + }), ]), - v.transform( - (data: string | ConfirmedTransaction) => { + v.transform( + (data: string | ConfirmedTransaction | { base64: string; communityId: Uuidv4 }) => { if (data instanceof ConfirmedTransaction) { return data } - return confirmedTransactionFromBase64(data) + if (typeof data === 'object' && 'base64' in data && 'communityId' in data) { + return confirmedTransactionFromBase64(data.base64, data.communityId) + } + throw new Error('invalid data, community id missing, couldn\'t deserialize') }, ), ) diff --git a/dlt-connector/src/server/index.ts b/dlt-connector/src/server/index.ts index 1692bacfb..8afae814d 100644 --- a/dlt-connector/src/server/index.ts +++ b/dlt-connector/src/server/index.ts @@ -69,9 +69,10 @@ export const appRoutes = new Elysia() // check if account exists by user, call example: // GET /isAccountExist/by-user/0.0.21732/408780b2-59b3-402a-94be-56a4f4f4e8ec/0 .get( - '/isAccountExist/by-user/:communityTopicId/:userUuid/:accountNr', - async ({ params: { communityTopicId, userUuid, accountNr } }) => ({ + '/isAccountExist/by-user/:communityId/:communityTopicId/:userUuid/:accountNr', + async ({ params: { communityId, communityTopicId, userUuid, accountNr } }) => ({ exists: await isAccountExist({ + communityId, communityTopicId, account: { userUuid, accountNr }, }), @@ -84,9 +85,10 @@ export const appRoutes = new Elysia() // check if account exists by seed, call example: // GET /isAccountExist/by-seed/0.0.21732/0c4676adfd96519a0551596c .get( - '/isAccountExist/by-seed/:communityTopicId/:seed', - async ({ params: { communityTopicId, seed } }) => ({ + '/isAccountExist/by-seed/:communityId/:communityTopicId/:seed', + async ({ params: { communityId, communityTopicId, seed } }) => ({ exists: await isAccountExist({ + communityId, communityTopicId, seed, }), @@ -145,7 +147,7 @@ async function isAccountExist(identifierAccount: IdentifierAccountInput): Promis // ask gradido node server for account type, if type !== NONE account exist const addressType = await GradidoNodeClient.getInstance().getAddressType( publicKey.convertToHex(), - identifierAccountParsed.communityTopicId, + identifierAccountParsed.communityId, ) const exists = addressType !== AddressType_NONE const endTime = Date.now() diff --git a/dlt-connector/src/server/input.schema.ts b/dlt-connector/src/server/input.schema.ts index df93f1d9b..501fdc829 100644 --- a/dlt-connector/src/server/input.schema.ts +++ b/dlt-connector/src/server/input.schema.ts @@ -3,6 +3,7 @@ import { t } from 'elysia' import { hieroIdSchema, uuidv4Schema } from '../schemas/typeGuard.schema' export const accountIdentifierUserTypeBoxSchema = t.Object({ + communityId: TypeBoxFromValibot(uuidv4Schema), communityTopicId: TypeBoxFromValibot(hieroIdSchema), userUuid: TypeBoxFromValibot(uuidv4Schema), accountNr: t.Number({ min: 0 }), @@ -10,6 +11,7 @@ export const accountIdentifierUserTypeBoxSchema = t.Object({ // identifier for a gradido account created by transaction link / deferred transfer export const accountIdentifierSeedTypeBoxSchema = t.Object({ + communityId: TypeBoxFromValibot(uuidv4Schema), communityTopicId: TypeBoxFromValibot(hieroIdSchema), seed: TypeBoxFromValibot(uuidv4Schema), }) diff --git a/dlt-connector/src/utils/filesystem.ts b/dlt-connector/src/utils/filesystem.ts index f01008a93..096437fb5 100644 --- a/dlt-connector/src/utils/filesystem.ts +++ b/dlt-connector/src/utils/filesystem.ts @@ -9,7 +9,7 @@ export function checkFileExist(filePath: string): boolean { fs.accessSync(filePath, fs.constants.R_OK | fs.constants.W_OK) return true } catch (_err) { - // logger.debug(`file ${filePath} does not exist: ${_err}`) + logger.debug(`file ${filePath} does not exist: ${_err}`) return false } } diff --git a/dlt-connector/src/utils/typeConverter.ts b/dlt-connector/src/utils/typeConverter.ts index 148efebd6..4124df7e6 100644 --- a/dlt-connector/src/utils/typeConverter.ts +++ b/dlt-connector/src/utils/typeConverter.ts @@ -6,14 +6,15 @@ import { } from 'gradido-blockchain-js' import { AccountType } from '../data/AccountType.enum' import { AddressType } from '../data/AddressType.enum' +import { Uuidv4 } from '../schemas/typeGuard.schema' -export const confirmedTransactionFromBase64 = (base64: string): ConfirmedTransaction => { +export const confirmedTransactionFromBase64 = (base64: string, communityId: Uuidv4): ConfirmedTransaction => { const confirmedTransactionBinaryPtr = MemoryBlock.createPtr(MemoryBlock.fromBase64(base64)) const deserializer = new InteractionDeserialize( confirmedTransactionBinaryPtr, DeserializeType_CONFIRMED_TRANSACTION, ) - deserializer.run() + deserializer.run(communityId) const confirmedTransaction = deserializer.getConfirmedTransaction() if (!confirmedTransaction) { throw new Error("invalid data, couldn't deserialize") From 36890d7420b8e60d208532458b680557ea4bba4f Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 25 Feb 2026 17:10:35 +0100 Subject: [PATCH 30/50] update gradido node default version --- dlt-connector/src/config/schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dlt-connector/src/config/schema.ts b/dlt-connector/src/config/schema.ts index ef4558545..f9fc14778 100644 --- a/dlt-connector/src/config/schema.ts +++ b/dlt-connector/src/config/schema.ts @@ -84,7 +84,7 @@ export const configSchema = v.object({ v.string('The version of the DLT node server, for example: 0.9.0'), v.regex(/^\d+\.\d+\.\d+$/), ), - '0.9.2', + '0.9.3', ), DLT_GRADIDO_NODE_SERVER_HOME_FOLDER: v.optional( v.string('The home folder for the gradido dlt node server'), From b6b00aa7462e222579533feadfcf20a4825e1610 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 25 Feb 2026 17:29:16 +0100 Subject: [PATCH 31/50] update gradido-blockchain-js version --- dlt-connector/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dlt-connector/package.json b/dlt-connector/package.json index 518e17418..ce74c27ba 100644 --- a/dlt-connector/package.json +++ b/dlt-connector/package.json @@ -20,7 +20,7 @@ "dependencies": { "bun-zigar": "^0.15.2", "cross-env": "^7.0.3", - "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#f265dbb1780a912cf8b0418dfe3eaf5cdc5b51cf" + "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#66c8f6839dd95fc34d323066951b80043690fdc2" }, "devDependencies": { "@biomejs/biome": "2.0.0", From 4025175ceac31afc95f96c52eab487c48da15921 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 26 Feb 2026 09:01:22 +0100 Subject: [PATCH 32/50] update inspector to newest version --- inspector | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inspector b/inspector index 0c14b7eea..173fb6a96 160000 --- a/inspector +++ b/inspector @@ -1 +1 @@ -Subproject commit 0c14b7eea29b8911cbe3cb303f5b0b61ce9bf6f4 +Subproject commit 173fb6a9626ba1e8c69ba0cc04f87e1c5ffb1570 From 31766b83efd489f3c837af6d444d62d4d2c8d93d Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 26 Feb 2026 09:08:29 +0100 Subject: [PATCH 33/50] fix linting --- .../dltConnector/model/AccountIdentifier.ts | 6 +- .../dltConnector/model/TransactionDraft.ts | 6 +- .../src/graphql/resolver/CommunityResolver.ts | 11 +- ...097-fix_production_data_for_blockchain2.ts | 2 - database/src/queries/communities.ts | 11 +- .../src/bootstrap/initGradidoNode.ts | 5 +- .../src/cache/KeyPairCacheManager.ts | 5 +- .../client/GradidoNode/GradidoNodeClient.ts | 22 +- .../client/GradidoNode/GradidoNodeProcess.ts | 11 +- .../src/client/GradidoNode/communities.ts | 3 +- .../client/GradidoNode/input.schema.test.ts | 4 +- .../src/client/GradidoNode/input.schema.ts | 2 +- dlt-connector/src/client/backend/graphql.ts | 2 - dlt-connector/src/data/deriveKeyPair.ts | 11 +- .../RegisterAddressTransaction.role.ts | 10 +- .../sendToHiero/SendToHiero.context.ts | 6 +- .../db-v2.7.0_to_blockchain-v3.7/Context.ts | 25 +- .../binaryExport.ts | 15 +- .../blockchain.ts | 15 +- .../db-v2.7.0_to_blockchain-v3.7/bootstrap.ts | 39 ++- .../data/Balance.ts | 174 +++++++------ .../data/keyPair.ts | 5 +- .../db-v2.7.0_to_blockchain-v3.7/database.ts | 25 +- .../drizzle.schema.ts | 54 ++-- .../db-v2.7.0_to_blockchain-v3.7/errors.ts | 18 +- .../db-v2.7.0_to_blockchain-v3.7/index.ts | 23 +- .../syncDbWithBlockchain/AbstractSync.role.ts | 56 ++-- .../ContributionLinkTransactionSync.role.ts | 44 ++-- .../CreationsSync.role.ts | 120 +++++---- .../DeletedTransactionLinksSync.role.ts | 219 +++++++++------- .../LocalTransactionsSync.role.ts | 126 +++++---- .../RedeemTransactionLinksSync.role.ts | 169 +++++++----- .../RemoteTransactionsSync.role.ts | 241 +++++++++++------- .../TransactionLinkFundingsSync.role.ts | 173 ++++++++----- .../syncDbWithBlockchain/UsersSync.role.ts | 71 +++--- .../syncDbWithBlockchain.context.ts | 50 ++-- .../db-v2.7.0_to_blockchain-v3.7/utils.ts | 8 +- .../valibot.schema.ts | 110 ++++---- .../src/schemas/typeConverter.schema.ts | 2 +- dlt-connector/src/utils/filesystem.ts | 2 +- dlt-connector/src/utils/typeConverter.ts | 5 +- shared/src/logic/decay.ts | 4 +- 42 files changed, 1112 insertions(+), 798 deletions(-) diff --git a/backend/src/apis/dltConnector/model/AccountIdentifier.ts b/backend/src/apis/dltConnector/model/AccountIdentifier.ts index 1d324a70f..28fc844cd 100644 --- a/backend/src/apis/dltConnector/model/AccountIdentifier.ts +++ b/backend/src/apis/dltConnector/model/AccountIdentifier.ts @@ -5,7 +5,11 @@ export class AccountIdentifier { account?: CommunityAccountIdentifier seed?: string // used for deferred transfers - constructor(communityTopicId: string, communityUuid: string, input: CommunityAccountIdentifier | string) { + constructor( + communityTopicId: string, + communityUuid: string, + input: CommunityAccountIdentifier | string, + ) { if (input instanceof CommunityAccountIdentifier) { this.account = input } else { diff --git a/backend/src/apis/dltConnector/model/TransactionDraft.ts b/backend/src/apis/dltConnector/model/TransactionDraft.ts index 5352bf841..162b4e2c5 100755 --- a/backend/src/apis/dltConnector/model/TransactionDraft.ts +++ b/backend/src/apis/dltConnector/model/TransactionDraft.ts @@ -133,7 +133,11 @@ export class TransactionDraft { sendingUser.community.communityUuid!, new CommunityAccountIdentifier(sendingUser.gradidoID), ) - draft.linkedUser = new AccountIdentifier(senderUserTopic, sendingUser.community.communityUuid!, transactionLink.code) + draft.linkedUser = new AccountIdentifier( + senderUserTopic, + sendingUser.community.communityUuid!, + transactionLink.code, + ) draft.type = TransactionType.GRADIDO_DEFERRED_TRANSFER draft.createdAt = createdAtOnlySeconds.toISOString() draft.amount = transactionLink.amount.toString() diff --git a/backend/src/graphql/resolver/CommunityResolver.ts b/backend/src/graphql/resolver/CommunityResolver.ts index d037fffb4..4932c0f93 100644 --- a/backend/src/graphql/resolver/CommunityResolver.ts +++ b/backend/src/graphql/resolver/CommunityResolver.ts @@ -2,7 +2,12 @@ import { Paginated } from '@arg/Paginated' import { EditCommunityInput } from '@input/EditCommunityInput' import { AdminCommunityView } from '@model/AdminCommunityView' import { Community } from '@model/Community' -import { Community as DbCommunity, getAuthorizedCommunities, getHomeCommunity, getReachableCommunities } from 'database' +import { + Community as DbCommunity, + getAuthorizedCommunities, + getHomeCommunity, + getReachableCommunities, +} from 'database' import { updateAllDefinedAndChanged } from 'shared' import { Arg, Args, Authorized, Mutation, Query, Resolver } from 'type-graphql' import { RIGHTS } from '@/auth/RIGHTS' @@ -38,9 +43,9 @@ export class CommunityResolver { @Query(() => [Community]) async authorizedCommunities(): Promise { const dbCommunities: DbCommunity[] = await getAuthorizedCommunities({ - // order by + // order by foreign: 'ASC', // home community first - name: 'ASC', + name: 'ASC', }) return dbCommunities.map((dbCom: DbCommunity) => new Community(dbCom)) } diff --git a/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts b/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts index 9f67e267a..6cd5f7c39 100644 --- a/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts +++ b/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts @@ -45,7 +45,6 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis * Fix 0: Update transaction links to match holdAvailableAmount with validUntil, because the old formula lead to incorrect values */ - let sumCount = 0 let count = 0 let lastProcessedId = 0 const LIMIT = 200 @@ -106,7 +105,6 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis `, [updates.map((u) => u.id)], ) - sumCount += updates.length } count = rows.length lastProcessedId = rows[rows.length - 1].id diff --git a/database/src/queries/communities.ts b/database/src/queries/communities.ts index 37fba27ff..6d1064f9b 100644 --- a/database/src/queries/communities.ts +++ b/database/src/queries/communities.ts @@ -103,13 +103,12 @@ export async function getNotReachableCommunities( // return the home community and all communities which had at least once make it through the first handshake export async function getAuthorizedCommunities( order?: FindOptionsOrder, -): Promise -{ +): Promise { return await DbCommunity.find({ where: [ { authenticatedAt: Not(IsNull()) }, // or - { foreign: false } - ], - order + { foreign: false }, + ], + order, }) -} \ No newline at end of file +} diff --git a/dlt-connector/src/bootstrap/initGradidoNode.ts b/dlt-connector/src/bootstrap/initGradidoNode.ts index dfc8d0c10..e0d1a047d 100644 --- a/dlt-connector/src/bootstrap/initGradidoNode.ts +++ b/dlt-connector/src/bootstrap/initGradidoNode.ts @@ -7,10 +7,7 @@ import { exportCommunities } from '../client/GradidoNode/communities' import { GradidoNodeProcess } from '../client/GradidoNode/GradidoNodeProcess' import { HieroClient } from '../client/hiero/HieroClient' import { CONFIG } from '../config' -import { - GRADIDO_NODE_HOME_FOLDER_NAME, - LOG4JS_BASE_CATEGORY, -} from '../config/const' +import { GRADIDO_NODE_HOME_FOLDER_NAME, LOG4JS_BASE_CATEGORY } from '../config/const' import { checkFileExist, checkPathExist } from '../utils/filesystem' import { isPortOpen } from '../utils/network' import { AppContextClients } from './appContext' diff --git a/dlt-connector/src/cache/KeyPairCacheManager.ts b/dlt-connector/src/cache/KeyPairCacheManager.ts index f38342904..21b24cae1 100644 --- a/dlt-connector/src/cache/KeyPairCacheManager.ts +++ b/dlt-connector/src/cache/KeyPairCacheManager.ts @@ -77,10 +77,7 @@ export class KeyPairCacheManager { return keyPair } - public getKeyPairSync( - input: string, - createKeyPair: () => KeyPairEd25519, - ): KeyPairEd25519 { + public getKeyPairSync(input: string, createKeyPair: () => KeyPairEd25519): KeyPairEd25519 { const keyPair = this.cache.get(input) if (!keyPair) { const keyPair = createKeyPair() diff --git a/dlt-connector/src/client/GradidoNode/GradidoNodeClient.ts b/dlt-connector/src/client/GradidoNode/GradidoNodeClient.ts index 0a3d0c7ae..9d1515bfc 100644 --- a/dlt-connector/src/client/GradidoNode/GradidoNodeClient.ts +++ b/dlt-connector/src/client/GradidoNode/GradidoNodeClient.ts @@ -8,7 +8,7 @@ import { LOG4JS_BASE_CATEGORY } from '../../config/const' import { AddressType } from '../../data/AddressType.enum' import { Uuidv4Hash } from '../../data/Uuidv4Hash' import { addressTypeSchema, confirmedTransactionSchema } from '../../schemas/typeConverter.schema' -import { Hex32, Hex32Input, Uuidv4, hex32Schema } from '../../schemas/typeGuard.schema' +import { Hex32, Hex32Input, hex32Schema, Uuidv4 } from '../../schemas/typeGuard.schema' import { isPortOpenRetry } from '../../utils/network' import { GradidoNodeErrorCodes } from './GradidoNodeErrorCodes' import { @@ -75,7 +75,10 @@ export class GradidoNodeClient { const response = await this.rpcCall<{ transaction: string }>('getTransaction', parameter) if (response.isSuccess()) { // this.logger.debug('result: ', response.result.transaction) - return v.parse(confirmedTransactionSchema, { base64: response.result.transaction, communityId: parameter.communityId }) + return v.parse(confirmedTransactionSchema, { + base64: response.result.transaction, + communityId: parameter.communityId, + }) } if (response.isError()) { if (response.error.code === GradidoNodeErrorCodes.TRANSACTION_NOT_FOUND) { @@ -100,7 +103,10 @@ export class GradidoNodeClient { } const response = await this.rpcCall<{ transaction: string }>('getLastTransaction', parameter) if (response.isSuccess()) { - return v.parse(confirmedTransactionSchema, { base64: response.result.transaction, communityId: parameter.communityId }) + return v.parse(confirmedTransactionSchema, { + base64: response.result.transaction, + communityId: parameter.communityId, + }) } if (response.isError()) { if (response.error.code === GradidoNodeErrorCodes.GRADIDO_NODE_ERROR) { @@ -137,7 +143,10 @@ export class GradidoNodeClient { parameter, ) return result.transactions.map((transactionBase64) => - v.parse(confirmedTransactionSchema, { base64: transactionBase64, communityId: parameter.communityId }), + v.parse(confirmedTransactionSchema, { + base64: transactionBase64, + communityId: parameter.communityId, + }), ) } @@ -163,7 +172,10 @@ export class GradidoNodeClient { parameter, ) return response.transactions.map((transactionBase64) => - v.parse(confirmedTransactionSchema, { base64: transactionBase64, communityId: parameter.communityId }), + v.parse(confirmedTransactionSchema, { + base64: transactionBase64, + communityId: parameter.communityId, + }), ) } diff --git a/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts b/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts index 526fd99be..0347d136e 100644 --- a/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts +++ b/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts @@ -1,3 +1,4 @@ +import path from 'node:path' import { Mutex } from 'async-mutex' import { Subprocess, spawn } from 'bun' import { getLogger, Logger } from 'log4js' @@ -10,7 +11,6 @@ import { LOG4JS_BASE_CATEGORY, } from '../../config/const' import { delay } from '../../utils/time' -import path from 'node:path' /** * A Singleton class defines the `getInstance` method that lets clients access * the unique singleton instance. @@ -48,14 +48,7 @@ export class GradidoNodeProcess { const isWindows = process.platform === 'win32' const binaryName = isWindows ? 'GradidoNode.exe' : 'GradidoNode' - return path.join( - __dirname, - '..', - '..', - 'gradido_node', - 'bin', - binaryName, - ) + return path.join(__dirname, '..', '..', 'gradido_node', 'bin', binaryName) } public start() { diff --git a/dlt-connector/src/client/GradidoNode/communities.ts b/dlt-connector/src/client/GradidoNode/communities.ts index b285eed92..8af18e3ef 100644 --- a/dlt-connector/src/client/GradidoNode/communities.ts +++ b/dlt-connector/src/client/GradidoNode/communities.ts @@ -39,7 +39,6 @@ export async function ensureCommunitiesAvailable(communityTopicIds: HieroId[]): export async function exportCommunities(homeFolder: string, client: BackendClient): Promise { const communities = await client.getAuthorizedCommunities() - console.log(`communities: ${JSON.stringify(communities, null, 2)}`) const communitiesPath = path.join(homeFolder, 'communities.json') checkPathExist(path.dirname(communitiesPath), true) const communitiesForDltNodeServer: CommunityForDltNodeServer[] = [] @@ -52,7 +51,7 @@ export async function exportCommunities(homeFolder: string, client: BackendClien hieroTopicId: com.hieroTopicId, alias: com.name, // use only alpha-numeric chars for folder name - folder: toFolderName(com.uuid), + folder: toFolderName(com.uuid), }) } fs.writeFileSync(communitiesPath, JSON.stringify(communitiesForDltNodeServer, null, 2)) diff --git a/dlt-connector/src/client/GradidoNode/input.schema.test.ts b/dlt-connector/src/client/GradidoNode/input.schema.test.ts index 550437100..4db653f26 100644 --- a/dlt-connector/src/client/GradidoNode/input.schema.test.ts +++ b/dlt-connector/src/client/GradidoNode/input.schema.test.ts @@ -1,14 +1,14 @@ import { beforeAll, describe, expect, it } from 'bun:test' +import { v4 as uuidv4 } from 'uuid' import * as v from 'valibot' import { HieroTransactionIdString, - Uuidv4, hieroIdSchema, hieroTransactionIdStringSchema, + Uuidv4, uuidv4Schema, } from '../../schemas/typeGuard.schema' import { transactionIdentifierSchema } from './input.schema' -import { v4 as uuidv4 } from 'uuid' let communityId: Uuidv4 const uuidv4String = uuidv4() diff --git a/dlt-connector/src/client/GradidoNode/input.schema.ts b/dlt-connector/src/client/GradidoNode/input.schema.ts index f8a21f562..61c9887d5 100644 --- a/dlt-connector/src/client/GradidoNode/input.schema.ts +++ b/dlt-connector/src/client/GradidoNode/input.schema.ts @@ -1,5 +1,5 @@ import * as v from 'valibot' -import { uuidv4Schema, hieroTransactionIdStringSchema } from '../../schemas/typeGuard.schema' +import { hieroTransactionIdStringSchema, uuidv4Schema } from '../../schemas/typeGuard.schema' export const transactionsRangeSchema = v.object({ // default value is 1, from first transactions diff --git a/dlt-connector/src/client/backend/graphql.ts b/dlt-connector/src/client/backend/graphql.ts index 2f1fbec74..466a16c16 100644 --- a/dlt-connector/src/client/backend/graphql.ts +++ b/dlt-connector/src/client/backend/graphql.ts @@ -53,5 +53,3 @@ export const getAuthorizedCommunities = gql` } ${communityFragment} ` - - diff --git a/dlt-connector/src/data/deriveKeyPair.ts b/dlt-connector/src/data/deriveKeyPair.ts index f0880b207..f4a805eb3 100644 --- a/dlt-connector/src/data/deriveKeyPair.ts +++ b/dlt-connector/src/data/deriveKeyPair.ts @@ -17,9 +17,7 @@ export function deriveFromCode(code: string): KeyPairEd25519 { const hash = new MemoryBlock(code).calculateHash() const keyPair = KeyPairEd25519.create(hash) if (!keyPair) { - throw new ParameterError( - `error creating Ed25519 KeyPair from seed: ${code.substring(0, 5)}...`, - ) + throw new ParameterError(`error creating Ed25519 KeyPair from seed: ${code.substring(0, 5)}...`) } return keyPair } @@ -31,7 +29,10 @@ export function deriveFromKeyPairAndUuid(keyPair: KeyPairEd25519, uuid: Uuidv4): parts[i] = hardenDerivationIndex(wholeHex.subarray(i * 4, (i + 1) * 4).readUInt32BE()) } // parts: [2206563009, 2629978174, 2324817329, 2405141782] - return parts.reduce((keyPair: KeyPairEd25519, node: number) => deriveFromKeyPairAndIndex(keyPair, node), keyPair) + return parts.reduce( + (keyPair: KeyPairEd25519, node: number) => deriveFromKeyPairAndIndex(keyPair, node), + keyPair, + ) } export function deriveFromKeyPairAndIndex(keyPair: KeyPairEd25519, index: number): KeyPairEd25519 { @@ -42,4 +43,4 @@ export function deriveFromKeyPairAndIndex(keyPair: KeyPairEd25519, index: number ) } return localKeyPair -} \ No newline at end of file +} diff --git a/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.ts index 98ddc01f3..7344c9a59 100644 --- a/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.ts @@ -35,10 +35,12 @@ export class RegisterAddressTransactionRole extends AbstractTransactionRole { public async getGradidoTransactionBuilder(): Promise { const builder = new GradidoTransactionBuilder() const communityTopicId = this.registerAddressTransaction.user.communityTopicId - const communityKeyPair = await ResolveKeyPair(new KeyPairIdentifierLogic({ - communityTopicId, - communityId: this.registerAddressTransaction.user.communityId, - })) + const communityKeyPair = await ResolveKeyPair( + new KeyPairIdentifierLogic({ + communityTopicId, + communityId: this.registerAddressTransaction.user.communityId, + }), + ) const keyPairIdentifier = this.registerAddressTransaction.user // when accountNr is 0 it is the user account keyPairIdentifier.account.accountNr = 0 diff --git a/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts b/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts index 9bb6f5c74..bacc01401 100644 --- a/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts +++ b/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts @@ -65,7 +65,9 @@ export async function SendToHieroContext( ) // attach Hiero transaction ID to the builder for the inbound transaction - builder.setParentLedgerAnchor(new LedgerAnchor(new HieroTransactionId(outboundHieroTransactionIdString))) + builder.setParentLedgerAnchor( + new LedgerAnchor(new HieroTransactionId(outboundHieroTransactionIdString)), + ) // build and validate inbound transaction const inboundTransaction = builder.buildInbound() @@ -103,7 +105,7 @@ function validate(transaction: GradidoTransaction): void { async function sendViaHiero( gradidoTransaction: GradidoTransaction, topic: HieroId, - communityId: Uuidv4 + communityId: Uuidv4, ): Promise { const client = HieroClient.getInstance() const transactionId = await client.sendMessage(topic, communityId, gradidoTransaction) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts index ea854ce7a..2a24867ef 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts @@ -23,7 +23,12 @@ export class Context { public balanceFixGradidoUser: UserDb | null private timeUsed: Profiler - constructor(logger: Logger, db: MySql2Database, cache: KeyPairCacheManager, balanceFixGradidoUser: UserDb | null) { + constructor( + logger: Logger, + db: MySql2Database, + cache: KeyPairCacheManager, + balanceFixGradidoUser: UserDb | null, + ) { this.logger = logger this.db = db this.cache = cache @@ -42,23 +47,23 @@ export class Context { database: CONFIG.MYSQL_DATABASE, port: CONFIG.MYSQL_PORT, }) - const db = drizzle({ client: connection }) + const db = drizzle({ client: connection }) const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5`) let balanceFixGradidoUser: UserDb | null = null if (process.env.MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID) { - balanceFixGradidoUser = await loadUserByGradidoId(db, process.env.MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID) + balanceFixGradidoUser = await loadUserByGradidoId( + db, + process.env.MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID, + ) if (!balanceFixGradidoUser) { - logger.error(`MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID was set to ${process.env.MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID} but user not found`) + logger.error( + `MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID was set to ${process.env.MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID} but user not found`, + ) } } else { logger.debug(`MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID was not set`) } - return new Context( - logger, - db, - KeyPairCacheManager.getInstance(), - balanceFixGradidoUser, - ) + return new Context(logger, db, KeyPairCacheManager.getInstance(), balanceFixGradidoUser) } getCommunityContextByUuid(communityUuid: Uuidv4): CommunityContext { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts index 8ec631201..6870740eb 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts @@ -43,7 +43,9 @@ export function exportCommunity( const isDebug = context.logger.isDebugEnabled() const printConsole = () => { if (triggeredTransactionsCount > 0) { - process.stdout.write(`exported ${count} transactions + ${triggeredTransactionsCount} triggered from timeouted transaction links\r`) + process.stdout.write( + `exported ${count} transactions + ${triggeredTransactionsCount} triggered from timeouted transaction links\r`, + ) } else { process.stdout.write(`exported ${count} transactions\r`) } @@ -59,7 +61,12 @@ export function exportCommunity( throw new Error(`invalid TransactionEntry at index: ${transactionNr} `) } hash = exportTransaction(confirmedTransaction, hash, binFilePath) - if (confirmedTransaction?.getGradidoTransaction()?.getTransactionBody()?.isTimeoutDeferredTransfer()) { + if ( + confirmedTransaction + ?.getGradidoTransaction() + ?.getTransactionBody() + ?.isTimeoutDeferredTransfer() + ) { triggeredTransactionsCount++ } else { count++ @@ -77,7 +84,7 @@ export function exportCommunity( } } } - f.pagination.page++ + f.pagination.page++ } while (lastTransactionCount === batchSize) printConsole() process.stdout.write(`\n`) @@ -86,7 +93,7 @@ export function exportCommunity( context.logger.info( `binary file for community ${communityContext.communityId} written to ${binFilePath}`, ) - const sumTransactionsCount = ((f.pagination.page - 2) * batchSize) + lastTransactionCount + const sumTransactionsCount = (f.pagination.page - 2) * batchSize + lastTransactionCount const fileSize = fs.statSync(binFilePath).size context.logger.info( `exported ${sumTransactionsCount} transactions (${bytesString(fileSize)}) in ${timeUsed.string()}`, diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts index 770122f92..edf752788 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts @@ -11,7 +11,7 @@ import { NotEnoughGradidoBalanceError } from './errors' export const defaultHieroAccount = new HieroAccountId(0, 0, 2) export let callTime: number = 0 -const timeUsed = new Profiler +const timeUsed = new Profiler() export function addToBlockchain( transaction: GradidoTransaction, @@ -19,7 +19,7 @@ export function addToBlockchain( ledgerAnchor: LedgerAnchor, accountBalances: AccountBalances, ): boolean { - try { + try { timeUsed.reset() const result = blockchain.createAndAddConfirmedTransactionExternFast( transaction, @@ -30,7 +30,9 @@ export function addToBlockchain( return result } catch (error) { if (error instanceof Error) { - const matches = error.message.match(/not enough Gradido Balance for (send coins|operation), needed: -?(\d+\.\d+), exist: (\d+\.\d+)/) + const matches = error.message.match( + /not enough Gradido Balance for (send coins|operation), needed: -?(\d+\.\d+), exist: (\d+\.\d+)/, + ) if (matches) { const needed = parseFloat(matches[2]) const exist = parseFloat(matches[3]) @@ -40,8 +42,9 @@ export function addToBlockchain( // const wekingheim = InMemoryBlockchainProvider.getInstance().getBlockchain('wekingheim') // const lastTransactionw = wekingheim?.findOne(Filter.LAST_TRANSACTION) - const lastTransaction = blockchain.findOne(Filter.LAST_TRANSACTION) - throw new Error(`Transaction ${transaction.toJson(true)} not added: ${error}, last transaction was: ${lastTransaction?.getConfirmedTransaction()?.toJson(true)}`) + const lastTransaction = blockchain.findOne(Filter.LAST_TRANSACTION) + throw new Error( + `Transaction ${transaction.toJson(true)} not added: ${error}, last transaction was: ${lastTransaction?.getConfirmedTransaction()?.toJson(true)}`, + ) } } - diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts index 3ccb24f1d..a95cc136a 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts @@ -1,23 +1,36 @@ import { randomBytes } from 'node:crypto' -import { AccountBalances, GradidoTransactionBuilder, InMemoryBlockchainProvider, LedgerAnchor } from 'gradido-blockchain-js' +import { + AccountBalances, + GradidoTransactionBuilder, + InMemoryBlockchainProvider, + LedgerAnchor, +} from 'gradido-blockchain-js' import * as v from 'valibot' import { CONFIG } from '../../config' import { deriveFromSeed } from '../../data/deriveKeyPair' import { Hex32, hex32Schema } from '../../schemas/typeGuard.schema' -import { AUF_ACCOUNT_DERIVATION_INDEX, GMW_ACCOUNT_DERIVATION_INDEX, hardenDerivationIndex } from '../../utils/derivationHelper' +import { + AUF_ACCOUNT_DERIVATION_INDEX, + GMW_ACCOUNT_DERIVATION_INDEX, + hardenDerivationIndex, +} from '../../utils/derivationHelper' +import { toFolderName } from '../../utils/filesystem' import { addToBlockchain } from './blockchain' import { Context } from './Context' import { Balance } from './data/Balance' -import { loadAdminUsersCache, loadCommunities, loadContributionLinkModeratorCache } from './database' +import { + loadAdminUsersCache, + loadCommunities, + loadContributionLinkModeratorCache, +} from './database' import { CommunityContext } from './valibot.schema' -import { toFolderName } from '../../utils/filesystem' export async function bootstrap(): Promise { const context = await Context.create() context.communities = await bootstrapCommunities(context) await Promise.all([ loadContributionLinkModeratorCache(context.db), - loadAdminUsersCache(context.db) + loadAdminUsersCache(context.db), ]) return context } @@ -39,17 +52,23 @@ async function bootstrapCommunities(context: Context): Promise() export const adminUsers = new Map() @@ -29,7 +19,10 @@ export async function loadContributionLinkModeratorCache(db: MySql2Database): Pr .orderBy(asc(eventsTable.id)) result.map((row: any) => { - contributionLinkModerators.set(row.event.involvedContributionLinkId, v.parse(userDbSchema, row.user)) + contributionLinkModerators.set( + row.event.involvedContributionLinkId, + v.parse(userDbSchema, row.user), + ) }) } @@ -69,7 +62,10 @@ export async function loadCommunities(db: MySql2Database): Promise { +export async function loadUserByGradidoId( + db: MySql2Database, + gradidoId: string, +): Promise { const result = await db .select() .from(usersTable) @@ -78,4 +74,3 @@ export async function loadUserByGradidoId(db: MySql2Database, gradidoId: string) return result.length ? v.parse(userDbSchema, result[0]) : null } - diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/drizzle.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/drizzle.schema.ts index 9f17d4144..71c5bd08a 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/drizzle.schema.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/drizzle.schema.ts @@ -25,23 +25,23 @@ export const communitiesTable = mysqlTable( ) export const contributionsTable = mysqlTable('contributions', { - id: int().autoincrement().notNull(), + id: int().autoincrement().notNull(), userId: int('user_id').default(sql`NULL`), - contributionDate: datetime("contribution_date", { mode: 'string'}).default(sql`NULL`), - memo: varchar({ length: 512 }).notNull(), - amount: decimal({ precision: 40, scale: 20 }).notNull(), - contributionLinkId: int('contribution_link_id').default(sql`NULL`), - confirmedBy: int('confirmed_by').default(sql`NULL`), - confirmedAt: datetime('confirmed_at', { mode: 'string'}).default(sql`NULL`), - contributionStatus: varchar('contribution_status', { length: 12 }).default('\'PENDING\'').notNull(), - transactionId: int('transaction_id').default(sql`NULL`), + contributionDate: datetime('contribution_date', { mode: 'string' }).default(sql`NULL`), + memo: varchar({ length: 512 }).notNull(), + amount: decimal({ precision: 40, scale: 20 }).notNull(), + contributionLinkId: int('contribution_link_id').default(sql`NULL`), + confirmedBy: int('confirmed_by').default(sql`NULL`), + confirmedAt: datetime('confirmed_at', { mode: 'string' }).default(sql`NULL`), + contributionStatus: varchar('contribution_status', { length: 12 }).default("'PENDING'").notNull(), + transactionId: int('transaction_id').default(sql`NULL`), }) export const eventsTable = mysqlTable('events', { - id: int().autoincrement().notNull(), - type: varchar({ length: 100 }).notNull(), - actingUserId: int('acting_user_id').notNull(), - involvedContributionLinkId: int('involved_contribution_link_id').default(sql`NULL`), + id: int().autoincrement().notNull(), + type: varchar({ length: 100 }).notNull(), + actingUserId: int('acting_user_id').notNull(), + involvedContributionLinkId: int('involved_contribution_link_id').default(sql`NULL`), }) export const usersTable = mysqlTable( @@ -55,19 +55,19 @@ export const usersTable = mysqlTable( createdAt: datetime('created_at', { mode: 'string', fsp: 3 }) .default(sql`current_timestamp(3)`) .notNull(), - }, (table) => [unique('uuid_key').on(table.gradidoId, table.communityUuid)], ) -export const userRolesTable = mysqlTable('user_roles', { - id: int().autoincrement().notNull(), - userId: int('user_id').notNull(), - role: varchar({ length: 40 }).notNull(), -}, -(table) => [ - index('user_id').on(table.userId), -]) +export const userRolesTable = mysqlTable( + 'user_roles', + { + id: int().autoincrement().notNull(), + userId: int('user_id').notNull(), + role: varchar({ length: 40 }).notNull(), + }, + (table) => [index('user_id').on(table.userId)], +) export const transactionsTable = mysqlTable( 'transactions', @@ -84,8 +84,8 @@ export const transactionsTable = mysqlTable( creationDate: datetime('creation_date', { mode: 'string', fsp: 3 }).default(sql`NULL`), userId: int('user_id').notNull(), linkedUserId: int('linked_user_id').default(sql`NULL`), - linkedUserCommunityUuid: char("linked_user_community_uuid", { length: 36 }).default(sql`NULL`), - linkedUserGradidoId: char("linked_user_gradido_id", { length: 36 }).default(sql`NULL`), + linkedUserCommunityUuid: char('linked_user_community_uuid', { length: 36 }).default(sql`NULL`), + linkedUserGradidoId: char('linked_user_gradido_id', { length: 36 }).default(sql`NULL`), linkedTransactionId: int('linked_transaction_id').default(sql`NULL`), }, (table) => [index('user_id').on(table.userId)], @@ -95,12 +95,12 @@ export const transactionLinksTable = mysqlTable('transaction_links', { id: int().autoincrement().notNull(), userId: int().notNull(), amount: decimal({ precision: 40, scale: 20 }).notNull(), - holdAvailableAmount: decimal("hold_available_amount", { precision: 40, scale: 20 }).notNull(), + holdAvailableAmount: decimal('hold_available_amount', { precision: 40, scale: 20 }).notNull(), memo: varchar({ length: 255 }).notNull(), code: varchar({ length: 24 }).notNull(), createdAt: datetime({ mode: 'string' }).notNull(), deletedAt: datetime({ mode: 'string' }).default(sql`NULL`), validUntil: datetime({ mode: 'string' }).notNull(), - redeemedAt: datetime({ mode: 'string'}).default(sql`NULL`), - redeemedBy: int().default(sql`NULL`), + redeemedAt: datetime({ mode: 'string' }).default(sql`NULL`), + redeemedBy: int().default(sql`NULL`), }) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/errors.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/errors.ts index 0c18102f6..8e09c6995 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/errors.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/errors.ts @@ -1,8 +1,13 @@ import * as v from 'valibot' export class NotEnoughGradidoBalanceError extends Error { - constructor(public needed: number, public exist: number) { - super(`Not enough Gradido Balance for send coins, needed: ${needed} Gradido, exist: ${exist} Gradido`) + constructor( + public needed: number, + public exist: number, + ) { + super( + `Not enough Gradido Balance for send coins, needed: ${needed} Gradido, exist: ${exist} Gradido`, + ) this.name = 'NotEnoughGradidoBalanceError' } } @@ -46,7 +51,12 @@ export class BlockchainError extends Error { } export class NegativeBalanceError extends Error { - constructor(message: string, previousBalanceString: string, amount: string, previousDecayedBalance: string) { + constructor( + message: string, + previousBalanceString: string, + amount: string, + previousDecayedBalance: string, + ) { const parts: string[] = [`NegativeBalanceError in ${message}`] parts.push(`Previous balance: ${previousBalanceString}`) parts.push(`Amount: ${amount}`) @@ -54,4 +64,4 @@ export class NegativeBalanceError extends Error { super(parts.join('\n')) this.name = 'NegativeBalanceError' } -} \ No newline at end of file +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts index aa418625d..1576f9770 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts @@ -1,15 +1,16 @@ +import { Filter, Profiler, ThreadingPolicy_Half, verifySignatures } from 'gradido-blockchain-js' import { onShutdown } from '../../../../shared/src/helper/onShutdown' import { exportAllCommunities } from './binaryExport' import { bootstrap } from './bootstrap' import { syncDbWithBlockchainContext } from './interaction/syncDbWithBlockchain/syncDbWithBlockchain.context' -import { Filter, Profiler, ThreadingPolicy_Half, verifySignatures } from 'gradido-blockchain-js' - // import { hello } from '../../../zig/hello.zig' + +// import { hello } from '../../../zig/hello.zig' const BATCH_SIZE = 1000 async function main() { // hello() - // return + // return // prepare in memory blockchains const context = await bootstrap() onShutdown(async (reason, error) => { @@ -22,8 +23,8 @@ async function main() { // synchronize to in memory blockchain try { await syncDbWithBlockchainContext(context, BATCH_SIZE) - } catch(e) { - console.error(e) + } catch (e) { + context.logger.error(e) //context.logBlogchain(v.parse(uuidv4Schema, 'e70da33e-5976-4767-bade-aa4e4fa1c01a')) } @@ -31,9 +32,15 @@ async function main() { // bulk verify transaction signatures for (const communityContext of context.communities.values()) { // verifySignatures(Filter.ALL_TRANSACTIONS, ThreadingPolicy_Half) - const result = verifySignatures(Filter.ALL_TRANSACTIONS, communityContext.communityId, ThreadingPolicy_Half) - if(!result.isEmpty()){ - throw new Error(`Verification of signatures failed for community ${communityContext.communityId}`) + const result = verifySignatures( + Filter.ALL_TRANSACTIONS, + communityContext.communityId, + ThreadingPolicy_Half, + ) + if (!result.isEmpty()) { + throw new Error( + `Verification of signatures failed for community ${communityContext.communityId}`, + ) } } context.logger.info(`verified in ${timeUsed.string()}`) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts index d6a6fc23a..572252fd9 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts @@ -1,12 +1,12 @@ -import { - AccountBalances, - Filter, - InMemoryBlockchain, - KeyPairEd25519, - MemoryBlockPtr, - Profiler, - SearchDirection_DESC, - GradidoTransactionBuilder +import { + AccountBalances, + Filter, + GradidoTransactionBuilder, + InMemoryBlockchain, + KeyPairEd25519, + MemoryBlockPtr, + Profiler, + SearchDirection_DESC, } from 'gradido-blockchain-js' import { getLogger, Logger } from 'log4js' import { LOG4JS_BASE_CATEGORY } from '../../../../config/const' @@ -21,7 +21,7 @@ export type IndexType = { id: number } export let nanosBalanceForUser = 0 -const lastBalanceOfUserTimeUsed = new Profiler +const lastBalanceOfUserTimeUsed = new Profiler() export abstract class AbstractSyncRole { private items: ItemType[] = [] @@ -40,16 +40,22 @@ export abstract class AbstractSyncRole { getAccountKeyPair(communityContext: CommunityContext, gradidoId: Uuidv4): KeyPairEd25519 { return this.context.cache.getKeyPairSync(gradidoId, () => { - return deriveFromKeyPairAndIndex(deriveFromKeyPairAndUuid(communityContext.keyPair, gradidoId), 1) + return deriveFromKeyPairAndIndex( + deriveFromKeyPairAndUuid(communityContext.keyPair, gradidoId), + 1, + ) }) } - getLastBalanceForUser(publicKey: MemoryBlockPtr, blockchain: InMemoryBlockchain, communityId: string + getLastBalanceForUser( + publicKey: MemoryBlockPtr, + blockchain: InMemoryBlockchain, + communityId: string, ): Balance { lastBalanceOfUserTimeUsed.reset() if (publicKey.isEmpty()) { throw new Error('publicKey is empty') - } + } const f = Filter.lastBalanceFor(publicKey) f.setCommunityId(communityId) const lastSenderTransaction = blockchain.findOne(f) @@ -58,19 +64,31 @@ export abstract class AbstractSyncRole { } const lastConfirmedTransaction = lastSenderTransaction.getConfirmedTransaction() if (!lastConfirmedTransaction) { - throw new Error(`invalid transaction, getConfirmedTransaction call failed for transaction nr: ${lastSenderTransaction.getTransactionNr()}`) - + throw new Error( + `invalid transaction, getConfirmedTransaction call failed for transaction nr: ${lastSenderTransaction.getTransactionNr()}`, + ) } - const senderLastAccountBalance = lastConfirmedTransaction.getAccountBalance(publicKey, communityId) + const senderLastAccountBalance = lastConfirmedTransaction.getAccountBalance( + publicKey, + communityId, + ) if (!senderLastAccountBalance) { return new Balance(publicKey, communityId) } - const result = Balance.fromAccountBalance(senderLastAccountBalance, lastConfirmedTransaction.getConfirmedAt().getDate(), communityId) + const result = Balance.fromAccountBalance( + senderLastAccountBalance, + lastConfirmedTransaction.getConfirmedAt().getDate(), + communityId, + ) nanosBalanceForUser += lastBalanceOfUserTimeUsed.nanos() return result } - logLastBalanceChangingTransactions(publicKey: MemoryBlockPtr, blockchain: InMemoryBlockchain, transactionCount: number = 5) { + logLastBalanceChangingTransactions( + publicKey: MemoryBlockPtr, + blockchain: InMemoryBlockchain, + transactionCount: number = 5, + ) { if (!this.context.logger.isDebugEnabled()) { return } @@ -111,7 +129,7 @@ export abstract class AbstractSyncRole { return this.items.length } return 0 - } + } toBlockchain(): void { if (this.isEmpty()) { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts index 299f4cec4..4e149f583 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts @@ -1,14 +1,14 @@ import { and, asc, eq, gt, isNotNull, or } from 'drizzle-orm' import * as v from 'valibot' import { Context } from '../../Context' -import { contributionLinkModerators } from '../../database' -import { CreationTransactionDb, creationTransactionDbSchema } from '../../valibot.schema' -import { CreationsSyncRole } from './CreationsSync.role' -import { contributionsTable, usersTable } from '../../drizzle.schema' import { ContributionStatus } from '../../data/ContributionStatus' +import { contributionLinkModerators } from '../../database' +import { contributionsTable, usersTable } from '../../drizzle.schema' import { DatabaseError } from '../../errors' -import { IndexType } from './AbstractSync.role' import { toMysqlDateTime } from '../../utils' +import { CreationTransactionDb, creationTransactionDbSchema } from '../../valibot.schema' +import { IndexType } from './AbstractSync.role' +import { CreationsSyncRole } from './CreationsSync.role' export class ContributionLinkTransactionSyncRole extends CreationsSyncRole { constructor(readonly context: Context) { @@ -25,30 +25,34 @@ export class ContributionLinkTransactionSyncRole extends CreationsSyncRole { user: usersTable, }) .from(contributionsTable) - .where(and( - isNotNull(contributionsTable.contributionLinkId), - eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED), - or( - gt(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), - and( - eq(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), - gt(contributionsTable.transactionId, lastIndex.id) - ) - ) - )) + .where( + and( + isNotNull(contributionsTable.contributionLinkId), + eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED), + or( + gt(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), + and( + eq(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), + gt(contributionsTable.transactionId, lastIndex.id), + ), + ), + ), + ) .innerJoin(usersTable, eq(contributionsTable.userId, usersTable.id)) .orderBy(asc(contributionsTable.confirmedAt), asc(contributionsTable.transactionId)) .limit(count) - + const verifiedCreationTransactions: CreationTransactionDb[] = [] - for(const row of result) { + for (const row of result) { if (!row.contribution.contributionLinkId) { - throw new Error(`expect contributionLinkId to be set: ${JSON.stringify(row.contribution, null, 2)}`) + throw new Error( + `expect contributionLinkId to be set: ${JSON.stringify(row.contribution, null, 2)}`, + ) } const item = { ...row.contribution, user: row.user, - confirmedByUser: contributionLinkModerators.get(row.contribution.contributionLinkId) + confirmedByUser: contributionLinkModerators.get(row.contribution.contributionLinkId), } if (!item.confirmedByUser || item.userId === item.confirmedByUser.id) { this.context.logger.warn(`skipped Contribution Link Transaction ${row.contribution.id}`) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts index cb4e9ca14..b5555c3a2 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts @@ -1,33 +1,33 @@ -import { and, asc, eq, isNull, gt, or } from 'drizzle-orm' +import { and, asc, eq, gt, isNull, or } from 'drizzle-orm' import { alias } from 'drizzle-orm/mysql-core' -import { - AccountBalances, - AuthenticatedEncryption, - EncryptedMemo, - Filter, - GradidoTransactionBuilder, - KeyPairEd25519, - LedgerAnchor, - MemoryBlockPtr, - SearchDirection_DESC, - TransactionType_CREATION, - TransferAmount +import { + AccountBalances, + AuthenticatedEncryption, + EncryptedMemo, + Filter, + GradidoTransactionBuilder, + KeyPairEd25519, + LedgerAnchor, + MemoryBlockPtr, + SearchDirection_DESC, + TransactionType_CREATION, + TransferAmount, } from 'gradido-blockchain-js' import * as v from 'valibot' import { addToBlockchain } from '../../blockchain' -import { ContributionStatus } from '../../data/ContributionStatus' import { Context } from '../../Context' -import { - contributionsTable, - usersTable -} from '../../drizzle.schema' +import { ContributionStatus } from '../../data/ContributionStatus' +import { contributionsTable, usersTable } from '../../drizzle.schema' import { BlockchainError, DatabaseError } from '../../errors' -import { CommunityContext, CreationTransactionDb, creationTransactionDbSchema } from '../../valibot.schema' -import { AbstractSyncRole, IndexType } from './AbstractSync.role' import { toMysqlDateTime } from '../../utils' +import { + CommunityContext, + CreationTransactionDb, + creationTransactionDbSchema, +} from '../../valibot.schema' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' -export class CreationsSyncRole extends AbstractSyncRole { - +export class CreationsSyncRole extends AbstractSyncRole { constructor(context: Context) { super(context) this.accountBalances.reserve(3) @@ -38,16 +38,16 @@ export class CreationsSyncRole extends AbstractSyncRole { } getLastIndex(): IndexType { - const lastItem = this.peekLast() - return { date: lastItem.confirmedAt, id: lastItem.transactionId } - } + const lastItem = this.peekLast() + return { date: lastItem.confirmedAt, id: lastItem.transactionId } + } itemTypeName(): string { return 'creationTransactions' } async loadFromDb(lastIndex: IndexType, count: number): Promise { - const confirmedByUsers = alias(usersTable, 'confirmedByUser') + const confirmedByUsers = alias(usersTable, 'confirmedByUser') const result = await this.context.db .select({ contribution: contributionsTable, @@ -55,22 +55,24 @@ export class CreationsSyncRole extends AbstractSyncRole { confirmedByUser: confirmedByUsers, }) .from(contributionsTable) - .where(and( - isNull(contributionsTable.contributionLinkId), - eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED), - or( - gt(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), - and( - eq(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), - gt(contributionsTable.transactionId, lastIndex.id) - ) - ) - )) + .where( + and( + isNull(contributionsTable.contributionLinkId), + eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED), + or( + gt(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), + and( + eq(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), + gt(contributionsTable.transactionId, lastIndex.id), + ), + ), + ), + ) .innerJoin(usersTable, eq(contributionsTable.userId, usersTable.id)) .innerJoin(confirmedByUsers, eq(contributionsTable.confirmedBy, confirmedByUsers.id)) .orderBy(asc(contributionsTable.confirmedAt), asc(contributionsTable.transactionId)) .limit(count) - + return result.map((row) => { const item = { ...row.contribution, @@ -86,12 +88,12 @@ export class CreationsSyncRole extends AbstractSyncRole { } buildTransaction( - item: CreationTransactionDb, - communityContext: CommunityContext, - recipientKeyPair: KeyPairEd25519, - signerKeyPair: KeyPairEd25519 + item: CreationTransactionDb, + communityContext: CommunityContext, + recipientKeyPair: KeyPairEd25519, + signerKeyPair: KeyPairEd25519, ): GradidoTransactionBuilder { - return this.transactionBuilder + return this.transactionBuilder .setCreatedAt(item.confirmedAt) .setRecipientCommunity(communityContext.communityId) .addMemo( @@ -102,19 +104,27 @@ export class CreationsSyncRole extends AbstractSyncRole { ), ) .setTransactionCreation( - new TransferAmount(recipientKeyPair.getPublicKey(), item.amount, communityContext.communityId), + new TransferAmount( + recipientKeyPair.getPublicKey(), + item.amount, + communityContext.communityId, + ), item.contributionDate, ) .sign(signerKeyPair) } calculateAccountBalances( - item: CreationTransactionDb, - communityContext: CommunityContext, - recipientPublicKey: MemoryBlockPtr + item: CreationTransactionDb, + communityContext: CommunityContext, + recipientPublicKey: MemoryBlockPtr, ): AccountBalances { this.accountBalances.clear() - const balance = this.getLastBalanceForUser(recipientPublicKey, communityContext.blockchain, communityContext.communityId) + const balance = this.getLastBalanceForUser( + recipientPublicKey, + communityContext.blockchain, + communityContext.communityId, + ) // calculate decay since last balance with legacy calculation method balance.updateLegacyDecay(item.amount, item.confirmedAt) @@ -131,9 +141,11 @@ export class CreationsSyncRole extends AbstractSyncRole { const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) const blockchain = communityContext.blockchain if (item.confirmedByUser.communityUuid !== item.user.communityUuid) { - throw new Error(`contribution was confirmed from other community: ${JSON.stringify(item, null, 2)}`) + throw new Error( + `contribution was confirmed from other community: ${JSON.stringify(item, null, 2)}`, + ) } - + const recipientKeyPair = this.getAccountKeyPair(communityContext, item.user.gradidoId) const recipientPublicKey = recipientKeyPair.getPublicKey() const signerKeyPair = this.getAccountKeyPair(communityContext, item.confirmedByUser.gradidoId) @@ -148,14 +160,16 @@ export class CreationsSyncRole extends AbstractSyncRole { new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_CONTRIBUTION_ID), this.calculateAccountBalances(item, communityContext, recipientPublicKey), ) - } catch(e) { - const f= new Filter() + } catch (e) { + const f = new Filter() f.transactionType = TransactionType_CREATION f.searchDirection = SearchDirection_DESC f.pagination.size = 1 const lastContribution = blockchain.findOne(f) if (lastContribution) { - this.context.logger.warn(`last contribution: ${lastContribution.getConfirmedTransaction()?.toJson(true)}`) + this.context.logger.warn( + `last contribution: ${lastContribution.getConfirmedTransaction()?.toJson(true)}`, + ) } throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts index 7e609ff2c..4aa5159af 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts @@ -1,34 +1,38 @@ -import { CommunityContext, DeletedTransactionLinkDb, deletedTransactionLinKDbSchema } from '../../valibot.schema' -import { AbstractSyncRole, IndexType } from './AbstractSync.role' -import { transactionLinksTable, usersTable } from '../../drizzle.schema' -import { and, lt, asc, isNotNull, eq, or, gt } from 'drizzle-orm' -import * as v from 'valibot' -import { - AccountBalance, - AccountBalances, - Filter, - GradidoDeferredTransfer, +import { and, asc, eq, gt, isNotNull, lt, or } from 'drizzle-orm' +import { + AccountBalance, + AccountBalances, + Filter, + GradidoDeferredTransfer, GradidoTransactionBuilder, - GradidoTransfer, - GradidoUnit, - KeyPairEd25519, - LedgerAnchor, + GradidoTransfer, + GradidoUnit, + KeyPairEd25519, + LedgerAnchor, MemoryBlockPtr, - TransferAmount + TransferAmount, } from 'gradido-blockchain-js' +import * as v from 'valibot' import { deriveFromCode } from '../../../../data/deriveKeyPair' import { addToBlockchain } from '../../blockchain' -import { BlockchainError, DatabaseError } from '../../errors' -import { Balance } from '../../data/Balance' -import { toMysqlDateTime } from '../../utils' import { Context } from '../../Context' +import { Balance } from '../../data/Balance' +import { transactionLinksTable, usersTable } from '../../drizzle.schema' +import { BlockchainError, DatabaseError } from '../../errors' +import { toMysqlDateTime } from '../../utils' +import { + CommunityContext, + DeletedTransactionLinkDb, + deletedTransactionLinKDbSchema, +} from '../../valibot.schema' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { constructor(context: Context) { super(context) this.accountBalances.reserve(2) } - + getDate(): Date { return this.peek().deletedAt } @@ -57,15 +61,15 @@ export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { const item = { ...row.transactionLink, @@ -80,95 +84,118 @@ export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { constructor(context: Context) { super(context) this.accountBalances.reserve(2) } - + getDate(): Date { return this.peek().balanceDate } @@ -58,74 +63,83 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { or( gt(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)), and( - eq(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)), - gt(transactionsTable.id, lastIndex.id) - ) - ) - ) + eq(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)), + gt(transactionsTable.id, lastIndex.id), + ), + ), + ), ) .innerJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) .innerJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id)) .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) .limit(count) - + return result.map((row) => { const item = { - ...row.transaction, - user: row.user, - linkedUser: row.linkedUser, - } + ...row.transaction, + user: row.user, + linkedUser: row.linkedUser, + } try { return v.parse(transactionDbSchema, item) } catch (e) { throw new DatabaseError('loadLocalTransferTransactions', item, e as Error) } - }) + }) } buildTransaction( - communityContext: CommunityContext, - item: TransactionDb, - senderKeyPair: KeyPairEd25519, - recipientKeyPair: KeyPairEd25519, - ): GradidoTransactionBuilder { - return this.transactionBuilder - .setCreatedAt(item.balanceDate) - .addMemo(new EncryptedMemo( - item.memo, - new AuthenticatedEncryption(senderKeyPair), - new AuthenticatedEncryption(recipientKeyPair), - ), - ) - .setSenderCommunity(communityContext.communityId) - .setTransactionTransfer( - new TransferAmount(senderKeyPair.getPublicKey(), item.amount, communityContext.communityId), - recipientKeyPair.getPublicKey(), - ) - .sign(senderKeyPair) + communityContext: CommunityContext, + item: TransactionDb, + senderKeyPair: KeyPairEd25519, + recipientKeyPair: KeyPairEd25519, + ): GradidoTransactionBuilder { + return this.transactionBuilder + .setCreatedAt(item.balanceDate) + .addMemo( + new EncryptedMemo( + item.memo, + new AuthenticatedEncryption(senderKeyPair), + new AuthenticatedEncryption(recipientKeyPair), + ), + ) + .setSenderCommunity(communityContext.communityId) + .setTransactionTransfer( + new TransferAmount(senderKeyPair.getPublicKey(), item.amount, communityContext.communityId), + recipientKeyPair.getPublicKey(), + ) + .sign(senderKeyPair) } calculateBalances( - item: TransactionDb, + item: TransactionDb, communityContext: CommunityContext, senderPublicKey: MemoryBlockPtr, recipientPublicKey: MemoryBlockPtr, ): AccountBalances { this.accountBalances.clear() - - const senderLastBalance = this.getLastBalanceForUser(senderPublicKey, communityContext.blockchain, communityContext.communityId) - const recipientLastBalance = this.getLastBalanceForUser(recipientPublicKey, communityContext.blockchain, communityContext.communityId) + + const senderLastBalance = this.getLastBalanceForUser( + senderPublicKey, + communityContext.blockchain, + communityContext.communityId, + ) + const recipientLastBalance = this.getLastBalanceForUser( + recipientPublicKey, + communityContext.blockchain, + communityContext.communityId, + ) try { senderLastBalance.updateLegacyDecay(item.amount.negated(), item.balanceDate) - } catch(e) { + } catch (e) { if (e instanceof NegativeBalanceError) { this.logLastBalanceChangingTransactions(senderPublicKey, communityContext.blockchain) throw e } } recipientLastBalance.updateLegacyDecay(item.amount, item.balanceDate) - + this.accountBalances.add(senderLastBalance.getAccountBalance()) this.accountBalances.add(recipientLastBalance.getAccountBalance()) return this.accountBalances @@ -135,15 +149,17 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) const blockchain = communityContext.blockchain if (item.linkedUser.communityUuid !== item.user.communityUuid) { - throw new Error(`transfer between user from different communities: ${JSON.stringify(item, null, 2)}`) + throw new Error( + `transfer between user from different communities: ${JSON.stringify(item, null, 2)}`, + ) } - + // I use the received transaction so user and linked user are swapped and user is recipient and linkedUser ist sender const senderKeyPair = this.getAccountKeyPair(communityContext, item.linkedUser.gradidoId) const senderPublicKey = senderKeyPair.getPublicKey() const recipientKeyPair = this.getAccountKeyPair(communityContext, item.user.gradidoId) const recipientPublicKey = recipientKeyPair.getPublicKey() - + if (!senderKeyPair || !senderPublicKey || !recipientKeyPair || !recipientPublicKey) { throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`) } @@ -155,9 +171,9 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_ID), this.calculateBalances(item, communityContext, senderPublicKey, recipientPublicKey), ) - } catch(e) { + } catch (e) { if (e instanceof NotEnoughGradidoBalanceError) { - this.logLastBalanceChangingTransactions(senderPublicKey, blockchain) + this.logLastBalanceChangingTransactions(senderPublicKey, blockchain) } throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts index e6d9d0c75..6e0c9f6ba 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts @@ -1,36 +1,40 @@ -import { and, asc, eq, isNotNull, isNull, or, gt } from 'drizzle-orm' -import { +import { and, asc, eq, gt, isNotNull, isNull, or } from 'drizzle-orm' +import { alias } from 'drizzle-orm/mysql-core' +import { AccountBalance, - AccountBalances, - AuthenticatedEncryption, - EncryptedMemo, + AccountBalances, + AuthenticatedEncryption, + EncryptedMemo, Filter, GradidoDeferredTransfer, - GradidoTransactionBuilder, - GradidoTransfer, - GradidoUnit, - KeyPairEd25519, - LedgerAnchor, - MemoryBlockPtr, - TransferAmount + GradidoTransactionBuilder, + GradidoTransfer, + GradidoUnit, + KeyPairEd25519, + LedgerAnchor, + MemoryBlockPtr, + TransferAmount, } from 'gradido-blockchain-js' import * as v from 'valibot' +import { deriveFromCode } from '../../../../data/deriveKeyPair' import { addToBlockchain } from '../../blockchain' +import { Context } from '../../Context' import { transactionLinksTable, usersTable } from '../../drizzle.schema' import { BlockchainError, DatabaseError } from '../../errors' -import { CommunityContext, RedeemedTransactionLinkDb, redeemedTransactionLinkDbSchema } from '../../valibot.schema' -import { AbstractSyncRole, IndexType } from './AbstractSync.role' -import { deriveFromCode } from '../../../../data/deriveKeyPair' -import { alias } from 'drizzle-orm/mysql-core' import { toMysqlDateTime } from '../../utils' -import { Context } from '../../Context' +import { + CommunityContext, + RedeemedTransactionLinkDb, + redeemedTransactionLinkDbSchema, +} from '../../valibot.schema' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' -export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { +export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { constructor(context: Context) { super(context) this.accountBalances.reserve(3) } - + getDate(): Date { return this.peek().redeemedAt } @@ -62,17 +66,17 @@ export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { const item = { ...row.transactionLink, @@ -88,66 +92,74 @@ export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { constructor(context: Context) { super(context) this.accountBalances.reserve(1) } - + getDate(): Date { return this.peek().balanceDate } @@ -50,23 +64,23 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole or( gt(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)), and( - eq(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)), - gt(transactionsTable.id, lastIndex.id) - ) - ) - ) + eq(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)), + gt(transactionsTable.id, lastIndex.id), + ), + ), + ), ) .innerJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) .innerJoin(linkedUsers, eq(transactionsTable.linkedUserGradidoId, linkedUsers.gradidoId)) .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) .limit(count) - + return result.map((row) => { const item = { - ...row.transaction, - user: row.user, - linkedUser: row.linkedUser, - } + ...row.transaction, + user: row.user, + linkedUser: row.linkedUser, + } if (item.typeId === TransactionTypeId.SEND && item.amount) { item.amount = new Decimal(item.amount).neg().toString() } @@ -75,101 +89,120 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole } catch (e) { throw new DatabaseError('loadRemoteTransferTransactions', item, e as Error) } - }) + }) } buildTransaction( - item: TransactionDb, - senderKeyPair: KeyPairEd25519, - recipientKeyPair: KeyPairEd25519, - senderCommunityId: string, - recipientCommunityId: string, - ): GradidoTransactionBuilder { - return this.transactionBuilder - .setCreatedAt(item.balanceDate) - .addMemo(new EncryptedMemo( - item.memo, - new AuthenticatedEncryption(senderKeyPair), - new AuthenticatedEncryption(recipientKeyPair), - ), - ) - .setSenderCommunity(senderCommunityId) - .setRecipientCommunity(recipientCommunityId) - .setTransactionTransfer( - new TransferAmount(senderKeyPair.getPublicKey(), item.amount, senderCommunityId), - recipientKeyPair.getPublicKey(), - ) - .sign(senderKeyPair) + item: TransactionDb, + senderKeyPair: KeyPairEd25519, + recipientKeyPair: KeyPairEd25519, + senderCommunityId: string, + recipientCommunityId: string, + ): GradidoTransactionBuilder { + return this.transactionBuilder + .setCreatedAt(item.balanceDate) + .addMemo( + new EncryptedMemo( + item.memo, + new AuthenticatedEncryption(senderKeyPair), + new AuthenticatedEncryption(recipientKeyPair), + ), + ) + .setSenderCommunity(senderCommunityId) + .setRecipientCommunity(recipientCommunityId) + .setTransactionTransfer( + new TransferAmount(senderKeyPair.getPublicKey(), item.amount, senderCommunityId), + recipientKeyPair.getPublicKey(), + ) + .sign(senderKeyPair) } calculateBalances( - item: TransactionDb, - communityContext: CommunityContext, - coinCommunityId: string, - amount: GradidoUnit, - publicKey: MemoryBlockPtr, - ): AccountBalances { - this.accountBalances.clear() - if (communityContext.foreign) { - this.accountBalances.add(new AccountBalance(publicKey, GradidoUnit.zero(), coinCommunityId)) - return this.accountBalances - } else { - // try to use same coins from this community - let lastBalance = this.getLastBalanceForUser(publicKey, communityContext.blockchain, coinCommunityId) - if (lastBalance.getBalance().equal(GradidoUnit.zero()) || lastBalance.getBalance().calculateDecay(lastBalance.getDate(), item.balanceDate).lt(amount)) { - // don't work, so we use or own coins - lastBalance = this.getLastBalanceForUser(publicKey, communityContext.blockchain, communityContext.communityId) - } - - try { - lastBalance.updateLegacyDecay(amount, item.balanceDate) - } catch(e) { - if (e instanceof NegativeBalanceError) { - console.log(`coin community id: ${coinCommunityId}, context community id: ${communityContext.communityId}`) - this.logLastBalanceChangingTransactions(publicKey, communityContext.blockchain, 1) - throw e - } - } - this.accountBalances.add(lastBalance.getAccountBalance()) - return this.accountBalances + item: TransactionDb, + communityContext: CommunityContext, + coinCommunityId: string, + amount: GradidoUnit, + publicKey: MemoryBlockPtr, + ): AccountBalances { + this.accountBalances.clear() + if (communityContext.foreign) { + this.accountBalances.add(new AccountBalance(publicKey, GradidoUnit.zero(), coinCommunityId)) + return this.accountBalances + } else { + // try to use same coins from this community + let lastBalance = this.getLastBalanceForUser( + publicKey, + communityContext.blockchain, + coinCommunityId, + ) + if ( + lastBalance.getBalance().equal(GradidoUnit.zero()) || + lastBalance.getBalance().calculateDecay(lastBalance.getDate(), item.balanceDate).lt(amount) + ) { + // don't work, so we use or own coins + lastBalance = this.getLastBalanceForUser( + publicKey, + communityContext.blockchain, + communityContext.communityId, + ) } + + try { + lastBalance.updateLegacyDecay(amount, item.balanceDate) + } catch (e) { + if (e instanceof NegativeBalanceError) { + this.logLastBalanceChangingTransactions(publicKey, communityContext.blockchain, 1) + throw e + } + } + this.accountBalances.add(lastBalance.getAccountBalance()) + return this.accountBalances + } } - getUser(item: TransactionDb): { senderUser: UserDb, recipientUser: UserDb } { - return ( - item.typeId === TransactionTypeId.RECEIVE - ? { senderUser: item.linkedUser, recipientUser: item.user } - : { senderUser: item.user, recipientUser: item.linkedUser } - ) + getUser(item: TransactionDb): { senderUser: UserDb; recipientUser: UserDb } { + return item.typeId === TransactionTypeId.RECEIVE + ? { senderUser: item.linkedUser, recipientUser: item.user } + : { senderUser: item.user, recipientUser: item.linkedUser } } pushToBlockchain(item: TransactionDb): void { const { senderUser, recipientUser } = this.getUser(item) - const ledgerAnchor = new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_ID) + const ledgerAnchor = new LedgerAnchor( + item.id, + LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_ID, + ) if (senderUser.communityUuid === recipientUser.communityUuid) { - throw new Error(`transfer between user from same community: ${JSON.stringify(item, null, 2)}, check db query`) + throw new Error( + `transfer between user from same community: ${JSON.stringify(item, null, 2)}, check db query`, + ) } const senderCommunityContext = this.context.getCommunityContextByUuid(senderUser.communityUuid) - const recipientCommunityContext = this.context.getCommunityContextByUuid(recipientUser.communityUuid) + const recipientCommunityContext = this.context.getCommunityContextByUuid( + recipientUser.communityUuid, + ) const senderBlockchain = senderCommunityContext.blockchain const recipientBlockchain = recipientCommunityContext.blockchain - + // I use the received transaction so user and linked user are swapped and user is recipient and linkedUser ist sender const senderKeyPair = this.getAccountKeyPair(senderCommunityContext, senderUser.gradidoId) const senderPublicKey = senderKeyPair.getPublicKey() - const recipientKeyPair = this.getAccountKeyPair(recipientCommunityContext, recipientUser.gradidoId) + const recipientKeyPair = this.getAccountKeyPair( + recipientCommunityContext, + recipientUser.gradidoId, + ) const recipientPublicKey = recipientKeyPair.getPublicKey() - + if (!senderKeyPair || !senderPublicKey || !recipientKeyPair || !recipientPublicKey) { throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`) } const transactionBuilder = this.buildTransaction( - item, - senderKeyPair, - recipientKeyPair, - senderCommunityContext.communityId, - recipientCommunityContext.communityId + item, + senderKeyPair, + recipientKeyPair, + senderCommunityContext.communityId, + recipientCommunityContext.communityId, ) const outboundTransaction = transactionBuilder.buildOutbound() @@ -178,11 +211,17 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole outboundTransaction, senderBlockchain, ledgerAnchor, - this.calculateBalances(item, senderCommunityContext, senderCommunityContext.communityId, item.amount.negated(), senderPublicKey), + this.calculateBalances( + item, + senderCommunityContext, + senderCommunityContext.communityId, + item.amount.negated(), + senderPublicKey, + ), ) - } catch(e) { + } catch (e) { if (e instanceof NotEnoughGradidoBalanceError) { - this.logLastBalanceChangingTransactions(senderPublicKey, senderBlockchain) + this.logLastBalanceChangingTransactions(senderPublicKey, senderBlockchain) } throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) } @@ -193,11 +232,17 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole inboundTransaction, recipientBlockchain, ledgerAnchor, - this.calculateBalances(item, recipientCommunityContext, senderCommunityContext.communityId, item.amount, recipientPublicKey), + this.calculateBalances( + item, + recipientCommunityContext, + senderCommunityContext.communityId, + item.amount, + recipientPublicKey, + ), ) - } catch(e) { + } catch (e) { if (e instanceof NotEnoughGradidoBalanceError) { - this.logLastBalanceChangingTransactions(recipientPublicKey, recipientBlockchain) + this.logLastBalanceChangingTransactions(recipientPublicKey, recipientBlockchain) } throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts index ddbddd9d4..918d33c3b 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts @@ -1,28 +1,28 @@ -import { asc, eq, or, gt, and } from 'drizzle-orm' -import { +import Decimal from 'decimal.js-light' +import { and, asc, eq, gt, or } from 'drizzle-orm' +import { AccountBalance, - AccountBalances, - AuthenticatedEncryption, - DurationSeconds, - EncryptedMemo, - GradidoTransactionBuilder, - GradidoTransfer, - GradidoUnit, - KeyPairEd25519, - LedgerAnchor, - MemoryBlockPtr, - TransferAmount + AccountBalances, + AuthenticatedEncryption, + DurationSeconds, + EncryptedMemo, + GradidoTransactionBuilder, + GradidoTransfer, + GradidoUnit, + KeyPairEd25519, + LedgerAnchor, + MemoryBlockPtr, + TransferAmount, } from 'gradido-blockchain-js' import * as v from 'valibot' +import { deriveFromCode } from '../../../../data/deriveKeyPair' import { addToBlockchain } from '../../blockchain' +import { Context } from '../../Context' import { transactionLinksTable, usersTable } from '../../drizzle.schema' import { BlockchainError, DatabaseError, NegativeBalanceError } from '../../errors' +import { reverseLegacyDecay, toMysqlDateTime } from '../../utils' import { CommunityContext, TransactionLinkDb, transactionLinkDbSchema } from '../../valibot.schema' import { AbstractSyncRole, IndexType } from './AbstractSync.role' -import { deriveFromCode } from '../../../../data/deriveKeyPair' -import { reverseLegacyDecay, toMysqlDateTime } from '../../utils' -import Decimal from 'decimal.js-light' -import { Context } from '../../Context' export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { constructor(context: Context) { @@ -47,16 +47,18 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { const item = { ...row.transaction_links, @@ -72,11 +74,11 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { constructor(context: Context) { @@ -40,20 +40,22 @@ export class UsersSyncRole extends AbstractSyncRole { async loadFromDb(lastIndex: IndexType, count: number): Promise { const result = await this.context.db - .select() - .from(usersTable) - .where(and( + .select() + .from(usersTable) + .where( + and( or( gt(usersTable.createdAt, toMysqlDateTime(lastIndex.date)), and( eq(usersTable.createdAt, toMysqlDateTime(lastIndex.date)), - gt(usersTable.id, lastIndex.id) - ) - ) - )) - .orderBy(asc(usersTable.createdAt), asc(usersTable.id)) - .limit(count) - + gt(usersTable.id, lastIndex.id), + ), + ), + ), + ) + .orderBy(asc(usersTable.createdAt), asc(usersTable.id)) + .limit(count) + return result.map((row) => { try { return v.parse(userDbSchema, row) @@ -65,10 +67,10 @@ export class UsersSyncRole extends AbstractSyncRole { buildTransaction( communityContext: CommunityContext, - item: UserDb, - communityKeyPair: KeyPairEd25519, - accountKeyPair: KeyPairEd25519, - userKeyPair: KeyPairEd25519 + item: UserDb, + communityKeyPair: KeyPairEd25519, + accountKeyPair: KeyPairEd25519, + userKeyPair: KeyPairEd25519, ): GradidoTransactionBuilder { return this.transactionBuilder .setCreatedAt(item.createdAt) @@ -84,9 +86,14 @@ export class UsersSyncRole extends AbstractSyncRole { .sign(userKeyPair) } - calculateAccountBalances(accountPublicKey: MemoryBlockPtr, communityContext: CommunityContext,): AccountBalances { + calculateAccountBalances( + accountPublicKey: MemoryBlockPtr, + communityContext: CommunityContext, + ): AccountBalances { this.accountBalances.clear() - this.accountBalances.add(new AccountBalance(accountPublicKey, GradidoUnit.zero(), communityContext.communityId)) + this.accountBalances.add( + new AccountBalance(accountPublicKey, GradidoUnit.zero(), communityContext.communityId), + ) return this.accountBalances } @@ -101,7 +108,13 @@ export class UsersSyncRole extends AbstractSyncRole { try { addToBlockchain( - this.buildTransaction(communityContext, item, communityContext.keyPair, accountKeyPair, userKeyPair).build(), + this.buildTransaction( + communityContext, + item, + communityContext.keyPair, + accountKeyPair, + userKeyPair, + ).build(), communityContext.blockchain, new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_USER_ID), this.calculateAccountBalances(accountPublicKey, communityContext), diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts index 833b7d6f8..201b7427f 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts @@ -1,15 +1,15 @@ import { Profiler } from 'gradido-blockchain-js' -import { Context } from '../../Context' -import { CreationsSyncRole } from './CreationsSync.role' -import { LocalTransactionsSyncRole } from './LocalTransactionsSync.role' -import { UsersSyncRole } from './UsersSync.role' -import { TransactionLinkFundingsSyncRole } from './TransactionLinkFundingsSync.role' -import { RedeemTransactionLinksSyncRole } from './RedeemTransactionLinksSync.role' -import { ContributionLinkTransactionSyncRole } from './ContributionLinkTransactionSync.role' -import { DeletedTransactionLinksSyncRole } from './DeletedTransactionLinksSync.role' -import { RemoteTransactionsSyncRole } from './RemoteTransactionsSync.role' import { callTime } from '../../blockchain' +import { Context } from '../../Context' import { nanosBalanceForUser } from './AbstractSync.role' +import { ContributionLinkTransactionSyncRole } from './ContributionLinkTransactionSync.role' +import { CreationsSyncRole } from './CreationsSync.role' +import { DeletedTransactionLinksSyncRole } from './DeletedTransactionLinksSync.role' +import { LocalTransactionsSyncRole } from './LocalTransactionsSync.role' +import { RedeemTransactionLinksSyncRole } from './RedeemTransactionLinksSync.role' +import { RemoteTransactionsSyncRole } from './RemoteTransactionsSync.role' +import { TransactionLinkFundingsSyncRole } from './TransactionLinkFundingsSync.role' +import { UsersSyncRole } from './UsersSync.role' export async function syncDbWithBlockchainContext(context: Context, batchSize: number) { const timeUsedDB = new Profiler() @@ -38,7 +38,7 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n const loadedItemsCount = results.reduce((acc, c) => acc + c, 0) // log only, if at least one new item was loaded if (loadedItemsCount && isDebug) { - context.logger.debug(`${loadedItemsCount} new items loaded from db in ${timeUsedDB.string()}`) + context.logger.debug(`${loadedItemsCount} new items loaded from db in ${timeUsedDB.string()}`) } // remove empty containers @@ -53,17 +53,21 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n available.sort((a, b) => a.getDate().getTime() - b.getDate().getTime()) // context.logger.debug(`sorted ${available.length} containers in ${sortTime.string()}`) } - available[0].toBlockchain() + available[0].toBlockchain() transactionsCount++ - if (isDebug) { - if (timeBetweenPrints.millis() > 100) { + if (isDebug) { + if (timeBetweenPrints.millis() > 100) { process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`) timeBetweenPrints.reset() } - transactionsCountSinceLastLog++ + transactionsCountSinceLastLog++ if (transactionsCountSinceLastLog >= batchSize) { - context.logger.debug(`${transactionsCountSinceLastLog} transactions added to blockchain in ${timeUsedBlockchain.string()}`) - context.logger.info(`Time for createAndConfirm: ${((callTime - lastPrintedCallTime) / 1000 / 1000).toFixed(2)} milliseconds`) + context.logger.debug( + `${transactionsCountSinceLastLog} transactions added to blockchain in ${timeUsedBlockchain.string()}`, + ) + context.logger.info( + `Time for createAndConfirm: ${((callTime - lastPrintedCallTime) / 1000 / 1000).toFixed(2)} milliseconds`, + ) lastPrintedCallTime = callTime timeUsedBlockchain.reset() transactionsCountSinceLastLog = 0 @@ -76,8 +80,14 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n } } } - process.stdout.write(`successfully added to blockchain: ${transactionsCount}\n`) - context.logger.info(`Synced ${transactionsCount} transactions to blockchain in ${timeUsedAll.string()}`) - context.logger.info(`Time for createAndConfirm: ${(callTime / 1000 / 1000 / 1000).toFixed(2)} seconds`) - context.logger.info(`Time for call lastBalance of user: ${(nanosBalanceForUser / 1000 / 1000 / 1000).toFixed(2)} seconds`) + process.stdout.write(`successfully added to blockchain: ${transactionsCount}\n`) + context.logger.info( + `Synced ${transactionsCount} transactions to blockchain in ${timeUsedAll.string()}`, + ) + context.logger.info( + `Time for createAndConfirm: ${(callTime / 1000 / 1000 / 1000).toFixed(2)} seconds`, + ) + context.logger.info( + `Time for call lastBalance of user: ${(nanosBalanceForUser / 1000 / 1000 / 1000).toFixed(2)} seconds`, + ) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/utils.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/utils.ts index 58d7c1133..86604b2c6 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/utils.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/utils.ts @@ -10,7 +10,7 @@ export function bytesToKbyte(bytes: number): string { } export function bytesString(bytes: number): string { - if (bytes > (1024 * 1024)) { + if (bytes > 1024 * 1024) { return `${bytesToMbyte(bytes)} MB` } else if (bytes > 1024) { return `${bytesToKbyte(bytes)} KB` @@ -50,7 +50,9 @@ export function legacyCalculateDecay(amount: Decimal, from: Date, to: Date): Dec const startBlockMs = DECAY_START_TIME.getTime() if (toMs < fromMs) { - throw new Error(`calculateDecay: to (${to.toISOString()}) < from (${from.toISOString()}), reverse decay calculation is invalid`) + throw new Error( + `calculateDecay: to (${to.toISOString()}) < from (${from.toISOString()}), reverse decay calculation is invalid`, + ) } // decay started after end date; no decay @@ -59,7 +61,7 @@ export function legacyCalculateDecay(amount: Decimal, from: Date, to: Date): Dec } // decay started before start date; decay for full duration let duration = (toMs - fromMs) / 1000 - + // decay started between start and end date; decay from decay start till end date if (startBlockMs >= fromMs) { duration = (toMs - startBlockMs) / 1000 diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/valibot.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/valibot.schema.ts index 904be605d..28cab0be7 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/valibot.schema.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/valibot.schema.ts @@ -1,3 +1,4 @@ +import Decimal from 'decimal.js-light' import { GradidoUnit, InMemoryBlockchain, KeyPairEd25519 } from 'gradido-blockchain-js' import * as v from 'valibot' import { booleanSchema, dateSchema } from '../../schemas/typeConverter.schema' @@ -9,7 +10,6 @@ import { } from '../../schemas/typeGuard.schema' import { Balance } from './data/Balance' import { TransactionTypeId } from './data/TransactionTypeId' -import Decimal from 'decimal.js-light' const positiveNumberSchema = v.pipe(v.number(), v.minValue(1)) @@ -51,53 +51,71 @@ export const transactionBaseSchema = v.object({ user: userDbSchema, }) +export const transactionDbSchema = v.pipe( + v.object({ + ...transactionBaseSchema.entries, + typeId: v.enum(TransactionTypeId), + balanceDate: dateSchema, + linkedUser: userDbSchema, + }), + v.custom((value: any) => { + if ( + value.user && + value.linkedUser && + !value.transactionLinkCode && + value.user.gradidoId === value.linkedUser.gradidoId + ) { + throw new Error( + `expect user to be different from linkedUser: ${JSON.stringify(value, null, 2)}`, + ) + } + // check that user and linked user exist before transaction balance date + const balanceDate = new Date(value.balanceDate) + if ( + value.user.createdAt.getTime() >= balanceDate.getTime() || + value.linkedUser?.createdAt.getTime() >= balanceDate.getTime() + ) { + throw new Error( + `at least one user was created after transaction balance date, logic error! ${JSON.stringify(value, null, 2)}`, + ) + } -export const transactionDbSchema = v.pipe(v.object({ - ...transactionBaseSchema.entries, - typeId: v.enum(TransactionTypeId), - balanceDate: dateSchema, - linkedUser: userDbSchema, -}), v.custom((value: any) => { - if (value.user && value.linkedUser && !value.transactionLinkCode && value.user.gradidoId === value.linkedUser.gradidoId) { - throw new Error(`expect user to be different from linkedUser: ${JSON.stringify(value, null, 2)}`) - } - // check that user and linked user exist before transaction balance date - const balanceDate = new Date(value.balanceDate) - if ( - value.user.createdAt.getTime() >= balanceDate.getTime() || - value.linkedUser?.createdAt.getTime() >= balanceDate.getTime() - ) { - throw new Error( - `at least one user was created after transaction balance date, logic error! ${JSON.stringify(value, null, 2)}`, - ) - } - - return value -})) + return value + }), +) -export const creationTransactionDbSchema = v.pipe(v.object({ - ...transactionBaseSchema.entries, - contributionDate: dateSchema, - confirmedAt: dateSchema, - confirmedByUser: userDbSchema, - transactionId: positiveNumberSchema, -}), v.custom((value: any) => { - if (value.user && value.confirmedByUser && value.user.gradidoId === value.confirmedByUser.gradidoId) { - throw new Error(`expect user to be different from confirmedByUser: ${JSON.stringify(value, null, 2)}`) - } - // check that user and confirmedByUser exist before transaction balance date - const confirmedAt = new Date(value.confirmedAt) - if ( - value.user.createdAt.getTime() >= confirmedAt.getTime() || - value.confirmedByUser?.createdAt.getTime() >= confirmedAt.getTime() - ) { - throw new Error( - `at least one user was created after transaction confirmedAt date, logic error! ${JSON.stringify(value, null, 2)}`, - ) - } - - return value -})) +export const creationTransactionDbSchema = v.pipe( + v.object({ + ...transactionBaseSchema.entries, + contributionDate: dateSchema, + confirmedAt: dateSchema, + confirmedByUser: userDbSchema, + transactionId: positiveNumberSchema, + }), + v.custom((value: any) => { + if ( + value.user && + value.confirmedByUser && + value.user.gradidoId === value.confirmedByUser.gradidoId + ) { + throw new Error( + `expect user to be different from confirmedByUser: ${JSON.stringify(value, null, 2)}`, + ) + } + // check that user and confirmedByUser exist before transaction balance date + const confirmedAt = new Date(value.confirmedAt) + if ( + value.user.createdAt.getTime() >= confirmedAt.getTime() || + value.confirmedByUser?.createdAt.getTime() >= confirmedAt.getTime() + ) { + throw new Error( + `at least one user was created after transaction confirmedAt date, logic error! ${JSON.stringify(value, null, 2)}`, + ) + } + + return value + }), +) export const transactionLinkDbSchema = v.object({ ...transactionBaseSchema.entries, diff --git a/dlt-connector/src/schemas/typeConverter.schema.ts b/dlt-connector/src/schemas/typeConverter.schema.ts index 572593f0c..167712660 100644 --- a/dlt-connector/src/schemas/typeConverter.schema.ts +++ b/dlt-connector/src/schemas/typeConverter.schema.ts @@ -89,7 +89,7 @@ export const confirmedTransactionSchema = v.pipe( if (typeof data === 'object' && 'base64' in data && 'communityId' in data) { return confirmedTransactionFromBase64(data.base64, data.communityId) } - throw new Error('invalid data, community id missing, couldn\'t deserialize') + throw new Error("invalid data, community id missing, couldn't deserialize") }, ), ) diff --git a/dlt-connector/src/utils/filesystem.ts b/dlt-connector/src/utils/filesystem.ts index 096437fb5..338d536d8 100644 --- a/dlt-connector/src/utils/filesystem.ts +++ b/dlt-connector/src/utils/filesystem.ts @@ -31,4 +31,4 @@ export function checkPathExist(path: string, createIfMissing: boolean = false): export function toFolderName(name: string): string { return name.toLowerCase().replace(/[^a-z0-9]/g, '_') -} \ No newline at end of file +} diff --git a/dlt-connector/src/utils/typeConverter.ts b/dlt-connector/src/utils/typeConverter.ts index 4124df7e6..1e0ac464a 100644 --- a/dlt-connector/src/utils/typeConverter.ts +++ b/dlt-connector/src/utils/typeConverter.ts @@ -8,7 +8,10 @@ import { AccountType } from '../data/AccountType.enum' import { AddressType } from '../data/AddressType.enum' import { Uuidv4 } from '../schemas/typeGuard.schema' -export const confirmedTransactionFromBase64 = (base64: string, communityId: Uuidv4): ConfirmedTransaction => { +export const confirmedTransactionFromBase64 = ( + base64: string, + communityId: Uuidv4, +): ConfirmedTransaction => { const confirmedTransactionBinaryPtr = MemoryBlock.createPtr(MemoryBlock.fromBase64(base64)) const deserializer = new InteractionDeserialize( confirmedTransactionBinaryPtr, diff --git a/shared/src/logic/decay.ts b/shared/src/logic/decay.ts index 23c37abf2..2346c2635 100644 --- a/shared/src/logic/decay.ts +++ b/shared/src/logic/decay.ts @@ -22,9 +22,7 @@ export function decayFormula(value: Decimal, seconds: number): Decimal { // chatgpt: We convert to string here to avoid precision loss: // .pow(seconds) can internally round the result, especially for large values of `seconds`. // Using .toString() ensures full precision is preserved in the multiplication. - return value.mul( - DECAY_FACTOR.pow(seconds).toString(), - ) + return value.mul(DECAY_FACTOR.pow(seconds).toString()) } // legacy reverse decay formula From f54463b2bfd4d107ced55b87b28f00368aaca683 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 26 Feb 2026 09:34:29 +0100 Subject: [PATCH 34/50] update lockfile with new gradido-blockchain-js version --- dlt-connector/bun.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dlt-connector/bun.lock b/dlt-connector/bun.lock index 1bbaedc8c..8d1e6925d 100644 --- a/dlt-connector/bun.lock +++ b/dlt-connector/bun.lock @@ -7,7 +7,7 @@ "dependencies": { "bun-zigar": "^0.15.2", "cross-env": "^7.0.3", - "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#f265dbb1780a912cf8b0418dfe3eaf5cdc5b51cf", + "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#66c8f6839dd95fc34d323066951b80043690fdc2", }, "devDependencies": { "@biomejs/biome": "2.0.0", @@ -585,7 +585,7 @@ "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], - "gradido-blockchain-js": ["gradido-blockchain-js@github:gradido/gradido-blockchain-js#f265dbb", { "dependencies": { "bindings": "^1.5.0", "nan": "^2.20.0", "node-addon-api": "^7.1.1", "node-gyp-build": "^4.8.1", "prebuildify": "git+https://github.com/einhornimmond/prebuildify#65d94455fab86b902c0d59bb9c06ac70470e56b2" } }, "gradido-gradido-blockchain-js-f265dbb"], + "gradido-blockchain-js": ["gradido-blockchain-js@github:gradido/gradido-blockchain-js#66c8f68", { "dependencies": { "bindings": "^1.5.0", "nan": "^2.20.0", "node-addon-api": "^7.1.1", "node-gyp-build": "^4.8.1", "prebuildify": "git+https://github.com/einhornimmond/prebuildify#65d94455fab86b902c0d59bb9c06ac70470e56b2" } }, "gradido-gradido-blockchain-js-66c8f68"], "graphql": ["graphql@16.11.0", "", {}, "sha512-mS1lbMsxgQj6hge1XZ6p7GPhbrtFwUFYi3wRzXAC/FmYnyXMTvvI3td3rjmQ2u8ewXueaSvRPWaEcgVVOT9Jnw=="], From 81d6f6dbf5ffc9b3d292fcd5cc30e13447496811 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 26 Feb 2026 11:06:13 +0100 Subject: [PATCH 35/50] removed not used zig/c stuff, fix problem with gradido node runtime path --- dlt-connector/bun-zigar.toml | 8 -- dlt-connector/c/grdc/grdc_account_balance.h | 23 ---- dlt-connector/c/grdc/grdc_confirmed_tx.h | 104 --------------- dlt-connector/c/grdc/grdc_public_key_index.h | 21 --- dlt-connector/c/grdc/grdc_tx_id.h | 25 ---- dlt-connector/c/grdc/grdc_tx_timestamps.h | 42 ------ dlt-connector/c/grdd/grdd_address_type.h | 23 ---- .../c/grdd/grdd_balance_derivation_type.h | 26 ---- dlt-connector/c/grdd/grdd_cross_group_type.h | 20 --- dlt-connector/c/grdd/grdd_timestamp.h | 38 ------ dlt-connector/c/grdd/grdd_transaction_type.h | 56 -------- dlt-connector/c/grdl/grdl_unit.c | 123 ------------------ dlt-connector/c/grdl/grdl_unit.h | 84 ------------ .../src/bootstrap/initGradidoNode.ts | 2 + .../client/GradidoNode/GradidoNodeProcess.ts | 6 +- dlt-connector/src/utils/filesystem.ts | 1 + dlt-connector/zig/hello.zig | 5 - 17 files changed, 6 insertions(+), 601 deletions(-) delete mode 100644 dlt-connector/bun-zigar.toml delete mode 100644 dlt-connector/c/grdc/grdc_account_balance.h delete mode 100644 dlt-connector/c/grdc/grdc_confirmed_tx.h delete mode 100644 dlt-connector/c/grdc/grdc_public_key_index.h delete mode 100644 dlt-connector/c/grdc/grdc_tx_id.h delete mode 100644 dlt-connector/c/grdc/grdc_tx_timestamps.h delete mode 100644 dlt-connector/c/grdd/grdd_address_type.h delete mode 100644 dlt-connector/c/grdd/grdd_balance_derivation_type.h delete mode 100644 dlt-connector/c/grdd/grdd_cross_group_type.h delete mode 100644 dlt-connector/c/grdd/grdd_timestamp.h delete mode 100644 dlt-connector/c/grdd/grdd_transaction_type.h delete mode 100644 dlt-connector/c/grdl/grdl_unit.c delete mode 100644 dlt-connector/c/grdl/grdl_unit.h delete mode 100644 dlt-connector/zig/hello.zig diff --git a/dlt-connector/bun-zigar.toml b/dlt-connector/bun-zigar.toml deleted file mode 100644 index 5a2dc3f56..000000000 --- a/dlt-connector/bun-zigar.toml +++ /dev/null @@ -1,8 +0,0 @@ -optimize = "ReleaseSmall" - -[modules."lib/hello.zigar"] -source = "zig/hello.zig" - -[[targets]] -arch = "x64" -platform = "linux" diff --git a/dlt-connector/c/grdc/grdc_account_balance.h b/dlt-connector/c/grdc/grdc_account_balance.h deleted file mode 100644 index fc38b1926..000000000 --- a/dlt-connector/c/grdc/grdc_account_balance.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef __GRADIDO_BLOCKCHAIN_C_COMPACT_ACCOUNT_BALANCE_H -#define __GRADIDO_BLOCKCHAIN_C_COMPACT_ACCOUNT_BALANCE_H - -#include -#include "grdc_public_key_index.h" -#include "../grdl/grdl_unit.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct grdc_account_balance -{ - grdl_unit balance; - grdc_public_key_index publicKeyIndex; -} grdc_account_balance; - -#ifdef __cplusplus -} -#endif - - -#endif // __GRADIDO_BLOCKCHAIN_C_COMPACT_ACCOUNT_BALANCE_H \ No newline at end of file diff --git a/dlt-connector/c/grdc/grdc_confirmed_tx.h b/dlt-connector/c/grdc/grdc_confirmed_tx.h deleted file mode 100644 index 999e4633d..000000000 --- a/dlt-connector/c/grdc/grdc_confirmed_tx.h +++ /dev/null @@ -1,104 +0,0 @@ -#ifndef __GRADIDO_BLOCKCHAIN_C_COMPACT_CONFIRMED_TRANSACTION_COMPACT_H -#define __GRADIDO_BLOCKCHAIN_C_COMPACT_CONFIRMED_TRANSACTION_COMPACT_H - -#include -#include "grdc_account_balance.h" -#include "grdc_public_key_index.h" -#include "grdc_tx_id.h" -#include "grdc_tx_timestamps.h" -#include "../grdd/grdd_address_type.h" -#include "../grdd/grdd_balance_derivation_type.h" -#include "../grdd/grdd_cross_group_type.h" -#include "../grdd/grdd_transaction_type.h" -#include "../grdl/grdl_unit.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Gradido Compact Confirmed Transaction -struct grdc_confirmed_tx -{ - grdc_tx_timestamps timestamps; - - // inline Timestamp getConfirmedAt() const { return timestamps.getConfirmedAt(); } - // inline Timestamp getCreatedAt() const { return timestamps.getCreatedAt(); } - - // txId and pairingTxId packed to save 8 Bytes padding - uint64_t txNrs[2]; - uint32_t communityIdIndex[2]; - // inline TxId getTxId() const { return TxId(txNrs[0], communityIdIndex[0]); } - // for cross group transactions, else empty - // inline TxId getPairingTxId() const { return TxId(txNrs[1], communityIdIndex[1]); } - - // account balances and memos via txId in separate list/manager/thingy - - // enums, usually uint8_t - uint8_t crossGroupType; // grdd_cross_group_type - uint8_t transactionType; // grdd_transaction_type - uint8_t balanceDerivationType; // grdd_balance_derivation_type - uint8_t accountBalanceCount; - - grdc_account_balance accountBalances[2]; - - // common fields for most transactions - union { // 24 Bytes - struct { - grdl_unit amount; // 8 Bytes - grdc_public_key_index recipientPublicKeyIndex; // 8 Bytes - uint64_t targetDate; // 8 Bytes - } creation; - struct { - grdl_unit amount; // 8 Bytes - grdc_public_key_index senderPublicKeyIndex; // 8 Bytes - grdc_public_key_index recipientPublicKeyIndex; // 8 Bytes - } transfer; // also used for redeem deferred transfer, and deferredTransferTransactionNr is stored in extra dictionary - struct { - grdl_unit amount; // 8 Bytes - // work only on local, take communityIdIndex from txId - uint32_t senderPublicKeyIndex; // 4 Bytes - uint32_t recipientPublicKeyIndex; // 4 Bytes - uint32_t timeoutDuration; // 4 Bytes - } deferredTransfer; // fund deferred transfer address only on your own community - struct { - grdc_tx_id deferredTransferTransactionNr; // 16 Bytes, contain 4 Bytes padding - } timeoutDeferredTransfer; - struct { - uint8_t addressType; // grdd_address_type // 1 Byte - uint16_t derivationIndex; // 2 Byte (for the time beeing, update if more than 65535 are needed) - uint32_t nameHashIndex; // 4 Bytes - grdc_public_key_index userPublicKeyIndex; // 8 Bytes - grdc_public_key_index accountPublicKeyIndex; // 8 Bytes - } registerAddress; - struct { - grdc_public_key_index communityPublicKeyIndex; // 8 Bytes - grdc_public_key_index communityAufPublicKeyIndex; // 8 Bytes - grdc_public_key_index communityGmwPublicKeyIndex; // 8 Bytes - } communityRoot; - }; -}; - -inline grdd_timestamp grdc_confirmed_tx_get_confirmed_at(const grdc_confirmed_tx* c) { - return grdc_tx_timestamps_get_confirmed_at(&c->timestamps); -} - -inline grdd_timestamp grdc_confirmed_tx_get_created_at(const grdc_confirmed_tx* c) { - return grdc_tx_timestamps_get_created_at(&c->timestamps); -} - -inline grdc_tx_id grdc_confirmed_tx_get_tx_id(const grdc_confirmed_tx* c) { - grdc_tx_id txId{.nr = c->txNrs[0], .communityIdIndex = c->communityIdIndex[0]}; - return txId; -} - -inline grdc_tx_id grdc_confirmed_tx_get_pairing_tx_id(const grdc_confirmed_tx* c) { - grdc_tx_id pairingTxId{.nr = c->txNrs[1], .communityIdIndex = c->communityIdIndex[1]}; - return pairingTxId; -} - -#ifdef __cplusplus -} -#endif - - -#endif //__GRADIDO_BLOCKCHAIN_C_COMPACT_CONFIRMED_TRANSACTION_COMPACT_H \ No newline at end of file diff --git a/dlt-connector/c/grdc/grdc_public_key_index.h b/dlt-connector/c/grdc/grdc_public_key_index.h deleted file mode 100644 index 6ab88fcbf..000000000 --- a/dlt-connector/c/grdc/grdc_public_key_index.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef __GRADIDO_BLOCKCHAIN_C_COMPACT_PUBLIC_KEY_INDEX_H -#define __GRADIDO_BLOCKCHAIN_C_COMPACT_PUBLIC_KEY_INDEX_H - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct grdc_public_key_index -{ - uint32_t publicKeyIndex; - uint32_t communityIdIndex; -} grdc_public_key_index; - -#ifdef __cplusplus -} -#endif - - -#endif // __GRADIDO_BLOCKCHAIN_C_COMPACT_PUBLIC_KEY_INDEX_H \ No newline at end of file diff --git a/dlt-connector/c/grdc/grdc_tx_id.h b/dlt-connector/c/grdc/grdc_tx_id.h deleted file mode 100644 index b8a1ed12c..000000000 --- a/dlt-connector/c/grdc/grdc_tx_id.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef __GRADIDO_BLOCKCHAIN_C_COMPACT_TRANSACTION_ID -#define __GRADIDO_BLOCKCHAIN_C_COMPACT_TRANSACTION_ID - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct grdc_tx_id -{ - uint64_t nr; - uint32_t communityIdIndex; -} grdc_tx_id; - -inline bool grdc_tx_id_empty(const grdc_tx_id* id) { - return !id->nr && !id->communityIdIndex; -} - -#ifdef __cplusplus -} -#endif - -#endif // __GRADIDO_BLOCKCHAIN_C_COMPACT_TRANSACTION_ID diff --git a/dlt-connector/c/grdc/grdc_tx_timestamps.h b/dlt-connector/c/grdc/grdc_tx_timestamps.h deleted file mode 100644 index ab2e331d6..000000000 --- a/dlt-connector/c/grdc/grdc_tx_timestamps.h +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef __GRADIDO_BLOCKCHAIN_C_COMPACT_TRANSACTION_TIMESTAMPS_H -#define __GRADIDO_BLOCKCHAIN_C_COMPACT_TRANSACTION_TIMESTAMPS_H - -#include -#include "../grdd/grdd_timestamp.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct grdc_tx_timestamps -{ - uint64_t confirmedSeconds; - uint32_t confirmedNanos; - int32_t deltaMs; // add to confirmed second and nanos to get created_at -} grdc_tx_timestamps; - -inline grdd_timestamp grdc_tx_timestamps_get_confirmed_at(const grdc_tx_timestamps* in) { - grdd_timestamp t; - t.seconds = in->confirmedSeconds; - t.nanos = in->confirmedNanos; - return t; -} - -inline grdd_timestamp grdc_tx_timestamps_get_created_at(const grdc_tx_timestamps* in) { - grdd_timestamp t; - uint64_t sec = (in->confirmedSeconds + in->deltaMs) / 1000; - uint32_t ns = (in->confirmedNanos + (in->deltaMs % 1000)) * 1000000; - if (ns >= 1000000000) { - sec += 1; - ns -= 1000000000; - } - t.seconds = sec; - t.nanos = ns; - return t; -} - -#ifdef __cplusplus -} -#endif - -#endif //__GRADIDO_BLOCKCHAIN_C_COMPACT_TRANSACTION_TIMESTAMPS_H \ No newline at end of file diff --git a/dlt-connector/c/grdd/grdd_address_type.h b/dlt-connector/c/grdd/grdd_address_type.h deleted file mode 100644 index c49593b00..000000000 --- a/dlt-connector/c/grdd/grdd_address_type.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef __GRADIDO_BLOCKCHAIN_C_DATA_ADDRESS_TYPE_H -#define __GRADIDO_BLOCKCHAIN_C_DATA_ADDRESS_TYPE_H - -#ifdef __cplusplus -extern "C" { -#endif - -enum grdd_address_type { - GRDD_ADDRESS_TYPE_NONE = 0, // if no address was found - GRDD_ADDRESS_TYPE_COMMUNITY_HUMAN = 1, // creation account for human - GRDD_ADDRESS_TYPE_COMMUNITY_GMW = 2, // community public budget account - GRDD_ADDRESS_TYPE_COMMUNITY_AUF = 3, // community compensation and environment founds account - GRDD_ADDRESS_TYPE_COMMUNITY_PROJECT = 4, // no creations allowed - GRDD_ADDRESS_TYPE_SUBACCOUNT = 5, // no creations allowed - GRDD_ADDRESS_TYPE_CRYPTO_ACCOUNT = 6, // user control his keys, no creations - GRDD_ADDRESS_TYPE_DEFERRED_TRANSFER = 7 // special type, no need for register address -}; - -#ifdef __cplusplus -} -#endif - -#endif //__GRADIDO_BLOCKCHAIN_C_DATA_ADDRESS_TYPE_H \ No newline at end of file diff --git a/dlt-connector/c/grdd/grdd_balance_derivation_type.h b/dlt-connector/c/grdd/grdd_balance_derivation_type.h deleted file mode 100644 index 1a24131ac..000000000 --- a/dlt-connector/c/grdd/grdd_balance_derivation_type.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef GRADIDO_BLOCKCHAIN_C_DATA_BALANCE_DERIVATION_TYPE_H -#define GRADIDO_BLOCKCHAIN_C_DATA_BALANCE_DERIVATION_TYPE_H - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * @enum grdd_balance_derivation_type - * - * Flag to decide if Node calculates balance and decay - * or trusts external data. - */ -enum grdd_balance_derivation_type { - GRDD_BALANCE_DERIVATION_UNSPECIFIED = 0, - /* Balances & decay can be recalculated deterministically */ - GRDD_BALANCE_DERIVATION_NODE = 1, - /* Balances are accepted as-is from external / legacy system */ - GRDD_BALANCE_DERIVATION_EXTERN = 2 -}; - -#ifdef __cplusplus -} -#endif - -#endif /* GRADIDO_BLOCKCHAIN_C_DATA_BALANCE_DERIVATION_TYPE_H */ diff --git a/dlt-connector/c/grdd/grdd_cross_group_type.h b/dlt-connector/c/grdd/grdd_cross_group_type.h deleted file mode 100644 index 2a0c4a72a..000000000 --- a/dlt-connector/c/grdd/grdd_cross_group_type.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef __GRADIDO_BLOCKCHAIN_C_DATA_CROSS_GROUP_TYPE_H -#define __GRADIDO_BLOCKCHAIN_C_DATA_CROSS_GROUP_TYPE_H - -#ifdef __cplusplus -extern "C" { -#endif - -enum grdd_cross_group_type -{ - GRDD_CROSS_GROUP_TYPE_LOCAL = 0, - GRDD_CROSS_GROUP_TYPE_INBOUND = 1, - GRDD_CROSS_GROUP_TYPE_OUTBOUND = 2, - GRDD_CROSS_GROUP_TYPE_CROSS = 3 -}; - -#ifdef __cplusplus -} -#endif - -#endif // __GRADIDO_BLOCKCHAIN_C_DATA_CROSS_GROUP_TYPE_H \ No newline at end of file diff --git a/dlt-connector/c/grdd/grdd_timestamp.h b/dlt-connector/c/grdd/grdd_timestamp.h deleted file mode 100644 index 2bf54de67..000000000 --- a/dlt-connector/c/grdd/grdd_timestamp.h +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef __GRADIDO_BLOCKCHAIN_C_DATA_TIMESTAMP_H -#define __GRADIDO_BLOCKCHAIN_C_DATA_TIMESTAMP_H - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct grdd_timestamp -{ - int64_t seconds; - int32_t nanos; -} grdd_timestamp; - -inline bool grdd_timestamp_empty(const grdd_timestamp* timestamp) { - return !timestamp->seconds && !timestamp->nanos; -} -inline bool grdd_timestamp_eq(const grdd_timestamp* t1, grdd_timestamp* t2) { - return t1->seconds == t2->seconds && t1->nanos == t2->nanos; -} -inline bool grdd_timestamp_gt(const grdd_timestamp* t1, const grdd_timestamp* t2) { - return t1->seconds > t2->seconds || t1->seconds == t2->seconds && t1->nanos > t2->nanos; -} -inline bool grdd_timestamp_lt(const grdd_timestamp* t1, const grdd_timestamp* t2) { - return t1->seconds < t2->seconds || t1->seconds == t2->seconds && t1->nanos < t2->nanos; -} -inline grdd_timestamp grdd_timestamp_sub(const grdd_timestamp* t1, grdd_timestamp* t2) { - grdd_timestamp result{.seconds = t1->seconds - t2->seconds, .nanos = t1->nanos - t2->nanos}; - return result; -} - -#ifdef __cplusplus -} -#endif - -#endif // __GRADIDO_BLOCKCHAIN_C_DATA_TIMESTAMP_H \ No newline at end of file diff --git a/dlt-connector/c/grdd/grdd_transaction_type.h b/dlt-connector/c/grdd/grdd_transaction_type.h deleted file mode 100644 index 83a0540ea..000000000 --- a/dlt-connector/c/grdd/grdd_transaction_type.h +++ /dev/null @@ -1,56 +0,0 @@ -#ifndef GRADIDO_BLOCKCHAIN_C_DATA_TRANSACTION_TYPE_H -#define GRADIDO_BLOCKCHAIN_C_DATA_TRANSACTION_TYPE_H - -#ifdef __cplusplus -extern "C" { -#endif - -/*! - * \addtogroup enums - * @{ - */ - -/*! - * \enum grdd_transaction_type - * Enum for different transaction types - * !!! don't change order - */ -enum grdd_transaction_type { - //! Invalid or Empty Transaction - GRDD_TRANSACTION_TYPE_NONE = 0, - - //! Creation Transaction, creates new Gradidos - GRDD_TRANSACTION_TYPE_CREATION, - - //! Transfer Transaction, move Gradidos from one account to another - GRDD_TRANSACTION_TYPE_TRANSFER, - - //! Group Friends Update Transaction, update relationship between groups - GRDD_TRANSACTION_TYPE_COMMUNITY_FRIENDS_UPDATE, - - //! Register new address or sub address to group or move address to another group - GRDD_TRANSACTION_TYPE_REGISTER_ADDRESS, - - //! Special Transfer Transaction with timeout used for Gradido Link - GRDD_TRANSACTION_TYPE_DEFERRED_TRANSFER, - - //! First Transaction in Blockchain - GRDD_TRANSACTION_TYPE_COMMUNITY_ROOT, - - //! Redeeming deferred transfer - GRDD_TRANSACTION_TYPE_REDEEM_DEFERRED_TRANSFER, - - //! Timeout deferred transfer, send back locked gdds - GRDD_TRANSACTION_TYPE_TIMEOUT_DEFERRED_TRANSFER, - - //! Technical type for using it in for loops, as max index - GRDD_TRANSACTION_TYPE_MAX_VALUE -}; - -/*! @} */ - -#ifdef __cplusplus -} -#endif - -#endif /* GRADIDO_BLOCKCHAIN_C_DATA_TRANSACTION_TYPE_H */ \ No newline at end of file diff --git a/dlt-connector/c/grdl/grdl_unit.c b/dlt-connector/c/grdl/grdl_unit.c deleted file mode 100644 index 90f68ad38..000000000 --- a/dlt-connector/c/grdl/grdl_unit.c +++ /dev/null @@ -1,123 +0,0 @@ -#include "grdl_unit.h" - -#include -#include -#include - -const double SECONDS_PER_YEAR = 31556952.0; // seconds in a year in gregorian calender -const grdd_timestamp DECAY_START_TIME = (grdd_timestamp){.seconds = 1620927991, .nanos = 0}; - -double roundToPrecision(double gdd, uint8_t precision) -{ - double factor = pow(10.0, precision); - return round(gdd * factor) / factor; -} - -grdl_unit grdl_unit_from_decimal(double gdd) -{ - grdl_unit gradidoUnit = { - (int64_t)(roundToPrecision(gdd, 4) * 10000.0) - }; - return gradidoUnit; -} - -grdl_unit grdl_unit_from_string(const char* gdd_string) -{ - if (!gdd_string) return (grdl_unit){0}; - char* end; - double gdd_double = strtod(gdd_string, &end); - if (end == gdd_string || *end != '\0') { - // invalid string - return (grdl_unit){0}; - } - return grdl_unit_from_decimal(gdd_double); -} - -int grdl_unit_to_string(const grdl_unit* u, char* buffer, size_t bufferSize, uint8_t precision) -{ - if (precision > 4) return 1; // C hat keine Exceptions - - // Convert to double - double decimal = (double)(u->gradidoCent) / 10000.0; - - // Round down like Node.js if precision < 4 - if (precision < 4) { - double factor = pow(10.0, precision); - decimal = round(decimal * factor) / factor; - } - - // Write to buffer - int written = snprintf(buffer, bufferSize, "%.*f", precision, decimal); - - // snprintf returns number of chars that would have been written (excluding null) - // snprintf return negative value on encoding error - if (written < 0) return 2; - if ((size_t)written < bufferSize) { - return 0; - } - return bufferSize - written; -} - -grdl_unit grdl_unit_calculate_decay(const grdl_unit* u, int64_t seconds) -{ - if (seconds == 0) return (grdl_unit){u->gradidoCent}; - - // decay for one year is 50% - /* - * while (seconds >= SECONDS_PER_YEAR) { - mGradidoCent *= 0.5; - seconds -= SECONDS_PER_YEAR; - } - */ - int64_t gradidoCent = u->gradidoCent; - // optimize version from above - if (seconds >= SECONDS_PER_YEAR) { - uint64_t times = (uint64_t)(seconds / SECONDS_PER_YEAR); - seconds = seconds - times * SECONDS_PER_YEAR; - gradidoCent = u->gradidoCent >> times; - if (!seconds) return (grdl_unit){gradidoCent}; - } -// */ - /*! - * calculate decay factor with compound interest formula converted to q
- * n = (lg Kn - lg K0) / lg q =>
- * lg q = (lg Kn - lg K0) / n =>
- * q = e^((lg Kn - lg K0) / n)
- *
- * with: - *
    - *
  • q = decay_factor
    • - *
  • n = days_per_year * 60 * 60 * 24 = seconds per year
    • - *
  • Kn = 50 (capital after a year)
    • - *
  • K0 = 100 (capital at start)
    • - *
- * further simplified: - * lg 50 - lg 100 = lg 2 => - * q = e^(lg 2 / n) = 2^(x/n) - * with x as seconds in which decay occured - */ - // https://www.wolframalpha.com/input?i=%28e%5E%28lg%282%29+%2F+31556952%29%29%5Ex&assumption=%7B%22FunClash%22%2C+%22lg%22%7D+-%3E+%7B%22Log%22%7D - // from wolframalpha, based on the interest rate formula - return (grdl_unit){((int64_t)((double)(gradidoCent) * pow(2.0, (double)((double)(-seconds) / SECONDS_PER_YEAR))))}; -} - -bool grdl_unit_calculate_duration_seconds( - const grdd_timestamp* startTime, - const grdd_timestamp* endTime, - int64_t* outSeconds -) { - if (!outSeconds) { - return false; - } - if(grdd_timestamp_gt(startTime, endTime) { - return false; - } - grdd_timestamp start = grdd_timestamp_gt(startTime, &DECAY_START_TIME) ? *startTime : DECAY_START_TIME; - grdd_timestamp end = grdd_timestamp_gt(endTime, &DECAY_START_TIME) ? *endTime : DECAY_START_TIME; - if (grdd_timestamp_eq(&start, &end)) { - *outSeconds = 0; - return true; - } - *outSeconds = grdd_timestamp_sub(&end, &start).seconds; - return true; -} diff --git a/dlt-connector/c/grdl/grdl_unit.h b/dlt-connector/c/grdl/grdl_unit.h deleted file mode 100644 index 66a658e1b..000000000 --- a/dlt-connector/c/grdl/grdl_unit.h +++ /dev/null @@ -1,84 +0,0 @@ -#ifndef __GRADIDO_BLOCKCHAIN_C_LOGIC_UNIT_H -#define __GRADIDO_BLOCKCHAIN_C_LOGIC_UNIT_H - -#include "../grdd/grdd_timestamp.h" - -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct grdl_unit -{ - int64_t gradidoCent; -} grdl_unit; - -grdl_unit grdl_unit_from_decimal(double gdd); -grdl_unit grdl_unit_from_string(const char* gdd_string); - -//! \return 0 for ok, 1 for invalid precision, 2 for printf encoding error -// and -x if string buffer wasn't big enough where x is the number of missing bytes -int grdl_unit_to_string(const grdl_unit* u, char* buffer, size_t bufferSize, uint8_t precision); - -inline grdl_unit grdl_unit_negated(const grdl_unit* u) -{ - grdl_unit gradidoUnit = {u->gradidoCent}; - gradidoUnit.gradidoCent *= -1; - return gradidoUnit; -} - -inline void grdl_unit_negate(grdl_unit* u) -{ - if (u) u->gradidoCent = -u->gradidoCent; -} - -static inline grdl_unit grdl_unit_zero() -{ - return (grdl_unit){0}; -} -//! return false if startTime > endTime -bool grdl_unit_calculate_duration_seconds( - const grdd_timestamp* startTime, - const grdd_timestamp* endTime, - int64_t* outSeconds -); - -grdl_unit grdl_unit_calculate_decay(const grdl_unit* u, int64_t seconds); - -inline grdl_unit grdl_unit_calculate_decay_timestamp( - const grdl_unit* u, - const grdd_timestamp* startTime, - const grdd_timestamp* endTime -) { - int64_t seconds = 0; - if(!grdl_unit_calculate_duration_seconds(startTime, endTime, &seconds)) { - return (grdl_unit){0}; - } - return grdl_unit_calculate_decay(u, seconds); -} - -inline grdl_unit grld_unit_calculate_compound_interest(const grdl_unit* u, int64_t seconds) { - return grdl_unit_calculate_decay(u, -seconds); -} - -inline grdl_unit grld_unit_calculate_compound_interest_timestamp( - const grdl_unit* u, - const grdd_timestamp* startTime, - const grdd_timestamp* endTime -) { - int64_t seconds = 0; - if(!grdl_unit_calculate_duration_seconds(startTime, endTime, &seconds)) { - return (grdl_unit){0}; - } - return grdl_unit_calculate_decay(u, -seconds); -} - -#ifdef __cplusplus -} -#endif - - -#endif // __GRADIDO_BLOCKCHAIN_C_LOGIC_UNIT_H \ No newline at end of file diff --git a/dlt-connector/src/bootstrap/initGradidoNode.ts b/dlt-connector/src/bootstrap/initGradidoNode.ts index e0d1a047d..f7e3036af 100644 --- a/dlt-connector/src/bootstrap/initGradidoNode.ts +++ b/dlt-connector/src/bootstrap/initGradidoNode.ts @@ -74,6 +74,8 @@ async function ensureGradidoNodeRuntimeAvailable(runtimeFileName: string): Promi fs.writeFileSync(archivePath, Buffer.from(compressedBuffer)) execSync(`tar -xzf ${archivePath}`, { cwd: runtimeFolder }) } + } else { + console.log('file exist') } } diff --git a/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts b/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts index 0347d136e..7780a815f 100644 --- a/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts +++ b/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts @@ -48,7 +48,7 @@ export class GradidoNodeProcess { const isWindows = process.platform === 'win32' const binaryName = isWindows ? 'GradidoNode.exe' : 'GradidoNode' - return path.join(__dirname, '..', '..', 'gradido_node', 'bin', binaryName) + return path.join(__dirname, CONFIG.DLT_GRADIDO_NODE_SERVER_HOME_FOLDER, 'bin', binaryName) } public start() { @@ -56,10 +56,10 @@ export class GradidoNodeProcess { this.logger.warn('GradidoNodeProcess already running.') return } - this.logger.info(`starting GradidoNodeProcess with path: ${GRADIDO_NODE_RUNTIME_PATH}`) + this.logger.info(`starting GradidoNodeProcess with path: ${GradidoNodeProcess.getRuntimePathFileName()}`) this.lastStarted = new Date() const logger = this.logger - this.proc = spawn([GRADIDO_NODE_RUNTIME_PATH], { + this.proc = spawn([GradidoNodeProcess.getRuntimePathFileName()], { env: { CLIENTS_HIERO_NETWORKTYPE: CONFIG.HIERO_HEDERA_NETWORK, SERVER_JSON_RPC_PORT: CONFIG.DLT_NODE_SERVER_PORT.toString(), diff --git a/dlt-connector/src/utils/filesystem.ts b/dlt-connector/src/utils/filesystem.ts index 338d536d8..bdb12ee8e 100644 --- a/dlt-connector/src/utils/filesystem.ts +++ b/dlt-connector/src/utils/filesystem.ts @@ -6,6 +6,7 @@ const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.utils.filesystem`) export function checkFileExist(filePath: string): boolean { try { + console.log(`check access to ${filePath}`) fs.accessSync(filePath, fs.constants.R_OK | fs.constants.W_OK) return true } catch (_err) { diff --git a/dlt-connector/zig/hello.zig b/dlt-connector/zig/hello.zig deleted file mode 100644 index 36bc9b556..000000000 --- a/dlt-connector/zig/hello.zig +++ /dev/null @@ -1,5 +0,0 @@ -const std = @import("std"); - -pub fn hello() void { - std.debug.print("Hello world!", .{}); -} From bfa53d9604c002fd84541b3405f79901155b9f39 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 26 Feb 2026 11:07:03 +0100 Subject: [PATCH 36/50] removed debug log --- dlt-connector/src/bootstrap/initGradidoNode.ts | 2 -- dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts | 5 +++-- dlt-connector/src/utils/filesystem.ts | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/dlt-connector/src/bootstrap/initGradidoNode.ts b/dlt-connector/src/bootstrap/initGradidoNode.ts index f7e3036af..e0d1a047d 100644 --- a/dlt-connector/src/bootstrap/initGradidoNode.ts +++ b/dlt-connector/src/bootstrap/initGradidoNode.ts @@ -74,8 +74,6 @@ async function ensureGradidoNodeRuntimeAvailable(runtimeFileName: string): Promi fs.writeFileSync(archivePath, Buffer.from(compressedBuffer)) execSync(`tar -xzf ${archivePath}`, { cwd: runtimeFolder }) } - } else { - console.log('file exist') } } diff --git a/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts b/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts index 7780a815f..a98db7d71 100644 --- a/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts +++ b/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts @@ -56,10 +56,11 @@ export class GradidoNodeProcess { this.logger.warn('GradidoNodeProcess already running.') return } - this.logger.info(`starting GradidoNodeProcess with path: ${GradidoNodeProcess.getRuntimePathFileName()}`) + const gradidoNodeRuntimePath = GradidoNodeProcess.getRuntimePathFileName() + this.logger.info(`starting GradidoNodeProcess with path: ${gradidoNodeRuntimePath}`) this.lastStarted = new Date() const logger = this.logger - this.proc = spawn([GradidoNodeProcess.getRuntimePathFileName()], { + this.proc = spawn([gradidoNodeRuntimePath], { env: { CLIENTS_HIERO_NETWORKTYPE: CONFIG.HIERO_HEDERA_NETWORK, SERVER_JSON_RPC_PORT: CONFIG.DLT_NODE_SERVER_PORT.toString(), diff --git a/dlt-connector/src/utils/filesystem.ts b/dlt-connector/src/utils/filesystem.ts index bdb12ee8e..338d536d8 100644 --- a/dlt-connector/src/utils/filesystem.ts +++ b/dlt-connector/src/utils/filesystem.ts @@ -6,7 +6,6 @@ const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.utils.filesystem`) export function checkFileExist(filePath: string): boolean { try { - console.log(`check access to ${filePath}`) fs.accessSync(filePath, fs.constants.R_OK | fs.constants.W_OK) return true } catch (_err) { From b94d6ae89afc81f3b684ead25263641e9acf83e9 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Fri, 27 Feb 2026 16:31:51 +0100 Subject: [PATCH 37/50] fix some issues with seeded datas, and fix cross group balance --- backend/src/seeds/index.ts | 1 + .../db-v2.7.0_to_blockchain-v3.7/Context.ts | 24 +------- .../db-v2.7.0_to_blockchain-v3.7/database.ts | 27 +++----- .../RemoteTransactionsSync.role.ts | 61 ++++++++++--------- .../syncDbWithBlockchain/UsersSync.role.ts | 3 +- 5 files changed, 48 insertions(+), 68 deletions(-) diff --git a/backend/src/seeds/index.ts b/backend/src/seeds/index.ts index 59cd5e5d5..f400d080b 100644 --- a/backend/src/seeds/index.ts +++ b/backend/src/seeds/index.ts @@ -86,6 +86,7 @@ async function clearDatabase(db: AppDatabase) { await trx.query(`SET FOREIGN_KEY_CHECKS = 0`) await trx.query(`TRUNCATE TABLE contributions`) await trx.query(`TRUNCATE TABLE contribution_links`) + await trx.query(`TRUNCATE TABLE events`) await trx.query(`TRUNCATE TABLE users`) await trx.query(`TRUNCATE TABLE user_contacts`) await trx.query(`TRUNCATE TABLE user_roles`) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts index 2a24867ef..f9bf2c7b2 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts @@ -9,9 +9,8 @@ import { KeyPairCacheManager } from '../../cache/KeyPairCacheManager' import { CONFIG } from '../../config' import { LOG4JS_BASE_CATEGORY } from '../../config/const' import { Uuidv4 } from '../../schemas/typeGuard.schema' -import { loadUserByGradidoId } from './database' import { bytesToMbyte } from './utils' -import { CommunityContext, UserDb } from './valibot.schema' +import { CommunityContext } from './valibot.schema' dotenv.config() @@ -20,21 +19,18 @@ export class Context { public db: MySql2Database public communities: Map public cache: KeyPairCacheManager - public balanceFixGradidoUser: UserDb | null private timeUsed: Profiler constructor( logger: Logger, db: MySql2Database, - cache: KeyPairCacheManager, - balanceFixGradidoUser: UserDb | null, + cache: KeyPairCacheManager ) { this.logger = logger this.db = db this.cache = cache this.communities = new Map() this.timeUsed = new Profiler() - this.balanceFixGradidoUser = balanceFixGradidoUser } static async create(): Promise { @@ -49,21 +45,7 @@ export class Context { }) const db = drizzle({ client: connection }) const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5`) - let balanceFixGradidoUser: UserDb | null = null - if (process.env.MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID) { - balanceFixGradidoUser = await loadUserByGradidoId( - db, - process.env.MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID, - ) - if (!balanceFixGradidoUser) { - logger.error( - `MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID was set to ${process.env.MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID} but user not found`, - ) - } - } else { - logger.debug(`MIGRATION_ACCOUNT_BALANCE_FIX_GRADIDO_ID was not set`) - } - return new Context(logger, db, KeyPairCacheManager.getInstance(), balanceFixGradidoUser) + return new Context(logger, db, KeyPairCacheManager.getInstance()) } getCommunityContextByUuid(communityUuid: Uuidv4): CommunityContext { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/database.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/database.ts index aed3aaae7..69f2bc669 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/database.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/database.ts @@ -1,4 +1,4 @@ -import { asc, eq, isNotNull, sql } from 'drizzle-orm' +import { and, asc, eq, isNotNull, isNull, or, sql } from 'drizzle-orm' import { MySql2Database } from 'drizzle-orm/mysql2' import * as v from 'valibot' import { communitiesTable, eventsTable, userRolesTable, usersTable } from './drizzle.schema' @@ -14,7 +14,7 @@ export async function loadContributionLinkModeratorCache(db: MySql2Database): Pr user: usersTable, }) .from(eventsTable) - .leftJoin(usersTable, eq(eventsTable.actingUserId, usersTable.id)) + .innerJoin(usersTable, eq(eventsTable.actingUserId, usersTable.id)) .where(eq(eventsTable.type, 'ADMIN_CONTRIBUTION_LINK_CREATE')) .orderBy(asc(eventsTable.id)) @@ -33,7 +33,7 @@ export async function loadAdminUsersCache(db: MySql2Database): Promise { }) .from(userRolesTable) .where(eq(userRolesTable.role, 'ADMIN')) - .leftJoin(usersTable, eq(userRolesTable.userId, usersTable.id)) + .innerJoin(usersTable, eq(userRolesTable.userId, usersTable.id)) result.map((row: any) => { adminUsers.set(row.gradidoId, v.parse(userDbSchema, row.user)) @@ -52,8 +52,12 @@ export async function loadCommunities(db: MySql2Database): Promise '2000-01-01'`), + ) .orderBy(asc(communitiesTable.id)) .groupBy(communitiesTable.communityUuid) @@ -61,16 +65,3 @@ export async function loadCommunities(db: MySql2Database): Promise { - const result = await db - .select() - .from(usersTable) - .where(eq(usersTable.gradidoId, gradidoId)) - .limit(1) - - return result.length ? v.parse(userDbSchema, result[0]) : null -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts index 2dcf0dcdb..ea43e5a2b 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts @@ -125,39 +125,44 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole publicKey: MemoryBlockPtr, ): AccountBalances { this.accountBalances.clear() - if (communityContext.foreign) { - this.accountBalances.add(new AccountBalance(publicKey, GradidoUnit.zero(), coinCommunityId)) - return this.accountBalances - } else { - // try to use same coins from this community - let lastBalance = this.getLastBalanceForUser( + + // try to use same coins from this community + let lastBalance = this.getLastBalanceForUser( + publicKey, + communityContext.blockchain, + coinCommunityId, + ) + if ( + coinCommunityId != communityContext.communityId && + ( + lastBalance.getBalance().equal(GradidoUnit.zero()) || + lastBalance.getBalance().calculateDecay(lastBalance.getDate(), item.balanceDate).lt(amount)) + ) { + // don't work, so we use or own coins + lastBalance = this.getLastBalanceForUser( publicKey, communityContext.blockchain, - coinCommunityId, + communityContext.communityId, ) - if ( - lastBalance.getBalance().equal(GradidoUnit.zero()) || - lastBalance.getBalance().calculateDecay(lastBalance.getDate(), item.balanceDate).lt(amount) - ) { - // don't work, so we use or own coins - lastBalance = this.getLastBalanceForUser( - publicKey, - communityContext.blockchain, - communityContext.communityId, - ) - } - - try { - lastBalance.updateLegacyDecay(amount, item.balanceDate) - } catch (e) { - if (e instanceof NegativeBalanceError) { - this.logLastBalanceChangingTransactions(publicKey, communityContext.blockchain, 1) - throw e - } - } - this.accountBalances.add(lastBalance.getAccountBalance()) + } + if ( + lastBalance.getBalance().calculateDecay(lastBalance.getDate(), item.balanceDate).add(amount).lt(GradidoUnit.zero()) + && communityContext.foreign + ) { + this.accountBalances.add(new AccountBalance(publicKey, GradidoUnit.zero(), coinCommunityId)) return this.accountBalances } + + try { + lastBalance.updateLegacyDecay(amount, item.balanceDate) + } catch (e) { + if (e instanceof NegativeBalanceError) { + this.logLastBalanceChangingTransactions(publicKey, communityContext.blockchain, 1) + throw e + } + } + this.accountBalances.add(lastBalance.getAccountBalance()) + return this.accountBalances } getUser(item: TransactionDb): { senderUser: UserDb; recipientUser: UserDb } { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/UsersSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/UsersSync.role.ts index 973b0f34c..2e5ae511a 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/UsersSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/UsersSync.role.ts @@ -1,4 +1,4 @@ -import { and, asc, eq, gt, or } from 'drizzle-orm' +import { and, asc, eq, gt, isNotNull, or } from 'drizzle-orm' import { AccountBalance, AccountBalances, @@ -51,6 +51,7 @@ export class UsersSyncRole extends AbstractSyncRole { gt(usersTable.id, lastIndex.id), ), ), + isNotNull(usersTable.communityUuid) ), ) .orderBy(asc(usersTable.createdAt), asc(usersTable.id)) From 94436bfc540c8406c0a206ad056e68958102dd18 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Fri, 27 Feb 2026 16:32:58 +0100 Subject: [PATCH 38/50] fix lint --- .../db-v2.7.0_to_blockchain-v3.7/Context.ts | 6 +----- .../db-v2.7.0_to_blockchain-v3.7/database.ts | 6 ++---- .../RemoteTransactionsSync.role.ts | 19 +++++++++++-------- .../syncDbWithBlockchain/UsersSync.role.ts | 2 +- 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts index f9bf2c7b2..cffccc044 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts @@ -21,11 +21,7 @@ export class Context { public cache: KeyPairCacheManager private timeUsed: Profiler - constructor( - logger: Logger, - db: MySql2Database, - cache: KeyPairCacheManager - ) { + constructor(logger: Logger, db: MySql2Database, cache: KeyPairCacheManager) { this.logger = logger this.db = db this.cache = cache diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/database.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/database.ts index 69f2bc669..1dccb7609 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/database.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/database.ts @@ -54,10 +54,8 @@ export async function loadCommunities(db: MySql2Database): Promise '2000-01-01'`), - ) + and(isNotNull(communitiesTable.communityUuid), sql`${usersTable.createdAt} > '2000-01-01'`), + ) .orderBy(asc(communitiesTable.id)) .groupBy(communitiesTable.communityUuid) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts index ea43e5a2b..8638bad25 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts @@ -125,7 +125,7 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole publicKey: MemoryBlockPtr, ): AccountBalances { this.accountBalances.clear() - + // try to use same coins from this community let lastBalance = this.getLastBalanceForUser( publicKey, @@ -133,11 +133,10 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole coinCommunityId, ) if ( - coinCommunityId != communityContext.communityId && - ( - lastBalance.getBalance().equal(GradidoUnit.zero()) || + coinCommunityId !== communityContext.communityId && + (lastBalance.getBalance().equal(GradidoUnit.zero()) || lastBalance.getBalance().calculateDecay(lastBalance.getDate(), item.balanceDate).lt(amount)) - ) { + ) { // don't work, so we use or own coins lastBalance = this.getLastBalanceForUser( publicKey, @@ -146,8 +145,12 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole ) } if ( - lastBalance.getBalance().calculateDecay(lastBalance.getDate(), item.balanceDate).add(amount).lt(GradidoUnit.zero()) - && communityContext.foreign + lastBalance + .getBalance() + .calculateDecay(lastBalance.getDate(), item.balanceDate) + .add(amount) + .lt(GradidoUnit.zero()) && + communityContext.foreign ) { this.accountBalances.add(new AccountBalance(publicKey, GradidoUnit.zero(), coinCommunityId)) return this.accountBalances @@ -156,7 +159,7 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole try { lastBalance.updateLegacyDecay(amount, item.balanceDate) } catch (e) { - if (e instanceof NegativeBalanceError) { + if (e instanceof NegativeBalanceError) { this.logLastBalanceChangingTransactions(publicKey, communityContext.blockchain, 1) throw e } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/UsersSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/UsersSync.role.ts index 2e5ae511a..ec716e547 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/UsersSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/UsersSync.role.ts @@ -51,7 +51,7 @@ export class UsersSyncRole extends AbstractSyncRole { gt(usersTable.id, lastIndex.id), ), ), - isNotNull(usersTable.communityUuid) + isNotNull(usersTable.communityUuid), ), ) .orderBy(asc(usersTable.createdAt), asc(usersTable.id)) From 9adf5166ce56c5f78c7150f6f948c24280c2ec26 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 28 Feb 2026 09:53:45 +0100 Subject: [PATCH 39/50] add community id to dlt-connector GradidoTransactionBuilder Calls --- .../src/client/GradidoNode/GradidoNodeProcess.ts | 2 +- .../sendToHiero/CommunityRootTransaction.role.ts | 1 + .../interactions/sendToHiero/CreationTransaction.role.ts | 1 + .../sendToHiero/DeferredTransferTransaction.role.ts | 1 + .../RedeemDeferredTransferTransaction.role.ts | 8 ++------ .../sendToHiero/RegisterAddressTransaction.role.ts | 1 + .../interactions/sendToHiero/TransferTransaction.role.ts | 9 ++------- 7 files changed, 9 insertions(+), 14 deletions(-) diff --git a/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts b/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts index a98db7d71..09c9b13ab 100644 --- a/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts +++ b/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts @@ -48,7 +48,7 @@ export class GradidoNodeProcess { const isWindows = process.platform === 'win32' const binaryName = isWindows ? 'GradidoNode.exe' : 'GradidoNode' - return path.join(__dirname, CONFIG.DLT_GRADIDO_NODE_SERVER_HOME_FOLDER, 'bin', binaryName) + return path.join(CONFIG.DLT_GRADIDO_NODE_SERVER_HOME_FOLDER, 'bin', binaryName) } public start() { diff --git a/dlt-connector/src/interactions/sendToHiero/CommunityRootTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/CommunityRootTransaction.role.ts index 4898098f6..1c7cc1fce 100644 --- a/dlt-connector/src/interactions/sendToHiero/CommunityRootTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/CommunityRootTransaction.role.ts @@ -50,6 +50,7 @@ export class CommunityRootTransactionRole extends AbstractTransactionRole { } builder .setCreatedAt(this.community.creationDate) + .setSenderCommunity(this.community.uuid) .setCommunityRoot( communityKeyPair.getPublicKey(), gmwKeyPair.getPublicKey(), diff --git a/dlt-connector/src/interactions/sendToHiero/CreationTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/CreationTransaction.role.ts index 9bd7a7e8c..25f55f145 100644 --- a/dlt-connector/src/interactions/sendToHiero/CreationTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/CreationTransaction.role.ts @@ -65,6 +65,7 @@ export class CreationTransactionRole extends AbstractTransactionRole { new AuthenticatedEncryption(recipientKeyPair), ), ) + .setRecipientCommunity(this.creationTransaction.user.communityId) .setTransactionCreation( new TransferAmount( recipientKeyPair.getPublicKey(), diff --git a/dlt-connector/src/interactions/sendToHiero/DeferredTransferTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/DeferredTransferTransaction.role.ts index 3ac72c0ad..4cad8f0b8 100644 --- a/dlt-connector/src/interactions/sendToHiero/DeferredTransferTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/DeferredTransferTransaction.role.ts @@ -55,6 +55,7 @@ export class DeferredTransferTransactionRole extends AbstractTransactionRole { new AuthenticatedEncryption(recipientKeyPair), ), ) + .setSenderCommunity(this.deferredTransferTransaction.user.communityId) .setDeferredTransfer( new GradidoTransfer( new TransferAmount( diff --git a/dlt-connector/src/interactions/sendToHiero/RedeemDeferredTransferTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/RedeemDeferredTransferTransaction.role.ts index 0c0a3c64e..5053e0211 100644 --- a/dlt-connector/src/interactions/sendToHiero/RedeemDeferredTransferTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/RedeemDeferredTransferTransaction.role.ts @@ -59,6 +59,8 @@ export class RedeemDeferredTransferTransactionRole extends AbstractTransactionRo builder .setCreatedAt(this.redeemDeferredTransferTransaction.createdAt) + .setSenderCommunity(this.redeemDeferredTransferTransaction.user.communityId) + .setRecipientCommunity(this.linkedUser.communityId) .setRedeemDeferredTransfer( this.parentDeferredTransaction.getId(), new GradidoTransfer( @@ -74,12 +76,6 @@ export class RedeemDeferredTransferTransactionRole extends AbstractTransactionRo for (let i = 0; i < memos.size(); i++) { builder.addMemo(memos.get(i)) } - const senderCommunity = this.redeemDeferredTransferTransaction.user.communityTopicId - const recipientCommunity = this.linkedUser.communityTopicId - if (senderCommunity !== recipientCommunity) { - // we have a cross group transaction - builder.setSenderCommunity(senderCommunity).setRecipientCommunity(recipientCommunity) - } builder.sign(senderKeyPair) return builder } diff --git a/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.ts index 7344c9a59..b85b6482d 100644 --- a/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.ts @@ -50,6 +50,7 @@ export class RegisterAddressTransactionRole extends AbstractTransactionRole { builder .setCreatedAt(this.registerAddressTransaction.createdAt) + .setSenderCommunity(this.registerAddressTransaction.user.communityId) .setRegisterAddress( userKeyPair.getPublicKey(), this.registerAddressTransaction.accountType as AddressType, diff --git a/dlt-connector/src/interactions/sendToHiero/TransferTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/TransferTransaction.role.ts index fe9258d74..eb2d19972 100644 --- a/dlt-connector/src/interactions/sendToHiero/TransferTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/TransferTransaction.role.ts @@ -40,7 +40,6 @@ export class TransferTransactionRole extends AbstractTransactionRole { const recipientKeyPair = await ResolveKeyPair( new KeyPairIdentifierLogic(this.transferTransaction.linkedUser), ) - builder .setCreatedAt(this.transferTransaction.createdAt) .addMemo( @@ -50,6 +49,8 @@ export class TransferTransactionRole extends AbstractTransactionRole { new AuthenticatedEncryption(recipientKeyPair), ), ) + .setSenderCommunity(this.transferTransaction.user.communityId) + .setRecipientCommunity(this.transferTransaction.linkedUser.communityId) .setTransactionTransfer( new TransferAmount( senderKeyPair.getPublicKey(), @@ -58,12 +59,6 @@ export class TransferTransactionRole extends AbstractTransactionRole { ), recipientKeyPair.getPublicKey(), ) - const senderCommunity = this.transferTransaction.user.communityTopicId - const recipientCommunity = this.transferTransaction.linkedUser.communityTopicId - if (senderCommunity !== recipientCommunity) { - // we have a cross group transaction - builder.setSenderCommunity(senderCommunity).setRecipientCommunity(recipientCommunity) - } builder.sign(senderKeyPair) return builder } From 941cc5421ef69a6a89c279728da16a766a5d02c7 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 3 Mar 2026 09:34:06 +0100 Subject: [PATCH 40/50] update version of used gradido node and gradido blockchain js --- .../apis/dltConnector/model/CommunityAccountIdentifier.ts | 4 ++-- dlt-connector/bun.lock | 4 ++-- dlt-connector/package.json | 2 +- dlt-connector/src/bootstrap/init.ts | 6 ++++++ dlt-connector/src/client/hiero/HieroClient.ts | 4 ++-- dlt-connector/src/config/schema.ts | 2 +- 6 files changed, 14 insertions(+), 8 deletions(-) diff --git a/backend/src/apis/dltConnector/model/CommunityAccountIdentifier.ts b/backend/src/apis/dltConnector/model/CommunityAccountIdentifier.ts index 1e8b6a64f..e40cb1e54 100644 --- a/backend/src/apis/dltConnector/model/CommunityAccountIdentifier.ts +++ b/backend/src/apis/dltConnector/model/CommunityAccountIdentifier.ts @@ -1,9 +1,9 @@ export class CommunityAccountIdentifier { // for community user, uuid and communityUuid used userUuid: string - accountNr?: number + accountNr: number - constructor(userUuid: string, accountNr?: number) { + constructor(userUuid: string, accountNr: number = 1) { this.userUuid = userUuid this.accountNr = accountNr } diff --git a/dlt-connector/bun.lock b/dlt-connector/bun.lock index 8d1e6925d..bf5b13837 100644 --- a/dlt-connector/bun.lock +++ b/dlt-connector/bun.lock @@ -7,7 +7,7 @@ "dependencies": { "bun-zigar": "^0.15.2", "cross-env": "^7.0.3", - "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#66c8f6839dd95fc34d323066951b80043690fdc2", + "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#5d41ff3839397588e74e77f555c8ff968c786d09", }, "devDependencies": { "@biomejs/biome": "2.0.0", @@ -585,7 +585,7 @@ "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], - "gradido-blockchain-js": ["gradido-blockchain-js@github:gradido/gradido-blockchain-js#66c8f68", { "dependencies": { "bindings": "^1.5.0", "nan": "^2.20.0", "node-addon-api": "^7.1.1", "node-gyp-build": "^4.8.1", "prebuildify": "git+https://github.com/einhornimmond/prebuildify#65d94455fab86b902c0d59bb9c06ac70470e56b2" } }, "gradido-gradido-blockchain-js-66c8f68"], + "gradido-blockchain-js": ["gradido-blockchain-js@github:gradido/gradido-blockchain-js#5d41ff3", { "dependencies": { "bindings": "^1.5.0", "nan": "^2.20.0", "node-addon-api": "^7.1.1", "node-gyp-build": "^4.8.1", "prebuildify": "git+https://github.com/einhornimmond/prebuildify#65d94455fab86b902c0d59bb9c06ac70470e56b2" } }, "gradido-gradido-blockchain-js-5d41ff3"], "graphql": ["graphql@16.11.0", "", {}, "sha512-mS1lbMsxgQj6hge1XZ6p7GPhbrtFwUFYi3wRzXAC/FmYnyXMTvvI3td3rjmQ2u8ewXueaSvRPWaEcgVVOT9Jnw=="], diff --git a/dlt-connector/package.json b/dlt-connector/package.json index ce74c27ba..9e93fa858 100644 --- a/dlt-connector/package.json +++ b/dlt-connector/package.json @@ -20,7 +20,7 @@ "dependencies": { "bun-zigar": "^0.15.2", "cross-env": "^7.0.3", - "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#66c8f6839dd95fc34d323066951b80043690fdc2" + "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#5d41ff3839397588e74e77f555c8ff968c786d09" }, "devDependencies": { "@biomejs/biome": "2.0.0", diff --git a/dlt-connector/src/bootstrap/init.ts b/dlt-connector/src/bootstrap/init.ts index 97a3a03ef..8faef0fd8 100644 --- a/dlt-connector/src/bootstrap/init.ts +++ b/dlt-connector/src/bootstrap/init.ts @@ -9,6 +9,8 @@ import { Community, communitySchema } from '../schemas/transaction.schema' import { isPortOpenRetry } from '../utils/network' import { type AppContext, type AppContextClients } from './appContext' import { initGradidoNode } from './initGradidoNode' +import { ResolveKeyPair } from '../interactions/resolveKeyPair/ResolveKeyPair.context' +import { KeyPairIdentifierLogic } from '../data/KeyPairIdentifier.logic' export function loadConfig(): Logger { // configure log4js @@ -67,6 +69,10 @@ export async function checkHomeCommunity( logger.info(`home community topic: ${homeCommunity.hieroTopicId}`) logger.info(`gradido node server: ${appContext.clients.gradidoNode.url}`) logger.info(`gradido backend server: ${appContext.clients.backend.url}`) + const keyPair = await ResolveKeyPair(new KeyPairIdentifierLogic({ + communityTopicId: homeCommunity.hieroTopicId, + communityId: homeCommunity.uuid, + })) return v.parse(communitySchema, homeCommunity) } diff --git a/dlt-connector/src/client/hiero/HieroClient.ts b/dlt-connector/src/client/hiero/HieroClient.ts index f82139854..088df0b4a 100644 --- a/dlt-connector/src/client/hiero/HieroClient.ts +++ b/dlt-connector/src/client/hiero/HieroClient.ts @@ -100,7 +100,7 @@ export class HieroClient { ) // TODO: fix issue in GradidoNode // hot fix, when gradido node is running some time, the hiero listener stop working, so we check if our new transaction is received - // after 10 seconds, else restart GradidoNode + // after 20 seconds, else restart GradidoNode setTimeout(async () => { const transaction = await GradidoNodeClient.getInstance().getTransaction({ communityId, @@ -122,7 +122,7 @@ export class HieroClient { GradidoNodeProcess.getInstance().start() } } - }, 10000) + }, 20000) if (logger.isInfoEnabled()) { // only for logging sendResponse.getReceiptWithSigner(this.wallet).then((receipt) => { diff --git a/dlt-connector/src/config/schema.ts b/dlt-connector/src/config/schema.ts index dd61e59d9..fcb1452de 100644 --- a/dlt-connector/src/config/schema.ts +++ b/dlt-connector/src/config/schema.ts @@ -84,7 +84,7 @@ export const configSchema = v.object({ v.string('The version of the DLT node server, for example: 0.9.0'), v.regex(/^\d+\.\d+\.\d+$/), ), - '0.9.3', + '0.9.4', ), DLT_GRADIDO_NODE_SERVER_HOME_FOLDER: v.optional( v.string('The home folder for the gradido dlt node server'), From 60799554f110e26ed973a2171bcd858ea8be36c3 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 3 Mar 2026 09:35:30 +0100 Subject: [PATCH 41/50] update inspector --- inspector | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inspector b/inspector index 173fb6a96..e1d13e333 160000 --- a/inspector +++ b/inspector @@ -1 +1 @@ -Subproject commit 173fb6a9626ba1e8c69ba0cc04f87e1c5ffb1570 +Subproject commit e1d13e3336199eae615557d11d6671c034860326 From f7e1f55a5b5a887fe17171a0530d64ea865afa62 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 3 Mar 2026 09:36:41 +0100 Subject: [PATCH 42/50] fix lint --- dlt-connector/src/bootstrap/init.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/dlt-connector/src/bootstrap/init.ts b/dlt-connector/src/bootstrap/init.ts index 8faef0fd8..f428b4218 100644 --- a/dlt-connector/src/bootstrap/init.ts +++ b/dlt-connector/src/bootstrap/init.ts @@ -4,13 +4,13 @@ import { configure, getLogger, Logger } from 'log4js' import * as v from 'valibot' import { CONFIG } from '../config' import { MIN_TOPIC_EXPIRE_MILLISECONDS_FOR_UPDATE } from '../config/const' +import { KeyPairIdentifierLogic } from '../data/KeyPairIdentifier.logic' +import { ResolveKeyPair } from '../interactions/resolveKeyPair/ResolveKeyPair.context' import { SendToHieroContext } from '../interactions/sendToHiero/SendToHiero.context' import { Community, communitySchema } from '../schemas/transaction.schema' import { isPortOpenRetry } from '../utils/network' import { type AppContext, type AppContextClients } from './appContext' import { initGradidoNode } from './initGradidoNode' -import { ResolveKeyPair } from '../interactions/resolveKeyPair/ResolveKeyPair.context' -import { KeyPairIdentifierLogic } from '../data/KeyPairIdentifier.logic' export function loadConfig(): Logger { // configure log4js @@ -69,10 +69,12 @@ export async function checkHomeCommunity( logger.info(`home community topic: ${homeCommunity.hieroTopicId}`) logger.info(`gradido node server: ${appContext.clients.gradidoNode.url}`) logger.info(`gradido backend server: ${appContext.clients.backend.url}`) - const keyPair = await ResolveKeyPair(new KeyPairIdentifierLogic({ - communityTopicId: homeCommunity.hieroTopicId, - communityId: homeCommunity.uuid, - })) + await ResolveKeyPair( + new KeyPairIdentifierLogic({ + communityTopicId: homeCommunity.hieroTopicId, + communityId: homeCommunity.uuid, + }), + ) return v.parse(communitySchema, homeCommunity) } From bf438b8b36496e8af5d01e73c99c5548f29fab97 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 3 Mar 2026 10:07:10 +0100 Subject: [PATCH 43/50] fix tests --- .../src/client/GradidoNode/input.schema.test.ts | 2 +- .../resolveKeyPair/ResolveKeyPair.context.test.ts | 5 ++++- .../RegisterAddressTransaction.role.test.ts | 5 +++++ dlt-connector/src/schemas/transaction.schema.test.ts | 2 +- dlt-connector/src/schemas/typeConverter.schema.test.ts | 10 ++++++++-- dlt-connector/src/schemas/typeGuard.schema.test.ts | 10 +++++----- dlt-connector/src/server/index.test.ts | 5 +++++ 7 files changed, 29 insertions(+), 10 deletions(-) diff --git a/dlt-connector/src/client/GradidoNode/input.schema.test.ts b/dlt-connector/src/client/GradidoNode/input.schema.test.ts index 4db653f26..0b7b4dc71 100644 --- a/dlt-connector/src/client/GradidoNode/input.schema.test.ts +++ b/dlt-connector/src/client/GradidoNode/input.schema.test.ts @@ -48,7 +48,7 @@ describe('transactionIdentifierSchema ', () => { transactionId: 1, hieroTransactionId: '0.0.261-1755348116-1281621', }), - ).toThrowError(new Error('Invalid key: Expected "topic" but received undefined')) + ).toThrowError(new Error('Invalid key: Expected "communityId" but received undefined')) }) it('invalid, transactionNr and iotaMessageId set', () => { expect(() => diff --git a/dlt-connector/src/interactions/resolveKeyPair/ResolveKeyPair.context.test.ts b/dlt-connector/src/interactions/resolveKeyPair/ResolveKeyPair.context.test.ts index 16a444cd8..3541ae943 100644 --- a/dlt-connector/src/interactions/resolveKeyPair/ResolveKeyPair.context.test.ts +++ b/dlt-connector/src/interactions/resolveKeyPair/ResolveKeyPair.context.test.ts @@ -33,6 +33,7 @@ mock.module('../../config', () => ({ })) const topicId = '0.0.21732' +const communityId = '1e88a0f4-d4fc-4cae-a7e8-a88e613ce324' const userUuid = 'aa25cf6f-2879-4745-b2ea-6d3c37fb44b0' afterAll(() => { @@ -45,7 +46,7 @@ describe('KeyPairCalculation', () => { }) it('community key pair', async () => { const identifier = new KeyPairIdentifierLogic( - v.parse(identifierKeyPairSchema, { communityTopicId: topicId }), + v.parse(identifierKeyPairSchema, { communityId, communityTopicId: topicId }), ) const keyPair = await ResolveKeyPair(identifier) expect(keyPair.getPublicKey()?.convertToHex()).toBe( @@ -55,6 +56,7 @@ describe('KeyPairCalculation', () => { it('user key pair', async () => { const identifier = new KeyPairIdentifierLogic( v.parse(identifierKeyPairSchema, { + communityId, communityTopicId: topicId, account: { userUuid }, }), @@ -70,6 +72,7 @@ describe('KeyPairCalculation', () => { it('account key pair', async () => { const identifier = new KeyPairIdentifierLogic( v.parse(identifierKeyPairSchema, { + communityId, communityTopicId: topicId, account: { userUuid, accountNr: 1 }, }), diff --git a/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.test.ts b/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.test.ts index ab07a2f83..c12694797 100644 --- a/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.test.ts +++ b/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.test.ts @@ -4,11 +4,14 @@ import * as v from 'valibot' import { transactionSchema } from '../../schemas/transaction.schema' import { hieroIdSchema } from '../../schemas/typeGuard.schema' import { RegisterAddressTransactionRole } from './RegisterAddressTransaction.role' +import { InMemoryBlockchainProvider } from 'gradido-blockchain-js' const userUuid = '408780b2-59b3-402a-94be-56a4f4f4e8ec' +const communityId = '1e88a0f4-d4fc-4cae-a7e8-a88e613ce324' const transaction = { user: { communityTopicId: '0.0.21732', + communityId, account: { userUuid, accountNr: 0, @@ -18,6 +21,8 @@ const transaction = { accountType: 'COMMUNITY_HUMAN', createdAt: '2022-01-01T00:00:00.000Z', } +// create blockchain in native module +InMemoryBlockchainProvider.getInstance().getBlockchain(communityId) describe('RegisterAddressTransaction.role', () => { it('get correct prepared builder', async () => { diff --git a/dlt-connector/src/schemas/transaction.schema.test.ts b/dlt-connector/src/schemas/transaction.schema.test.ts index e8a6a3b3c..de2692632 100644 --- a/dlt-connector/src/schemas/transaction.schema.test.ts +++ b/dlt-connector/src/schemas/transaction.schema.test.ts @@ -35,7 +35,7 @@ const transactionLinkCode = (date: Date): string => { let topic: HieroId const topicString = '0.0.261' let communityUuid: Uuidv4 -const communityUuidString = '123e4567-e89b-12d3-a456-426614174000' +const communityUuidString = 'fcd48487-6d31-4f4c-be9b-b3c8ca853912' beforeAll(() => { topic = v.parse(hieroIdSchema, topicString) diff --git a/dlt-connector/src/schemas/typeConverter.schema.test.ts b/dlt-connector/src/schemas/typeConverter.schema.test.ts index a46341599..75dc0f1fe 100644 --- a/dlt-connector/src/schemas/typeConverter.schema.test.ts +++ b/dlt-connector/src/schemas/typeConverter.schema.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'bun:test' import { TypeCompiler } from '@sinclair/typebox/compiler' import { Static, TypeBoxFromValibot } from '@sinclair/typemap' -import { AddressType_COMMUNITY_AUF } from 'gradido-blockchain-js' +import { AddressType_COMMUNITY_AUF, InMemoryBlockchainProvider } from 'gradido-blockchain-js' import * as v from 'valibot' import { AccountType } from '../data/AccountType.enum' import { @@ -96,9 +96,15 @@ describe('basic.schema', () => { }) it('confirmedTransactionSchema', () => { + // create blockchain in native module + const communityId = 'fcd48487-6d31-4f4c-be9b-b3c8ca853912' + InMemoryBlockchainProvider.getInstance().getBlockchain(communityId) const confirmedTransaction = v.parse( confirmedTransactionSchema, - 'CAcS5AEKZgpkCiCBZwMplGmI7fRR9MQkaR2Dz1qQQ5BCiC1btyJD71Ue9BJABODQ9sS70th9yHn8X3K+SNv2gsiIdX/V09baCvQCb+yEj2Dd/fzShIYqf3pooIMwJ01BkDJdNGBZs5MDzEAkChJ6ChkIAhIVRGFua2UgZnVlciBkZWluIFNlaW4hEggIgMy5/wUQABoDMy41IAAyTAooCiDbDtYSWhTwMKvtG/yDHgohjPn6v87n7NWBwMDniPAXxxCUmD0aABIgJE0o18xb6P6PsNjh0bkN52AzhggteTzoh09jV+blMq0aCAjC8rn/BRAAIgMzLjUqICiljeEjGHifWe4VNzoe+DN9oOLNZvJmv3VlkP+1RH7MMiAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADomCiDbDtYSWhTwMKvtG/yDHgohjPn6v87n7NWBwMDniPAXxxDAhD06JwogJE0o18xb6P6PsNjh0bkN52AzhggteTzoh09jV+blMq0Q65SlBA==', + { + base64: 'CAcS4AEKZgpkCiCBZwMplGmI7fRR9MQkaR2Dz1qQQ5BCiC1btyJD71Ue9BJABODQ9sS70th9yHn8X3K+SNv2gsiIdX/V09baCvQCb+zo7nEQgCUXOEe/tN7YaRppwt6TDcXBPxkwnw4gfpCODhJ0ChkIAhIVRGFua2UgZnVlciBkZWluIFNlaW4hEgYIgMy5/wUaAzMuNTJKCiYKINsO1hJaFPAwq+0b/IMeCiGM+fq/zufs1YHAwOeI8BfHEJSYPRIgJE0o18xb6P6PsNjh0bkN52AzhggteTzoh09jV+blMq0aABoGCMLyuf8FIgMzLjcqIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMhUIAhoRCgkIqemnUhD+4wESBBj8sgc6Jgog2w7WEloU8DCr7Rv8gx4KIYz5+r/O5+zVgcDA54jwF8cQwIQ9OicKICRNKNfMW+j+j7DY4dG5DedgM4YILXk86IdPY1fm5TKtEOuUpQRAAg==', + communityId, + }, ) expect(confirmedTransaction.getId()).toBe(7) expect(confirmedTransaction.getConfirmedAt().getSeconds()).toBe(1609464130) diff --git a/dlt-connector/src/schemas/typeGuard.schema.test.ts b/dlt-connector/src/schemas/typeGuard.schema.test.ts index ecd9eca7a..52e05c76e 100644 --- a/dlt-connector/src/schemas/typeGuard.schema.test.ts +++ b/dlt-connector/src/schemas/typeGuard.schema.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'bun:test' import { v4 as uuidv4 } from 'uuid' import * as v from 'valibot' -import { memoSchema, uuidv4Schema } from './typeGuard.schema' +import { memoSchema, MEMO_MAX_CHARS, MEMO_MIN_CHARS, uuidv4Schema } from './typeGuard.schema' describe('typeGuard.schema', () => { describe('Uuidv4', () => { @@ -20,18 +20,18 @@ describe('typeGuard.schema', () => { expect(memoValueParsed.toString()).toBe(memoValue) }) it('max length', () => { - const memoValue = 's'.repeat(255) + const memoValue = 's'.repeat(MEMO_MAX_CHARS) const memoValueParsed = v.parse(memoSchema, memoValue) expect(memoValueParsed.toString()).toBe(memoValue) }) it('to short', () => { const memoValue = 'memo' - expect(() => v.parse(memoSchema, memoValue)).toThrow(new Error('expect string length >= 5')) + expect(() => v.parse(memoSchema, memoValue)).toThrow(new Error(`expect string length >= ${MEMO_MIN_CHARS}`)) }) it('to long', () => { - const memoValue = 's'.repeat(256) + const memoValue = 's'.repeat(MEMO_MAX_CHARS + 1) expect(() => v.parse(memoSchema, memoValue)).toThrow( - new Error('expect string length <= 255'), + new Error(`expect string length <= ${MEMO_MAX_CHARS}`), ) }) }) diff --git a/dlt-connector/src/server/index.test.ts b/dlt-connector/src/server/index.test.ts index 4b6b8be76..72419de0d 100644 --- a/dlt-connector/src/server/index.test.ts +++ b/dlt-connector/src/server/index.test.ts @@ -5,6 +5,7 @@ import * as v from 'valibot' import { KeyPairCacheManager } from '../cache/KeyPairCacheManager' import { HieroId, hieroIdSchema } from '../schemas/typeGuard.schema' import { appRoutes } from '.' +import { InMemoryBlockchainProvider } from 'gradido-blockchain-js' const userUuid = '408780b2-59b3-402a-94be-56a4f4f4e8ec' @@ -55,9 +56,13 @@ beforeAll(() => { describe('Server', () => { it('send register address transaction', async () => { + // create blockchain in native module + const communityId = '1e88a0f4-d4fc-4cae-a7e8-a88e613ce324' + InMemoryBlockchainProvider.getInstance().getBlockchain(communityId) const transaction = { user: { communityTopicId: '0.0.21732', + communityId, account: { userUuid, accountNr: 0, From fd061e3de4306f720435ef7687cf34208e770999 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 3 Mar 2026 10:16:29 +0100 Subject: [PATCH 44/50] run core text core test separate --- .github/workflows/test_submodules.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_submodules.yml b/.github/workflows/test_submodules.yml index cbdfd16a2..8d54e55de 100644 --- a/.github/workflows/test_submodules.yml +++ b/.github/workflows/test_submodules.yml @@ -45,5 +45,7 @@ jobs: bun install --global turbo@^2 - name: typecheck, locales && unit test - run: turbo core#test core#typecheck core#locales database#test database#typecheck shared#test shared#typecheck config-schema#test config-schema#typecheck + run: turbo core#typecheck core#locales database#test database#typecheck shared#test shared#typecheck config-schema#test config-schema#typecheck + - name: core test extra + run: turbo core#test From 22b73712b928b1af81890c792f72f153397c4cba Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 3 Mar 2026 11:35:35 +0100 Subject: [PATCH 45/50] fix lint of updated tests, fix wrong import --- .../RegisterAddressTransaction.role.test.ts | 7 +++++-- .../RemoteTransactionsSync.role.ts | 2 +- .../src/schemas/typeConverter.schema.test.ts | 12 +++++------- dlt-connector/src/schemas/typeGuard.schema.test.ts | 6 ++++-- dlt-connector/src/server/index.test.ts | 8 ++++++-- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.test.ts b/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.test.ts index c12694797..c1be4d10e 100644 --- a/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.test.ts +++ b/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.test.ts @@ -1,10 +1,13 @@ import { describe, expect, it } from 'bun:test' -import { InteractionValidate, ValidateType_SINGLE } from 'gradido-blockchain-js' +import { + InMemoryBlockchainProvider, + InteractionValidate, + ValidateType_SINGLE, +} from 'gradido-blockchain-js' import * as v from 'valibot' import { transactionSchema } from '../../schemas/transaction.schema' import { hieroIdSchema } from '../../schemas/typeGuard.schema' import { RegisterAddressTransactionRole } from './RegisterAddressTransaction.role' -import { InMemoryBlockchainProvider } from 'gradido-blockchain-js' const userUuid = '408780b2-59b3-402a-94be-56a4f4f4e8ec' const communityId = '1e88a0f4-d4fc-4cae-a7e8-a88e613ce324' diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts index 8638bad25..308af4911 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts @@ -1,4 +1,4 @@ -import { Decimal } from 'decimal.js' +import { Decimal } from 'decimal.js-light' import { and, asc, eq, gt, inArray, isNull, ne, or } from 'drizzle-orm' import { alias } from 'drizzle-orm/mysql-core' import { diff --git a/dlt-connector/src/schemas/typeConverter.schema.test.ts b/dlt-connector/src/schemas/typeConverter.schema.test.ts index 75dc0f1fe..27259ee1e 100644 --- a/dlt-connector/src/schemas/typeConverter.schema.test.ts +++ b/dlt-connector/src/schemas/typeConverter.schema.test.ts @@ -99,13 +99,11 @@ describe('basic.schema', () => { // create blockchain in native module const communityId = 'fcd48487-6d31-4f4c-be9b-b3c8ca853912' InMemoryBlockchainProvider.getInstance().getBlockchain(communityId) - const confirmedTransaction = v.parse( - confirmedTransactionSchema, - { - base64: 'CAcS4AEKZgpkCiCBZwMplGmI7fRR9MQkaR2Dz1qQQ5BCiC1btyJD71Ue9BJABODQ9sS70th9yHn8X3K+SNv2gsiIdX/V09baCvQCb+zo7nEQgCUXOEe/tN7YaRppwt6TDcXBPxkwnw4gfpCODhJ0ChkIAhIVRGFua2UgZnVlciBkZWluIFNlaW4hEgYIgMy5/wUaAzMuNTJKCiYKINsO1hJaFPAwq+0b/IMeCiGM+fq/zufs1YHAwOeI8BfHEJSYPRIgJE0o18xb6P6PsNjh0bkN52AzhggteTzoh09jV+blMq0aABoGCMLyuf8FIgMzLjcqIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMhUIAhoRCgkIqemnUhD+4wESBBj8sgc6Jgog2w7WEloU8DCr7Rv8gx4KIYz5+r/O5+zVgcDA54jwF8cQwIQ9OicKICRNKNfMW+j+j7DY4dG5DedgM4YILXk86IdPY1fm5TKtEOuUpQRAAg==', - communityId, - }, - ) + const confirmedTransaction = v.parse(confirmedTransactionSchema, { + base64: + 'CAcS4AEKZgpkCiCBZwMplGmI7fRR9MQkaR2Dz1qQQ5BCiC1btyJD71Ue9BJABODQ9sS70th9yHn8X3K+SNv2gsiIdX/V09baCvQCb+zo7nEQgCUXOEe/tN7YaRppwt6TDcXBPxkwnw4gfpCODhJ0ChkIAhIVRGFua2UgZnVlciBkZWluIFNlaW4hEgYIgMy5/wUaAzMuNTJKCiYKINsO1hJaFPAwq+0b/IMeCiGM+fq/zufs1YHAwOeI8BfHEJSYPRIgJE0o18xb6P6PsNjh0bkN52AzhggteTzoh09jV+blMq0aABoGCMLyuf8FIgMzLjcqIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMhUIAhoRCgkIqemnUhD+4wESBBj8sgc6Jgog2w7WEloU8DCr7Rv8gx4KIYz5+r/O5+zVgcDA54jwF8cQwIQ9OicKICRNKNfMW+j+j7DY4dG5DedgM4YILXk86IdPY1fm5TKtEOuUpQRAAg==', + communityId, + }) expect(confirmedTransaction.getId()).toBe(7) expect(confirmedTransaction.getConfirmedAt().getSeconds()).toBe(1609464130) }) diff --git a/dlt-connector/src/schemas/typeGuard.schema.test.ts b/dlt-connector/src/schemas/typeGuard.schema.test.ts index 52e05c76e..445d83c9c 100644 --- a/dlt-connector/src/schemas/typeGuard.schema.test.ts +++ b/dlt-connector/src/schemas/typeGuard.schema.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'bun:test' import { v4 as uuidv4 } from 'uuid' import * as v from 'valibot' -import { memoSchema, MEMO_MAX_CHARS, MEMO_MIN_CHARS, uuidv4Schema } from './typeGuard.schema' +import { MEMO_MAX_CHARS, MEMO_MIN_CHARS, memoSchema, uuidv4Schema } from './typeGuard.schema' describe('typeGuard.schema', () => { describe('Uuidv4', () => { @@ -26,7 +26,9 @@ describe('typeGuard.schema', () => { }) it('to short', () => { const memoValue = 'memo' - expect(() => v.parse(memoSchema, memoValue)).toThrow(new Error(`expect string length >= ${MEMO_MIN_CHARS}`)) + expect(() => v.parse(memoSchema, memoValue)).toThrow( + new Error(`expect string length >= ${MEMO_MIN_CHARS}`), + ) }) it('to long', () => { const memoValue = 's'.repeat(MEMO_MAX_CHARS + 1) diff --git a/dlt-connector/src/server/index.test.ts b/dlt-connector/src/server/index.test.ts index 72419de0d..22d1c775b 100644 --- a/dlt-connector/src/server/index.test.ts +++ b/dlt-connector/src/server/index.test.ts @@ -1,11 +1,15 @@ import { beforeAll, describe, expect, it, mock } from 'bun:test' import { AccountId, Timestamp, TransactionId } from '@hashgraph/sdk' -import { GradidoTransaction, KeyPairEd25519, MemoryBlock } from 'gradido-blockchain-js' +import { + GradidoTransaction, + InMemoryBlockchainProvider, + KeyPairEd25519, + MemoryBlock, +} from 'gradido-blockchain-js' import * as v from 'valibot' import { KeyPairCacheManager } from '../cache/KeyPairCacheManager' import { HieroId, hieroIdSchema } from '../schemas/typeGuard.schema' import { appRoutes } from '.' -import { InMemoryBlockchainProvider } from 'gradido-blockchain-js' const userUuid = '408780b2-59b3-402a-94be-56a4f4f4e8ec' From f9e1be946e1242b7a744bec4267137ffc205f3c4 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 3 Mar 2026 12:20:21 +0100 Subject: [PATCH 46/50] explicit disable email test modus in e-email test --- core/src/emails/sendEmailVariants.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/emails/sendEmailVariants.test.ts b/core/src/emails/sendEmailVariants.test.ts index 1fbbbce8f..d7556a4d4 100644 --- a/core/src/emails/sendEmailVariants.test.ts +++ b/core/src/emails/sendEmailVariants.test.ts @@ -25,6 +25,7 @@ CONFIG.EMAIL_SENDER = 'info@gradido.net' CONFIG.EMAIL_SMTP_HOST = testMailServerHost CONFIG.EMAIL_SMTP_PORT = testMailServerPort CONFIG.EMAIL_TLS = testMailTLS +CONFIG.EMAIL_TEST_MODUS = false mock.module('nodemailer', () => { return { From 9f4fa9f5b2943dc048a4d69257718c50e356705a Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Fri, 6 Mar 2026 08:37:51 +0100 Subject: [PATCH 47/50] add hieroTopicId to publicCommunityInfo query --- .../src/federation/client/1_0/query/getPublicCommunityInfo.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/federation/client/1_0/query/getPublicCommunityInfo.ts b/backend/src/federation/client/1_0/query/getPublicCommunityInfo.ts index 36350fae8..8e00bb506 100644 --- a/backend/src/federation/client/1_0/query/getPublicCommunityInfo.ts +++ b/backend/src/federation/client/1_0/query/getPublicCommunityInfo.ts @@ -8,6 +8,7 @@ export const getPublicCommunityInfo = gql` creationDate publicKey publicJwtKey + hieroTopicId } } ` From aff9e5f816f93302b50a87471c2523aa78fe8233 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Fri, 6 Mar 2026 08:39:17 +0100 Subject: [PATCH 48/50] trigger timeout transactions directly from typeScript --- bun.lock | 35 +++++++------- .../data/Balance.ts | 9 ++++ .../syncDbWithBlockchain/AbstractSync.role.ts | 1 + .../CreationsSync.role.ts | 4 ++ .../DeletedTransactionLinksSync.role.ts | 4 ++ .../LocalTransactionsSync.role.ts | 5 ++ .../RedeemTransactionLinksSync.role.ts | 5 ++ .../RemoteTransactionsSync.role.ts | 6 +++ .../TransactionLinkFundingsSync.role.ts | 9 ++++ .../syncDbWithBlockchain/UsersSync.role.ts | 5 ++ .../syncDbWithBlockchain.context.ts | 47 ++++++++++++++++++- package.json | 2 +- 12 files changed, 112 insertions(+), 20 deletions(-) diff --git a/bun.lock b/bun.lock index 9a4a4c2bb..24dd8bf42 100644 --- a/bun.lock +++ b/bun.lock @@ -1,6 +1,5 @@ { "lockfileVersion": 1, - "configVersion": 0, "workspaces": { "": { "name": "gradido", @@ -8,7 +7,7 @@ "auto-changelog": "^2.4.0", "cross-env": "^7.0.3", "jose": "^4.14.4", - "turbo": "^2.5.0", + "turbo": "^2.8.12", "uuid": "^8.3.2", }, "devDependencies": { @@ -19,7 +18,7 @@ }, "admin": { "name": "admin", - "version": "2.7.3", + "version": "2.7.4", "dependencies": { "@iconify/json": "^2.2.228", "@popperjs/core": "^2.11.8", @@ -89,7 +88,7 @@ }, "backend": { "name": "backend", - "version": "2.7.3", + "version": "2.7.4", "dependencies": { "cross-env": "^7.0.3", "email-templates": "^10.0.1", @@ -166,7 +165,7 @@ }, "config-schema": { "name": "config-schema", - "version": "2.7.3", + "version": "2.7.4", "dependencies": { "esbuild": "^0.25.2", "joi": "17.13.3", @@ -184,7 +183,7 @@ }, "core": { "name": "core", - "version": "2.7.3", + "version": "2.7.4", "dependencies": { "database": "*", "email-templates": "^10.0.1", @@ -221,7 +220,7 @@ }, "database": { "name": "database", - "version": "2.7.3", + "version": "2.7.4", "dependencies": { "@types/uuid": "^8.3.4", "cross-env": "^7.0.3", @@ -257,7 +256,7 @@ }, "dht-node": { "name": "dht-node", - "version": "2.7.3", + "version": "2.7.4", "dependencies": { "cross-env": "^7.0.3", "dht-rpc": "6.18.1", @@ -295,7 +294,7 @@ }, "federation": { "name": "federation", - "version": "2.7.3", + "version": "2.7.4", "dependencies": { "cross-env": "^7.0.3", "email-templates": "^10.0.1", @@ -356,7 +355,7 @@ }, "frontend": { "name": "frontend", - "version": "2.7.3", + "version": "2.7.4", "dependencies": { "@morev/vue-transitions": "^3.0.2", "@types/leaflet": "^1.9.12", @@ -452,7 +451,7 @@ }, "shared": { "name": "shared", - "version": "2.7.3", + "version": "2.7.4", "dependencies": { "decimal.js-light": "^2.5.1", "esbuild": "^0.25.2", @@ -3358,19 +3357,19 @@ "tunnel": ["tunnel@0.0.6", "", {}, "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg=="], - "turbo": ["turbo@2.6.1", "", { "optionalDependencies": { "turbo-darwin-64": "2.6.1", "turbo-darwin-arm64": "2.6.1", "turbo-linux-64": "2.6.1", "turbo-linux-arm64": "2.6.1", "turbo-windows-64": "2.6.1", "turbo-windows-arm64": "2.6.1" }, "bin": { "turbo": "bin/turbo" } }, "sha512-qBwXXuDT3rA53kbNafGbT5r++BrhRgx3sAo0cHoDAeG9g1ItTmUMgltz3Hy7Hazy1ODqNpR+C7QwqL6DYB52yA=="], + "turbo": ["turbo@2.8.12", "", { "optionalDependencies": { "turbo-darwin-64": "2.8.12", "turbo-darwin-arm64": "2.8.12", "turbo-linux-64": "2.8.12", "turbo-linux-arm64": "2.8.12", "turbo-windows-64": "2.8.12", "turbo-windows-arm64": "2.8.12" }, "bin": { "turbo": "bin/turbo" } }, "sha512-auUAMLmi0eJhxDhQrxzvuhfEbICnVt0CTiYQYY8WyRJ5nwCDZxD0JG8bCSxT4nusI2CwJzmZAay5BfF6LmK7Hw=="], - "turbo-darwin-64": ["turbo-darwin-64@2.6.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-Dm0HwhyZF4J0uLqkhUyCVJvKM9Rw7M03v3J9A7drHDQW0qAbIGBrUijQ8g4Q9Cciw/BXRRd8Uzkc3oue+qn+ZQ=="], + "turbo-darwin-64": ["turbo-darwin-64@2.8.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-EiHJmW2MeQQx+21x8hjMHw/uPhXt9PIxvDrxzOtyVwrXzL0tQmsxtO4qHf2l7uA+K6PUJ4+TjY1MHZDuCvWXrw=="], - "turbo-darwin-arm64": ["turbo-darwin-arm64@2.6.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-U0PIPTPyxdLsrC3jN7jaJUwgzX5sVUBsKLO7+6AL+OASaa1NbT1pPdiZoTkblBAALLP76FM0LlnsVQOnmjYhyw=="], + "turbo-darwin-arm64": ["turbo-darwin-arm64@2.8.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-cbqqGN0vd7ly2TeuaM8k9AK9u1CABO4kBA5KPSqovTiLL3sORccn/mZzJSbvQf0EsYRfU34MgW5FotfwW3kx8Q=="], - "turbo-linux-64": ["turbo-linux-64@2.6.1", "", { "os": "linux", "cpu": "x64" }, "sha512-eM1uLWgzv89bxlK29qwQEr9xYWBhmO/EGiH22UGfq+uXr+QW1OvNKKMogSN65Ry8lElMH4LZh0aX2DEc7eC0Mw=="], + "turbo-linux-64": ["turbo-linux-64@2.8.12", "", { "os": "linux", "cpu": "x64" }, "sha512-jXKw9j4r4q6s0goSXuKI3aKbQK2qiNeP25lGGEnq018TM6SWRW1CCpPMxyG91aCKrub7wDm/K45sGNT4ZFBcFQ=="], - "turbo-linux-arm64": ["turbo-linux-arm64@2.6.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-MFFh7AxAQAycXKuZDrbeutfWM5Ep0CEZ9u7zs4Hn2FvOViTCzIfEhmuJou3/a5+q5VX1zTxQrKGy+4Lf5cdpsA=="], + "turbo-linux-arm64": ["turbo-linux-arm64@2.8.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-BRJCMdyXjyBoL0GYpvj9d2WNfMHwc3tKmJG5ATn2Efvil9LsiOsd/93/NxDqW0jACtHFNVOPnd/CBwXRPiRbwA=="], - "turbo-windows-64": ["turbo-windows-64@2.6.1", "", { "os": "win32", "cpu": "x64" }, "sha512-buq7/VAN7KOjMYi4tSZT5m+jpqyhbRU2EUTTvp6V0Ii8dAkY2tAAjQN1q5q2ByflYWKecbQNTqxmVploE0LVwQ=="], + "turbo-windows-64": ["turbo-windows-64@2.8.12", "", { "os": "win32", "cpu": "x64" }, "sha512-vyFOlpFFzQFkikvSVhVkESEfzIopgs2J7J1rYvtSwSHQ4zmHxkC95Q8Kjkus8gg+8X2mZyP1GS5jirmaypGiPw=="], - "turbo-windows-arm64": ["turbo-windows-arm64@2.6.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-7w+AD5vJp3R+FB0YOj1YJcNcOOvBior7bcHTodqp90S3x3bLgpr7tE6xOea1e8JkP7GK6ciKVUpQvV7psiwU5Q=="], + "turbo-windows-arm64": ["turbo-windows-arm64@2.8.12", "", { "os": "win32", "cpu": "arm64" }, "sha512-9nRnlw5DF0LkJClkIws1evaIF36dmmMEO84J5Uj4oQ8C0QTHwlH7DNe5Kq2Jdmu8GXESCNDNuUYG8Cx6W/vm3g=="], "type-check": ["type-check@0.4.0", "", { "dependencies": { "prelude-ls": "^1.2.1" } }, "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew=="], diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/Balance.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/Balance.ts index bc5dcc9cd..7f96b1d16 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/Balance.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/Balance.ts @@ -101,4 +101,13 @@ export class Balance { getAccountBalance(): AccountBalance { return new AccountBalance(this.publicKey, this.balance, this.communityId) } + + toString(): string { + return JSON.stringify({ + balance: this.balance.toString(), + date: this.date, + publicKey: this.publicKey.convertToHex(), + communityId: this.communityId + }, null, 2) + } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts index 572252fd9..9a2cfe660 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts @@ -109,6 +109,7 @@ export abstract class AbstractSyncRole { abstract loadFromDb(lastIndex: IndexType, count: number): Promise abstract pushToBlockchain(item: ItemType): void abstract itemTypeName(): string + abstract getCommunityUuids(): Uuidv4[] // return count of new loaded items async ensureFilled(batchSize: number): Promise { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts index b5555c3a2..9ffa11231 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts @@ -26,6 +26,7 @@ import { creationTransactionDbSchema, } from '../../valibot.schema' import { AbstractSyncRole, IndexType } from './AbstractSync.role' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' export class CreationsSyncRole extends AbstractSyncRole { constructor(context: Context) { @@ -36,6 +37,9 @@ export class CreationsSyncRole extends AbstractSyncRole { getDate(): Date { return this.peek().confirmedAt } + getCommunityUuids(): Uuidv4[] { + return [ this.peek().user.communityUuid ] + } getLastIndex(): IndexType { const lastItem = this.peekLast() diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts index 4aa5159af..3ce1bcef4 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts @@ -26,6 +26,7 @@ import { deletedTransactionLinKDbSchema, } from '../../valibot.schema' import { AbstractSyncRole, IndexType } from './AbstractSync.role' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { constructor(context: Context) { @@ -36,6 +37,9 @@ export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { constructor(context: Context) { @@ -35,6 +36,10 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole { return this.peek().balanceDate } + getCommunityUuids(): Uuidv4[] { + return [this.peek().user.communityUuid] + } + getLastIndex(): IndexType { const lastItem = this.peekLast() return { date: lastItem.balanceDate, id: lastItem.id } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts index 6e0c9f6ba..ffdb84d04 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts @@ -28,6 +28,7 @@ import { redeemedTransactionLinkDbSchema, } from '../../valibot.schema' import { AbstractSyncRole, IndexType } from './AbstractSync.role' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { constructor(context: Context) { @@ -39,6 +40,10 @@ export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { constructor(context: Context) { @@ -38,6 +39,11 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole return this.peek().balanceDate } + getCommunityUuids(): Uuidv4[] { + const currentItem = this.peek() + return [ currentItem.user.communityUuid, currentItem.linkedUser.communityUuid ] + } + getLastIndex(): IndexType { const lastItem = this.peekLast() return { date: lastItem.balanceDate, id: lastItem.id } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts index 918d33c3b..7d93d6921 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts @@ -6,12 +6,15 @@ import { AuthenticatedEncryption, DurationSeconds, EncryptedMemo, + Filter, GradidoTransactionBuilder, GradidoTransfer, GradidoUnit, KeyPairEd25519, LedgerAnchor, MemoryBlockPtr, + SearchDirection_DESC, + transactionTypeToString, TransferAmount, } from 'gradido-blockchain-js' import * as v from 'valibot' @@ -23,6 +26,7 @@ import { BlockchainError, DatabaseError, NegativeBalanceError } from '../../erro import { reverseLegacyDecay, toMysqlDateTime } from '../../utils' import { CommunityContext, TransactionLinkDb, transactionLinkDbSchema } from '../../valibot.schema' import { AbstractSyncRole, IndexType } from './AbstractSync.role' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { constructor(context: Context) { @@ -33,6 +37,10 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { constructor(context: Context) { @@ -29,6 +30,10 @@ export class UsersSyncRole extends AbstractSyncRole { return this.peek().createdAt } + getCommunityUuids(): Uuidv4[] { + return [this.peek().communityUuid] + } + getLastIndex(): IndexType { const lastItem = this.peekLast() return { date: lastItem.createdAt, id: lastItem.id } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts index 201b7427f..93deba85f 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts @@ -1,4 +1,4 @@ -import { Profiler } from 'gradido-blockchain-js' +import { Filter, Profiler, Timestamp, InteractionCreateTransactionByEvent, LedgerAnchor, Abstract } from 'gradido-blockchain-js' import { callTime } from '../../blockchain' import { Context } from '../../Context' import { nanosBalanceForUser } from './AbstractSync.role' @@ -10,6 +10,46 @@ import { RedeemTransactionLinksSyncRole } from './RedeemTransactionLinksSync.rol import { RemoteTransactionsSyncRole } from './RemoteTransactionsSync.role' import { TransactionLinkFundingsSyncRole } from './TransactionLinkFundingsSync.role' import { UsersSyncRole } from './UsersSync.role' +import { CommunityContext } from '../../valibot.schema' +import { Logger } from 'log4js' + +function processTransactionTrigger(context: CommunityContext, endDate: Date, logger: Logger) { + while(true) { + const lastTx = context.blockchain.findOne(Filter.LAST_TRANSACTION) + let confirmedAt: Timestamp | undefined = undefined + if (!lastTx) { + // no transaction, no triggers + return + } else { + const confirmedTx = lastTx.getConfirmedTransaction() + if (!confirmedTx) { + throw new Error("missing confirmed tx in transaction entry") + } + confirmedAt = confirmedTx.getConfirmedAt() + } + const triggerEvent = context.blockchain.findNextTransactionTriggerEventInRange(confirmedAt, new Timestamp(endDate)) + if (!triggerEvent) { + // no trigger, we can exit here + return + } + context.blockchain.removeTransactionTriggerEvent(triggerEvent) + try { + // InMemoryBlockchain extend Abstract, but between C++ -> Swig -> TypeScript it seems the info is gone, so I need to cheat a bit here + const createTransactionByEvent = new InteractionCreateTransactionByEvent(context.blockchain as unknown as Abstract) + if (!context.blockchain.createAndAddConfirmedTransaction( + createTransactionByEvent.run(triggerEvent), + new LedgerAnchor(triggerEvent.getLinkedTransactionId(), LedgerAnchor.Type_NODE_TRIGGER_TRANSACTION_ID), + triggerEvent.getTargetDate() + )) { + throw new Error('Adding trigger created Transaction Failed') + } + } catch(e) { + context.blockchain.addTransactionTriggerEvent(triggerEvent) + logger.error(`Error processing transaction trigger event for transaction: ${triggerEvent.getLinkedTransactionId()}`) + throw e + } + } +} export async function syncDbWithBlockchainContext(context: Context, batchSize: number) { const timeUsedDB = new Profiler() @@ -53,6 +93,11 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n available.sort((a, b) => a.getDate().getTime() - b.getDate().getTime()) // context.logger.debug(`sorted ${available.length} containers in ${sortTime.string()}`) } + const communityUuids = available[0].getCommunityUuids() + for (let i = 0; i < communityUuids.length; ++i) { + processTransactionTrigger(context.getCommunityContextByUuid(communityUuids[i]), available[0].getDate(), context.logger) + } + available[0].toBlockchain() transactionsCount++ if (isDebug) { diff --git a/package.json b/package.json index 92696ba63..0ae1230fa 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "auto-changelog": "^2.4.0", "cross-env": "^7.0.3", "jose": "^4.14.4", - "turbo": "^2.5.0", + "turbo": "^2.8.12", "uuid": "^8.3.2" }, "devDependencies": { From 757139c3ca45e4955182c86920bed3713d975d9b Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 10 Mar 2026 10:48:50 +0100 Subject: [PATCH 49/50] check version of gradido node, fix lint --- dlt-connector/src/bootstrap/init.ts | 20 +++++-- .../src/bootstrap/initGradidoNode.ts | 17 +++++- .../client/GradidoNode/GradidoNodeProcess.ts | 7 ++- dlt-connector/src/config/schema.ts | 4 +- dlt-connector/src/index.ts | 3 + .../data/Balance.ts | 16 ++++-- .../CreationsSync.role.ts | 4 +- .../DeletedTransactionLinksSync.role.ts | 2 +- .../LocalTransactionsSync.role.ts | 2 +- .../RedeemTransactionLinksSync.role.ts | 2 +- .../RemoteTransactionsSync.role.ts | 4 +- .../TransactionLinkFundingsSync.role.ts | 5 +- .../syncDbWithBlockchain/UsersSync.role.ts | 2 +- .../syncDbWithBlockchain.context.ts | 57 +++++++++++++------ dlt-connector/src/utils/network.ts | 2 +- 15 files changed, 100 insertions(+), 47 deletions(-) diff --git a/dlt-connector/src/bootstrap/init.ts b/dlt-connector/src/bootstrap/init.ts index f428b4218..cc8d1ac69 100644 --- a/dlt-connector/src/bootstrap/init.ts +++ b/dlt-connector/src/bootstrap/init.ts @@ -35,11 +35,18 @@ export async function checkHieroAccount(logger: Logger, clients: AppContextClien export async function checkHomeCommunity( appContext: AppContext, logger: Logger, -): Promise { +): Promise { const { backend, hiero } = appContext.clients // wait for backend server - await isPortOpenRetry(backend.url) + try { + logger.info(`Waiting for backend server to become available at ${backend.url}`) + await isPortOpenRetry(backend.url) + } catch (e) { + logger.error(`Backend server at ${backend.url} is not reachable`) + return + } + // ask backend for home community let homeCommunity = await backend.getHomeCommunityDraft() // on missing topicId, create one @@ -58,7 +65,7 @@ export async function checkHomeCommunity( await hiero.updateTopic(homeCommunity.hieroTopicId) topicInfo = await hiero.getTopicInfo(homeCommunity.hieroTopicId) logger.info( - `updated topic info, new expiration time: ${topicInfo.expirationTime.toLocaleDateString()}`, + `Topic expiration extended. New expiration time: ${topicInfo.expirationTime.toLocaleDateString()}`, ) } } @@ -66,9 +73,10 @@ export async function checkHomeCommunity( throw new Error('still no topic id, after creating topic and update community in backend.') } appContext.cache.setHomeCommunityTopicId(homeCommunity.hieroTopicId) - logger.info(`home community topic: ${homeCommunity.hieroTopicId}`) - logger.info(`gradido node server: ${appContext.clients.gradidoNode.url}`) - logger.info(`gradido backend server: ${appContext.clients.backend.url}`) + logger.info(`Home community topic id: ${homeCommunity.hieroTopicId}`) + logger.info(`Gradido node server: ${appContext.clients.gradidoNode.url}`) + logger.info(`Gradido backend server: ${appContext.clients.backend.url}`) + await ResolveKeyPair( new KeyPairIdentifierLogic({ communityTopicId: homeCommunity.hieroTopicId, diff --git a/dlt-connector/src/bootstrap/initGradidoNode.ts b/dlt-connector/src/bootstrap/initGradidoNode.ts index e0d1a047d..395bec16a 100644 --- a/dlt-connector/src/bootstrap/initGradidoNode.ts +++ b/dlt-connector/src/bootstrap/initGradidoNode.ts @@ -53,11 +53,24 @@ async function exportHederaAddressbooks( async function ensureGradidoNodeRuntimeAvailable(runtimeFileName: string): Promise { const runtimeFolder = path.dirname(runtimeFileName) + const wantedVersion = `v${CONFIG.DLT_GRADIDO_NODE_SERVER_VERSION}` checkPathExist(runtimeFolder, true) - if (!checkFileExist(runtimeFileName)) { + let versionMatch = false + const isFileExist = checkFileExist(runtimeFileName) + if (isFileExist) { + const foundVersion = await GradidoNodeProcess.checkRuntimeVersion() + if (wantedVersion !== foundVersion) { + logger.info( + `GradidoNode version detected: ${foundVersion}, required: ${wantedVersion}` + ) + } else { + versionMatch = true + } + } + if (!isFileExist || !versionMatch) { const runtimeArchiveFilename = createGradidoNodeRuntimeArchiveFilename() const downloadUrl = new URL( - `https://github.com/gradido/gradido_node/releases/download/v${CONFIG.DLT_GRADIDO_NODE_SERVER_VERSION}/${runtimeArchiveFilename}`, + `https://github.com/gradido/gradido_node/releases/download/${wantedVersion}/${runtimeArchiveFilename}`, ) logger.debug(`download GradidoNode Runtime from ${downloadUrl}`) const archive = await fetch(downloadUrl) diff --git a/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts b/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts index 09c9b13ab..35c56f4b1 100644 --- a/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts +++ b/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts @@ -1,13 +1,12 @@ import path from 'node:path' import { Mutex } from 'async-mutex' -import { Subprocess, spawn } from 'bun' +import { $, Subprocess, spawn } from 'bun' import { getLogger, Logger } from 'log4js' import { CONFIG } from '../../config' import { GRADIDO_NODE_KILL_TIMEOUT_MILLISECONDS, GRADIDO_NODE_MIN_RUNTIME_BEFORE_EXIT_MILLISECONDS, GRADIDO_NODE_MIN_RUNTIME_BEFORE_RESTART_MILLISECONDS, - GRADIDO_NODE_RUNTIME_PATH, LOG4JS_BASE_CATEGORY, } from '../../config/const' import { delay } from '../../utils/time' @@ -51,6 +50,10 @@ export class GradidoNodeProcess { return path.join(CONFIG.DLT_GRADIDO_NODE_SERVER_HOME_FOLDER, 'bin', binaryName) } + public static async checkRuntimeVersion(): Promise { + return (await $`${GradidoNodeProcess.getRuntimePathFileName()} --version`.text()).trim() + } + public start() { if (this.proc) { this.logger.warn('GradidoNodeProcess already running.') diff --git a/dlt-connector/src/config/schema.ts b/dlt-connector/src/config/schema.ts index fcb1452de..c89db3cc3 100644 --- a/dlt-connector/src/config/schema.ts +++ b/dlt-connector/src/config/schema.ts @@ -82,9 +82,9 @@ export const configSchema = v.object({ DLT_GRADIDO_NODE_SERVER_VERSION: v.optional( v.pipe( v.string('The version of the DLT node server, for example: 0.9.0'), - v.regex(/^\d+\.\d+\.\d+$/), + v.regex(/^\d+\.\d+\.\d+(.\d+)?$/), ), - '0.9.4', + '0.9.6.10', ), DLT_GRADIDO_NODE_SERVER_HOME_FOLDER: v.optional( v.string('The home folder for the gradido dlt node server'), diff --git a/dlt-connector/src/index.ts b/dlt-connector/src/index.ts index 1d4513e83..3f0081ffb 100644 --- a/dlt-connector/src/index.ts +++ b/dlt-connector/src/index.ts @@ -21,6 +21,9 @@ async function main() { // get home community, create topic if not exist, or check topic expiration and update it if needed const homeCommunity = await checkHomeCommunity(appContext, logger) + if (!homeCommunity) { + process.exit(1) + } // ask gradido node if community blockchain was created // if not exist, create community root transaction diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/Balance.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/Balance.ts index 7f96b1d16..d140f1c93 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/Balance.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/Balance.ts @@ -103,11 +103,15 @@ export class Balance { } toString(): string { - return JSON.stringify({ - balance: this.balance.toString(), - date: this.date, - publicKey: this.publicKey.convertToHex(), - communityId: this.communityId - }, null, 2) + return JSON.stringify( + { + balance: this.balance.toString(), + date: this.date, + publicKey: this.publicKey.convertToHex(), + communityId: this.communityId, + }, + null, + 2, + ) } } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts index 9ffa11231..683953a06 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts @@ -14,6 +14,7 @@ import { TransferAmount, } from 'gradido-blockchain-js' import * as v from 'valibot' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' import { addToBlockchain } from '../../blockchain' import { Context } from '../../Context' import { ContributionStatus } from '../../data/ContributionStatus' @@ -26,7 +27,6 @@ import { creationTransactionDbSchema, } from '../../valibot.schema' import { AbstractSyncRole, IndexType } from './AbstractSync.role' -import { Uuidv4 } from '../../../../schemas/typeGuard.schema' export class CreationsSyncRole extends AbstractSyncRole { constructor(context: Context) { @@ -38,7 +38,7 @@ export class CreationsSyncRole extends AbstractSyncRole { return this.peek().confirmedAt } getCommunityUuids(): Uuidv4[] { - return [ this.peek().user.communityUuid ] + return [this.peek().user.communityUuid] } getLastIndex(): IndexType { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts index 3ce1bcef4..05aeaa5a8 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts @@ -14,6 +14,7 @@ import { } from 'gradido-blockchain-js' import * as v from 'valibot' import { deriveFromCode } from '../../../../data/deriveKeyPair' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' import { addToBlockchain } from '../../blockchain' import { Context } from '../../Context' import { Balance } from '../../data/Balance' @@ -26,7 +27,6 @@ import { deletedTransactionLinKDbSchema, } from '../../valibot.schema' import { AbstractSyncRole, IndexType } from './AbstractSync.role' -import { Uuidv4 } from '../../../../schemas/typeGuard.schema' export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { constructor(context: Context) { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts index 4092b41d5..d85921a4b 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts @@ -11,6 +11,7 @@ import { TransferAmount, } from 'gradido-blockchain-js' import * as v from 'valibot' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' import { addToBlockchain } from '../../blockchain' import { Context } from '../../Context' import { TransactionTypeId } from '../../data/TransactionTypeId' @@ -24,7 +25,6 @@ import { import { toMysqlDateTime } from '../../utils' import { CommunityContext, TransactionDb, transactionDbSchema } from '../../valibot.schema' import { AbstractSyncRole, IndexType } from './AbstractSync.role' -import { Uuidv4 } from '../../../../schemas/typeGuard.schema' export class LocalTransactionsSyncRole extends AbstractSyncRole { constructor(context: Context) { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts index ffdb84d04..012a07b08 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts @@ -17,6 +17,7 @@ import { } from 'gradido-blockchain-js' import * as v from 'valibot' import { deriveFromCode } from '../../../../data/deriveKeyPair' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' import { addToBlockchain } from '../../blockchain' import { Context } from '../../Context' import { transactionLinksTable, usersTable } from '../../drizzle.schema' @@ -28,7 +29,6 @@ import { redeemedTransactionLinkDbSchema, } from '../../valibot.schema' import { AbstractSyncRole, IndexType } from './AbstractSync.role' -import { Uuidv4 } from '../../../../schemas/typeGuard.schema' export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { constructor(context: Context) { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts index da9ef8a70..b951641ab 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts @@ -14,6 +14,7 @@ import { TransferAmount, } from 'gradido-blockchain-js' import * as v from 'valibot' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' import { addToBlockchain } from '../../blockchain' import { Context } from '../../Context' import { TransactionTypeId } from '../../data/TransactionTypeId' @@ -27,7 +28,6 @@ import { import { toMysqlDateTime } from '../../utils' import { CommunityContext, TransactionDb, transactionDbSchema, UserDb } from '../../valibot.schema' import { AbstractSyncRole, IndexType } from './AbstractSync.role' -import { Uuidv4 } from '../../../../schemas/typeGuard.schema' export class RemoteTransactionsSyncRole extends AbstractSyncRole { constructor(context: Context) { @@ -41,7 +41,7 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole getCommunityUuids(): Uuidv4[] { const currentItem = this.peek() - return [ currentItem.user.communityUuid, currentItem.linkedUser.communityUuid ] + return [currentItem.user.communityUuid, currentItem.linkedUser.communityUuid] } getLastIndex(): IndexType { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts index 7d93d6921..5364a2bdf 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts @@ -14,11 +14,12 @@ import { LedgerAnchor, MemoryBlockPtr, SearchDirection_DESC, - transactionTypeToString, TransferAmount, + transactionTypeToString, } from 'gradido-blockchain-js' import * as v from 'valibot' import { deriveFromCode } from '../../../../data/deriveKeyPair' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' import { addToBlockchain } from '../../blockchain' import { Context } from '../../Context' import { transactionLinksTable, usersTable } from '../../drizzle.schema' @@ -26,7 +27,6 @@ import { BlockchainError, DatabaseError, NegativeBalanceError } from '../../erro import { reverseLegacyDecay, toMysqlDateTime } from '../../utils' import { CommunityContext, TransactionLinkDb, transactionLinkDbSchema } from '../../valibot.schema' import { AbstractSyncRole, IndexType } from './AbstractSync.role' -import { Uuidv4 } from '../../../../schemas/typeGuard.schema' export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { constructor(context: Context) { @@ -119,7 +119,6 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { constructor(context: Context) { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts index 93deba85f..3824e9b8d 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts @@ -1,6 +1,15 @@ -import { Filter, Profiler, Timestamp, InteractionCreateTransactionByEvent, LedgerAnchor, Abstract } from 'gradido-blockchain-js' +import { + Abstract, + Filter, + InteractionCreateTransactionByEvent, + LedgerAnchor, + Profiler, + Timestamp, +} from 'gradido-blockchain-js' +import { Logger } from 'log4js' import { callTime } from '../../blockchain' import { Context } from '../../Context' +import { CommunityContext } from '../../valibot.schema' import { nanosBalanceForUser } from './AbstractSync.role' import { ContributionLinkTransactionSyncRole } from './ContributionLinkTransactionSync.role' import { CreationsSyncRole } from './CreationsSync.role' @@ -10,24 +19,25 @@ import { RedeemTransactionLinksSyncRole } from './RedeemTransactionLinksSync.rol import { RemoteTransactionsSyncRole } from './RemoteTransactionsSync.role' import { TransactionLinkFundingsSyncRole } from './TransactionLinkFundingsSync.role' import { UsersSyncRole } from './UsersSync.role' -import { CommunityContext } from '../../valibot.schema' -import { Logger } from 'log4js' function processTransactionTrigger(context: CommunityContext, endDate: Date, logger: Logger) { - while(true) { + while (true) { const lastTx = context.blockchain.findOne(Filter.LAST_TRANSACTION) - let confirmedAt: Timestamp | undefined = undefined + let confirmedAt: Timestamp | undefined if (!lastTx) { // no transaction, no triggers return } else { const confirmedTx = lastTx.getConfirmedTransaction() if (!confirmedTx) { - throw new Error("missing confirmed tx in transaction entry") + throw new Error('missing confirmed tx in transaction entry') } confirmedAt = confirmedTx.getConfirmedAt() } - const triggerEvent = context.blockchain.findNextTransactionTriggerEventInRange(confirmedAt, new Timestamp(endDate)) + const triggerEvent = context.blockchain.findNextTransactionTriggerEventInRange( + confirmedAt, + new Timestamp(endDate), + ) if (!triggerEvent) { // no trigger, we can exit here return @@ -35,17 +45,26 @@ function processTransactionTrigger(context: CommunityContext, endDate: Date, log context.blockchain.removeTransactionTriggerEvent(triggerEvent) try { // InMemoryBlockchain extend Abstract, but between C++ -> Swig -> TypeScript it seems the info is gone, so I need to cheat a bit here - const createTransactionByEvent = new InteractionCreateTransactionByEvent(context.blockchain as unknown as Abstract) - if (!context.blockchain.createAndAddConfirmedTransaction( - createTransactionByEvent.run(triggerEvent), - new LedgerAnchor(triggerEvent.getLinkedTransactionId(), LedgerAnchor.Type_NODE_TRIGGER_TRANSACTION_ID), - triggerEvent.getTargetDate() - )) { + const createTransactionByEvent = new InteractionCreateTransactionByEvent( + context.blockchain as unknown as Abstract, + ) + if ( + !context.blockchain.createAndAddConfirmedTransaction( + createTransactionByEvent.run(triggerEvent), + new LedgerAnchor( + triggerEvent.getLinkedTransactionId(), + LedgerAnchor.Type_NODE_TRIGGER_TRANSACTION_ID, + ), + triggerEvent.getTargetDate(), + ) + ) { throw new Error('Adding trigger created Transaction Failed') } - } catch(e) { + } catch (e) { context.blockchain.addTransactionTriggerEvent(triggerEvent) - logger.error(`Error processing transaction trigger event for transaction: ${triggerEvent.getLinkedTransactionId()}`) + logger.error( + `Error processing transaction trigger event for transaction: ${triggerEvent.getLinkedTransactionId()}`, + ) throw e } } @@ -95,9 +114,13 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n } const communityUuids = available[0].getCommunityUuids() for (let i = 0; i < communityUuids.length; ++i) { - processTransactionTrigger(context.getCommunityContextByUuid(communityUuids[i]), available[0].getDate(), context.logger) + processTransactionTrigger( + context.getCommunityContextByUuid(communityUuids[i]), + available[0].getDate(), + context.logger, + ) } - + available[0].toBlockchain() transactionsCount++ if (isDebug) { diff --git a/dlt-connector/src/utils/network.ts b/dlt-connector/src/utils/network.ts index 5f348c640..954cbfd76 100644 --- a/dlt-connector/src/utils/network.ts +++ b/dlt-connector/src/utils/network.ts @@ -29,7 +29,7 @@ export async function isPortOpen( socket.destroy() const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.network.isPortOpen`) logger.addContext('url', url) - logger.error(`${err.message}: ${err.code}`) + logger.debug(`${err.message}: ${err.code}`) resolve(false) }) }) From 3559f6525d0e41916acc5e5a59a86a0ecece1bed Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Tue, 10 Mar 2026 11:38:03 +0100 Subject: [PATCH 50/50] update gradido blockchain js version, fix migration name because other pr already used the 97 --- ...chain2.ts => 0098-fix_production_data_for_blockchain2.ts} | 0 dlt-connector/bun.lock | 5 ++--- dlt-connector/package.json | 2 +- dlt-connector/src/bootstrap/init.ts | 4 ++-- dlt-connector/src/bootstrap/initGradidoNode.ts | 4 +--- 5 files changed, 6 insertions(+), 9 deletions(-) rename database/migration/migrations/{0097-fix_production_data_for_blockchain2.ts => 0098-fix_production_data_for_blockchain2.ts} (100%) diff --git a/database/migration/migrations/0097-fix_production_data_for_blockchain2.ts b/database/migration/migrations/0098-fix_production_data_for_blockchain2.ts similarity index 100% rename from database/migration/migrations/0097-fix_production_data_for_blockchain2.ts rename to database/migration/migrations/0098-fix_production_data_for_blockchain2.ts diff --git a/dlt-connector/bun.lock b/dlt-connector/bun.lock index bf5b13837..793c3d00c 100644 --- a/dlt-connector/bun.lock +++ b/dlt-connector/bun.lock @@ -1,13 +1,12 @@ { "lockfileVersion": 1, - "configVersion": 0, "workspaces": { "": { "name": "dlt-connector", "dependencies": { "bun-zigar": "^0.15.2", "cross-env": "^7.0.3", - "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#5d41ff3839397588e74e77f555c8ff968c786d09", + "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#785fd766289726d41ae01f1e80a274aed871a7fb", }, "devDependencies": { "@biomejs/biome": "2.0.0", @@ -585,7 +584,7 @@ "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], - "gradido-blockchain-js": ["gradido-blockchain-js@github:gradido/gradido-blockchain-js#5d41ff3", { "dependencies": { "bindings": "^1.5.0", "nan": "^2.20.0", "node-addon-api": "^7.1.1", "node-gyp-build": "^4.8.1", "prebuildify": "git+https://github.com/einhornimmond/prebuildify#65d94455fab86b902c0d59bb9c06ac70470e56b2" } }, "gradido-gradido-blockchain-js-5d41ff3"], + "gradido-blockchain-js": ["gradido-blockchain-js@github:gradido/gradido-blockchain-js#785fd76", { "dependencies": { "bindings": "^1.5.0", "nan": "^2.20.0", "node-addon-api": "^7.1.1", "node-gyp-build": "^4.8.1", "prebuildify": "git+https://github.com/einhornimmond/prebuildify#65d94455fab86b902c0d59bb9c06ac70470e56b2" } }, "gradido-gradido-blockchain-js-785fd76"], "graphql": ["graphql@16.11.0", "", {}, "sha512-mS1lbMsxgQj6hge1XZ6p7GPhbrtFwUFYi3wRzXAC/FmYnyXMTvvI3td3rjmQ2u8ewXueaSvRPWaEcgVVOT9Jnw=="], diff --git a/dlt-connector/package.json b/dlt-connector/package.json index 9e93fa858..e8e28936a 100644 --- a/dlt-connector/package.json +++ b/dlt-connector/package.json @@ -20,7 +20,7 @@ "dependencies": { "bun-zigar": "^0.15.2", "cross-env": "^7.0.3", - "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#5d41ff3839397588e74e77f555c8ff968c786d09" + "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#785fd766289726d41ae01f1e80a274aed871a7fb" }, "devDependencies": { "@biomejs/biome": "2.0.0", diff --git a/dlt-connector/src/bootstrap/init.ts b/dlt-connector/src/bootstrap/init.ts index cc8d1ac69..3911b4af7 100644 --- a/dlt-connector/src/bootstrap/init.ts +++ b/dlt-connector/src/bootstrap/init.ts @@ -43,7 +43,7 @@ export async function checkHomeCommunity( logger.info(`Waiting for backend server to become available at ${backend.url}`) await isPortOpenRetry(backend.url) } catch (e) { - logger.error(`Backend server at ${backend.url} is not reachable`) + logger.error(`Backend server at ${backend.url} is not reachable (${e})`) return } @@ -76,7 +76,7 @@ export async function checkHomeCommunity( logger.info(`Home community topic id: ${homeCommunity.hieroTopicId}`) logger.info(`Gradido node server: ${appContext.clients.gradidoNode.url}`) logger.info(`Gradido backend server: ${appContext.clients.backend.url}`) - + await ResolveKeyPair( new KeyPairIdentifierLogic({ communityTopicId: homeCommunity.hieroTopicId, diff --git a/dlt-connector/src/bootstrap/initGradidoNode.ts b/dlt-connector/src/bootstrap/initGradidoNode.ts index 395bec16a..183b3c731 100644 --- a/dlt-connector/src/bootstrap/initGradidoNode.ts +++ b/dlt-connector/src/bootstrap/initGradidoNode.ts @@ -60,9 +60,7 @@ async function ensureGradidoNodeRuntimeAvailable(runtimeFileName: string): Promi if (isFileExist) { const foundVersion = await GradidoNodeProcess.checkRuntimeVersion() if (wantedVersion !== foundVersion) { - logger.info( - `GradidoNode version detected: ${foundVersion}, required: ${wantedVersion}` - ) + logger.info(`GradidoNode version detected: ${foundVersion}, required: ${wantedVersion}`) } else { versionMatch = true }