diff --git a/.github/workflows/test_submodules.yml b/.github/workflows/test_submodules.yml index cbdfd16a2..8d54e55de 100644 --- a/.github/workflows/test_submodules.yml +++ b/.github/workflows/test_submodules.yml @@ -45,5 +45,7 @@ jobs: bun install --global turbo@^2 - name: typecheck, locales && unit test - run: turbo core#test core#typecheck core#locales database#test database#typecheck shared#test shared#typecheck config-schema#test config-schema#typecheck + run: turbo core#typecheck core#locales database#test database#typecheck shared#test shared#typecheck config-schema#test config-schema#typecheck + - name: core test extra + run: turbo core#test diff --git a/backend/src/apis/dltConnector/model/AccountIdentifier.ts b/backend/src/apis/dltConnector/model/AccountIdentifier.ts index 3882e8584..28fc844cd 100644 --- a/backend/src/apis/dltConnector/model/AccountIdentifier.ts +++ b/backend/src/apis/dltConnector/model/AccountIdentifier.ts @@ -1,15 +1,21 @@ import { CommunityAccountIdentifier } from './CommunityAccountIdentifier' export class AccountIdentifier { communityTopicId: string + communityId: string account?: CommunityAccountIdentifier seed?: string // used for deferred transfers - constructor(communityTopicId: string, input: CommunityAccountIdentifier | string) { + constructor( + communityTopicId: string, + communityUuid: string, + input: CommunityAccountIdentifier | string, + ) { if (input instanceof CommunityAccountIdentifier) { this.account = input } else { this.seed = input } this.communityTopicId = communityTopicId + this.communityId = communityUuid } } diff --git a/backend/src/apis/dltConnector/model/CommunityAccountIdentifier.ts b/backend/src/apis/dltConnector/model/CommunityAccountIdentifier.ts index 1e8b6a64f..e40cb1e54 100644 --- a/backend/src/apis/dltConnector/model/CommunityAccountIdentifier.ts +++ b/backend/src/apis/dltConnector/model/CommunityAccountIdentifier.ts @@ -1,9 +1,9 @@ export class CommunityAccountIdentifier { // for community user, uuid and communityUuid used userUuid: string - accountNr?: number + accountNr: number - constructor(userUuid: string, accountNr?: number) { + constructor(userUuid: string, accountNr: number = 1) { this.userUuid = userUuid this.accountNr = accountNr } diff --git a/backend/src/apis/dltConnector/model/TransactionDraft.ts b/backend/src/apis/dltConnector/model/TransactionDraft.ts index 1727c7fd8..162b4e2c5 100755 --- a/backend/src/apis/dltConnector/model/TransactionDraft.ts +++ b/backend/src/apis/dltConnector/model/TransactionDraft.ts @@ -36,6 +36,7 @@ export class TransactionDraft { const draft = new TransactionDraft() draft.user = new AccountIdentifier( community.hieroTopicId, + community.communityUuid!, new CommunityAccountIdentifier(user.gradidoID), ) draft.type = TransactionType.REGISTER_ADDRESS @@ -58,10 +59,12 @@ export class TransactionDraft { const draft = new TransactionDraft() draft.user = new AccountIdentifier( community.hieroTopicId, + community.communityUuid!, new CommunityAccountIdentifier(contribution.user.gradidoID), ) draft.linkedUser = new AccountIdentifier( community.hieroTopicId, + community.communityUuid!, new CommunityAccountIdentifier(signingUser.gradidoID), ) draft.type = TransactionType.GRADIDO_CREATION @@ -96,10 +99,12 @@ export class TransactionDraft { const draft = new TransactionDraft() draft.user = new AccountIdentifier( senderUserTopic, + sendingUser.community.communityUuid!, new CommunityAccountIdentifier(sendingUser.gradidoID), ) draft.linkedUser = new AccountIdentifier( receiverUserTopic, + receivingUser.community.communityUuid!, new CommunityAccountIdentifier(receivingUser.gradidoID), ) draft.type = TransactionType.GRADIDO_TRANSFER @@ -125,9 +130,14 @@ export class TransactionDraft { const draft = new TransactionDraft() draft.user = new AccountIdentifier( senderUserTopic, + sendingUser.community.communityUuid!, new CommunityAccountIdentifier(sendingUser.gradidoID), ) - draft.linkedUser = new AccountIdentifier(senderUserTopic, transactionLink.code) + draft.linkedUser = new AccountIdentifier( + senderUserTopic, + sendingUser.community.communityUuid!, + transactionLink.code, + ) draft.type = TransactionType.GRADIDO_DEFERRED_TRANSFER draft.createdAt = createdAtOnlySeconds.toISOString() draft.amount = transactionLink.amount.toString() @@ -159,9 +169,14 @@ export class TransactionDraft { const createdAtOnlySeconds = createdAt createdAtOnlySeconds.setMilliseconds(0) const draft = new TransactionDraft() - draft.user = new AccountIdentifier(senderUserTopic, transactionLink.code) + draft.user = new AccountIdentifier( + senderUserTopic, + transactionLink.user.community.communityUuid!, + transactionLink.code, + ) draft.linkedUser = new AccountIdentifier( recipientUserTopic, + recipientUser.community.communityUuid!, new CommunityAccountIdentifier(recipientUser.gradidoID), ) draft.type = TransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER diff --git a/backend/src/federation/client/1_0/query/getPublicCommunityInfo.ts b/backend/src/federation/client/1_0/query/getPublicCommunityInfo.ts index 36350fae8..8e00bb506 100644 --- a/backend/src/federation/client/1_0/query/getPublicCommunityInfo.ts +++ b/backend/src/federation/client/1_0/query/getPublicCommunityInfo.ts @@ -8,6 +8,7 @@ export const getPublicCommunityInfo = gql` creationDate publicKey publicJwtKey + hieroTopicId } } ` diff --git a/backend/src/graphql/resolver/CommunityResolver.ts b/backend/src/graphql/resolver/CommunityResolver.ts index 0209c284d..4932c0f93 100644 --- a/backend/src/graphql/resolver/CommunityResolver.ts +++ b/backend/src/graphql/resolver/CommunityResolver.ts @@ -2,7 +2,12 @@ import { Paginated } from '@arg/Paginated' import { EditCommunityInput } from '@input/EditCommunityInput' import { AdminCommunityView } from '@model/AdminCommunityView' import { Community } from '@model/Community' -import { Community as DbCommunity, getHomeCommunity, getReachableCommunities } from 'database' +import { + Community as DbCommunity, + getAuthorizedCommunities, + getHomeCommunity, + getReachableCommunities, +} from 'database' import { updateAllDefinedAndChanged } from 'shared' import { Arg, Args, Authorized, Mutation, Query, Resolver } from 'type-graphql' import { RIGHTS } from '@/auth/RIGHTS' @@ -34,6 +39,17 @@ export class CommunityResolver { return dbCommunities.map((dbCom: DbCommunity) => new Community(dbCom)) } + @Authorized([RIGHTS.COMMUNITIES]) + @Query(() => [Community]) + async authorizedCommunities(): Promise { + const dbCommunities: DbCommunity[] = await getAuthorizedCommunities({ + // order by + foreign: 'ASC', // home community first + name: 'ASC', + }) + return dbCommunities.map((dbCom: DbCommunity) => new Community(dbCom)) + } + @Authorized([RIGHTS.COMMUNITIES]) @Query(() => Community) async communityByIdentifier( diff --git a/backend/src/seeds/index.ts b/backend/src/seeds/index.ts index 59cd5e5d5..f400d080b 100644 --- a/backend/src/seeds/index.ts +++ b/backend/src/seeds/index.ts @@ -86,6 +86,7 @@ async function clearDatabase(db: AppDatabase) { await trx.query(`SET FOREIGN_KEY_CHECKS = 0`) await trx.query(`TRUNCATE TABLE contributions`) await trx.query(`TRUNCATE TABLE contribution_links`) + await trx.query(`TRUNCATE TABLE events`) await trx.query(`TRUNCATE TABLE users`) await trx.query(`TRUNCATE TABLE user_contacts`) await trx.query(`TRUNCATE TABLE user_roles`) diff --git a/bun.lock b/bun.lock index 984f39857..24dd8bf42 100644 --- a/bun.lock +++ b/bun.lock @@ -7,7 +7,7 @@ "auto-changelog": "^2.4.0", "cross-env": "^7.0.3", "jose": "^4.14.4", - "turbo": "^2.5.0", + "turbo": "^2.8.12", "uuid": "^8.3.2", }, "devDependencies": { @@ -18,7 +18,7 @@ }, "admin": { "name": "admin", - "version": "2.7.3", + "version": "2.7.4", "dependencies": { "@iconify/json": "^2.2.228", "@popperjs/core": "^2.11.8", @@ -88,7 +88,7 @@ }, "backend": { "name": "backend", - "version": "2.7.3", + "version": "2.7.4", "dependencies": { "cross-env": "^7.0.3", "email-templates": "^10.0.1", @@ -165,7 +165,7 @@ }, "config-schema": { "name": "config-schema", - "version": "2.7.3", + "version": "2.7.4", "dependencies": { "esbuild": "^0.25.2", "joi": "17.13.3", @@ -183,7 +183,7 @@ }, "core": { "name": "core", - "version": "2.7.3", + "version": "2.7.4", "dependencies": { "database": "*", "email-templates": "^10.0.1", @@ -220,7 +220,7 @@ }, "database": { "name": "database", - "version": "2.7.3", + "version": "2.7.4", "dependencies": { "@types/uuid": "^8.3.4", "cross-env": "^7.0.3", @@ -256,7 +256,7 @@ }, "dht-node": { "name": "dht-node", - "version": "2.7.3", + "version": "2.7.4", "dependencies": { "cross-env": "^7.0.3", "dht-rpc": "6.18.1", @@ -294,7 +294,7 @@ }, "federation": { "name": "federation", - "version": "2.7.3", + "version": "2.7.4", "dependencies": { "cross-env": "^7.0.3", "email-templates": "^10.0.1", @@ -355,7 +355,7 @@ }, "frontend": { "name": "frontend", - "version": "2.7.3", + "version": "2.7.4", "dependencies": { "@morev/vue-transitions": "^3.0.2", "@types/leaflet": "^1.9.12", @@ -451,7 +451,7 @@ }, "shared": { "name": "shared", - "version": "2.7.3", + "version": "2.7.4", "dependencies": { "decimal.js-light": "^2.5.1", "esbuild": "^0.25.2", @@ -3357,19 +3357,19 @@ "tunnel": ["tunnel@0.0.6", "", {}, "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg=="], - "turbo": ["turbo@2.6.1", "", { "optionalDependencies": { "turbo-darwin-64": "2.6.1", "turbo-darwin-arm64": "2.6.1", "turbo-linux-64": "2.6.1", "turbo-linux-arm64": "2.6.1", "turbo-windows-64": "2.6.1", "turbo-windows-arm64": "2.6.1" }, "bin": { "turbo": "bin/turbo" } }, "sha512-qBwXXuDT3rA53kbNafGbT5r++BrhRgx3sAo0cHoDAeG9g1ItTmUMgltz3Hy7Hazy1ODqNpR+C7QwqL6DYB52yA=="], + "turbo": ["turbo@2.8.12", "", { "optionalDependencies": { "turbo-darwin-64": "2.8.12", "turbo-darwin-arm64": "2.8.12", "turbo-linux-64": "2.8.12", "turbo-linux-arm64": "2.8.12", "turbo-windows-64": "2.8.12", "turbo-windows-arm64": "2.8.12" }, "bin": { "turbo": "bin/turbo" } }, "sha512-auUAMLmi0eJhxDhQrxzvuhfEbICnVt0CTiYQYY8WyRJ5nwCDZxD0JG8bCSxT4nusI2CwJzmZAay5BfF6LmK7Hw=="], - "turbo-darwin-64": ["turbo-darwin-64@2.6.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-Dm0HwhyZF4J0uLqkhUyCVJvKM9Rw7M03v3J9A7drHDQW0qAbIGBrUijQ8g4Q9Cciw/BXRRd8Uzkc3oue+qn+ZQ=="], + "turbo-darwin-64": ["turbo-darwin-64@2.8.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-EiHJmW2MeQQx+21x8hjMHw/uPhXt9PIxvDrxzOtyVwrXzL0tQmsxtO4qHf2l7uA+K6PUJ4+TjY1MHZDuCvWXrw=="], - "turbo-darwin-arm64": ["turbo-darwin-arm64@2.6.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-U0PIPTPyxdLsrC3jN7jaJUwgzX5sVUBsKLO7+6AL+OASaa1NbT1pPdiZoTkblBAALLP76FM0LlnsVQOnmjYhyw=="], + "turbo-darwin-arm64": ["turbo-darwin-arm64@2.8.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-cbqqGN0vd7ly2TeuaM8k9AK9u1CABO4kBA5KPSqovTiLL3sORccn/mZzJSbvQf0EsYRfU34MgW5FotfwW3kx8Q=="], - "turbo-linux-64": ["turbo-linux-64@2.6.1", "", { "os": "linux", "cpu": "x64" }, "sha512-eM1uLWgzv89bxlK29qwQEr9xYWBhmO/EGiH22UGfq+uXr+QW1OvNKKMogSN65Ry8lElMH4LZh0aX2DEc7eC0Mw=="], + "turbo-linux-64": ["turbo-linux-64@2.8.12", "", { "os": "linux", "cpu": "x64" }, "sha512-jXKw9j4r4q6s0goSXuKI3aKbQK2qiNeP25lGGEnq018TM6SWRW1CCpPMxyG91aCKrub7wDm/K45sGNT4ZFBcFQ=="], - "turbo-linux-arm64": ["turbo-linux-arm64@2.6.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-MFFh7AxAQAycXKuZDrbeutfWM5Ep0CEZ9u7zs4Hn2FvOViTCzIfEhmuJou3/a5+q5VX1zTxQrKGy+4Lf5cdpsA=="], + "turbo-linux-arm64": ["turbo-linux-arm64@2.8.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-BRJCMdyXjyBoL0GYpvj9d2WNfMHwc3tKmJG5ATn2Efvil9LsiOsd/93/NxDqW0jACtHFNVOPnd/CBwXRPiRbwA=="], - "turbo-windows-64": ["turbo-windows-64@2.6.1", "", { "os": "win32", "cpu": "x64" }, "sha512-buq7/VAN7KOjMYi4tSZT5m+jpqyhbRU2EUTTvp6V0Ii8dAkY2tAAjQN1q5q2ByflYWKecbQNTqxmVploE0LVwQ=="], + "turbo-windows-64": ["turbo-windows-64@2.8.12", "", { "os": "win32", "cpu": "x64" }, "sha512-vyFOlpFFzQFkikvSVhVkESEfzIopgs2J7J1rYvtSwSHQ4zmHxkC95Q8Kjkus8gg+8X2mZyP1GS5jirmaypGiPw=="], - "turbo-windows-arm64": ["turbo-windows-arm64@2.6.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-7w+AD5vJp3R+FB0YOj1YJcNcOOvBior7bcHTodqp90S3x3bLgpr7tE6xOea1e8JkP7GK6ciKVUpQvV7psiwU5Q=="], + "turbo-windows-arm64": ["turbo-windows-arm64@2.8.12", "", { "os": "win32", "cpu": "arm64" }, "sha512-9nRnlw5DF0LkJClkIws1evaIF36dmmMEO84J5Uj4oQ8C0QTHwlH7DNe5Kq2Jdmu8GXESCNDNuUYG8Cx6W/vm3g=="], "type-check": ["type-check@0.4.0", "", { "dependencies": { "prelude-ls": "^1.2.1" } }, "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew=="], diff --git a/database/migration/migrations/0098-fix_production_data_for_blockchain2.ts b/database/migration/migrations/0098-fix_production_data_for_blockchain2.ts new file mode 100644 index 000000000..6cd5f7c39 --- /dev/null +++ b/database/migration/migrations/0098-fix_production_data_for_blockchain2.ts @@ -0,0 +1,345 @@ +import Decimal from 'decimal.js-light' +import { DECAY_FACTOR, reverseLegacyDecay } from 'shared' + +function calculateEffectiveSeconds(holdOriginal: Decimal, holdCorrected: Decimal): Decimal { + return holdOriginal.div(holdCorrected).ln().div(DECAY_FACTOR.ln()) +} + +export async function upgrade(queryFn: (query: string, values?: any[]) => Promise>) { + /** + * Migration: Correct historical inconsistencies in transactions, users, and contribution_links. + * + * Background: + * Early Gradido production data contains several inconsistencies that violate + * stricter blockchain validation rules. These inconsistencies include: + * - Contribution transactions confirmed by users who did not exist at the + * time of the transaction. + * - Users confirming their own contributions (self-signed transactions). + * - Users whose `created_at` timestamp is after or equal to their first + * transaction. + * - Transaction memos shorter than the required minimum length (5 characters). + * - Existing contribution_links without an associated 'ADMIN_CONTRIBUTION_LINK_CREATE' event, + * which is used to find someone who confirmed the contribution. + * + * Purpose: + * This migration performs the following corrections to ensure historical + * consistency and full compatibility with blockchain validation rules: + * 1. Fix self-signed contributions by assigning the actual moderator. + * 2. Replace invalid moderators with the earliest ADMIN or MODERATOR where + * the linked user was created after the transaction. + * 3. Update user creation dates to be before their first transaction. + * 4. Ensure all transaction memos meet the minimum length requirement. + * 5. Insert missing 'ADMIN_CONTRIBUTION_LINK_CREATE' events for contribution_links + * that do not yet have such events, using the first Admin as acting_user. + * + * Outcome: + * After this migration: + * - All contribution transactions reference a valid moderator existing at the time of the transaction. + * - User creation dates are logically consistent with their transactions. + * - Transaction memos meet the minimum formatting rules. + * - Every contribution_link has a corresponding 'ADMIN_CONTRIBUTION_LINK_CREATE' event, + * ensuring blockchain consistency for contributions. + */ + + /** + * Fix 0: Update transaction links to match holdAvailableAmount with validUntil, because the old formula lead to incorrect values + */ + + let count = 0 + let lastProcessedId = 0 + const LIMIT = 200 + do { + const rows = await queryFn( + ` + SELECT id, amount, hold_available_amount, validUntil, createdAt, redeemedAt, deletedAt + FROM transaction_links + WHERE id > ? + ORDER BY id ASC + LIMIT ? + `, + [lastProcessedId, LIMIT], + ) + if (!rows.length) { + break + } + const updates: Array<{ id: number; newValidUntil: string }> = [] + for (const row of rows) { + const validUntil = new Date(row.validUntil) + const redeemedAt = row.redeemedAt ? new Date(row.redeemedAt) : null + const deletedAt = row.deletedAt ? new Date(row.deletedAt) : null + const createdAt = new Date(row.createdAt) + const amount = new Decimal(row.amount) + const duration = (validUntil.getTime() - createdAt.getTime()) / 1000 + const blockedAmountCorrected = reverseLegacyDecay(amount, duration) + // fix only if the difference is big enough to have an impact + if (blockedAmountCorrected.sub(amount).abs().lt(new Decimal('0.001'))) { + continue + } + const holdAvailableAmount = new Decimal(row.hold_available_amount) + const secondsDiff = calculateEffectiveSeconds( + new Decimal(holdAvailableAmount.toString()), + new Decimal(blockedAmountCorrected.toString()), + ) + const newValidUntil = new Date(validUntil.getTime() - secondsDiff.mul(1000).toNumber()) + if ( + (redeemedAt && redeemedAt.getTime() < validUntil.getTime()) || + (deletedAt && deletedAt.getTime() < validUntil.getTime()) + ) { + continue + } + updates.push({ + id: row.id, + newValidUntil: newValidUntil.toISOString().replace('T', ' ').replace('Z', ''), + }) + } + if (updates.length > 0) { + const caseStatements = updates.map((u) => `WHEN ${u.id} THEN '${u.newValidUntil}'`).join('\n') + + await queryFn( + ` + UPDATE transaction_links + SET validUntil = CASE id + ${caseStatements} + END + WHERE id IN (?) + `, + [updates.map((u) => u.id)], + ) + } + count = rows.length + lastProcessedId = rows[rows.length - 1].id + } while (count === LIMIT) + ///*/ + /** + * Fix 1: Remove self-signed contributions. + * + * Background: + * A core rule in the system states that *no user may confirm their own + * contribution* — a moderator must always be someone else. + * + * However, early production data contains transactions where the `linked_user_id` + * matches the `user_id`, meaning the contributor confirmed their own contribution. + * + * This query corrects those records by replacing the `linked_user` with the + * moderator stored in `contributions.moderator_id`. + * + * Only transactions where: + * - the type is a contribution (type_id = 1), + * - the linked user equals the contributor (`t.user_id = t.linked_user_id`), + * - the moderator existed before the time of the transaction, + * - and the moderator is not the same person, + * are updated. + */ + await queryFn(` + UPDATE transactions t + JOIN contributions c ON(t.id = c.transaction_id) + JOIN users u ON(c.moderator_id = u.id) + SET t.linked_user_id = u.id, + t.linked_user_community_uuid = u.community_uuid, + t.linked_user_gradido_id = u.gradido_id, + t.linked_user_name = CONCAT(u.first_name, ' ', u.last_name) + WHERE t.type_id = 1 + AND t.user_id = t.linked_user_id + AND u.created_at < t.balance_date + AND t.user_id <> u.id + ;`) + + await queryFn(` + UPDATE contributions c + JOIN users u ON(c.moderator_id = u.id) + SET c.confirmed_by = u.id + WHERE c.contribution_status = 'CONFIRMED' + AND c.user_id = c.confirmed_by + AND u.created_at < c.confirmed_at + AND c.user_id <> u.id + ;`) + + /** + * Fix 2: Replace invalid moderators with the earliest ADMIN. + * + * Background: + * Early production records contain contribution transactions where the assigned + * moderator ("linked_user" or "contribution.moderator_id") was created *after* the contribution itself. This + * is invalid in the blockchain verification process, which requires that the + * moderator account must have existed *before* the time of the transaction. + * + * This migration: + * 1. Identifies the earliest ADMIN or MODERATOR user in the system. + * 2. Reassigns them as moderator for all affected transactions where: + * - the type is a contribution (type_id = 1), + * - the linked user was created after or at the transaction date, + * - the transaction occurred after the ADMIN’s or MODERATOR's creation, + * - and the contributor is not the ADMIN or MODERATOR. + * + * Using the earliest ADMIN or MODERATOR ensures: + * - historical consistency, + * - minimal intrusion, + * - and compatibility with blockchain validation rules. + */ + await queryFn(` + UPDATE transactions t + JOIN ( + SELECT t_sub.id as sub_t_id, u_sub.created_at, u_sub.id, u_sub.community_uuid, u_sub.gradido_id, CONCAT(u_sub.first_name, ' ', u_sub.last_name) AS linked_user_name + FROM transactions t_sub + JOIN users u_sub on(t_sub.user_id <> u_sub.id) + JOIN user_roles r_sub ON u_sub.id = r_sub.user_id + WHERE r_sub.role IN ('ADMIN', 'MODERATOR') + GROUP BY t_sub.id + ORDER BY r_sub.created_at ASC + ) moderator ON (t.id = moderator.sub_t_id) + LEFT JOIN users u on(t.linked_user_id = u.id) + SET t.linked_user_id = moderator.id, + t.linked_user_community_uuid = moderator.community_uuid, + t.linked_user_gradido_id = moderator.gradido_id, + t.linked_user_name = moderator.linked_user_name + WHERE t.type_id = 1 + AND t.balance_date <= u.created_at + AND t.balance_date > moderator.created_at + AND t.user_id <> moderator.id + ;`) + + // similar but with confirmed by user + await queryFn(` + UPDATE contributions c + JOIN ( + SELECT c_sub.id as sub_c_id, u_sub.created_at, u_sub.id + FROM contributions c_sub + JOIN users u_sub ON (c_sub.confirmed_by <> u_sub.id AND c_sub.user_id <> u_sub.id) + JOIN user_roles r_sub ON (u_sub.id = r_sub.user_id) + WHERE r_sub.role IN ('ADMIN', 'MODERATOR') + GROUP BY c_sub.id + ORDER BY r_sub.created_at ASC + ) confirmingUser ON (c.id = confirmingUser.sub_c_id) + LEFT JOIN users u on(c.confirmed_by = u.id) + SET c.confirmed_by = confirmingUser.id + WHERE c.confirmed_at <= u.created_at + AND c.confirmed_at > confirmingUser.created_at + AND c.user_id <> confirmingUser.id + ;`) + + /** + * Fix 3: Update user creation dates to ensure historical consistency. + * + * Background: + * In early production data, some users have a `created_at` timestamp that is + * **after or equal** to their first recorded transaction (`balance_date`). + * This violates logical consistency, because a user cannot exist *after* their + * own transaction. + * + * What this query does: + * - For each user, it finds the earliest transaction date (`first_date`) from + * the `transactions` table. + * - It updates the user's `created_at` timestamp to **1 second before** their + * first transaction. + * + * Notes: + * - Only users where `created_at >= first transaction date` are affected. + * - This is a historical data fix to ensure all transactions reference a user + * that already exists at the time of the transaction, which is required for + * blockchain validation and logical consistency in the system. + */ + await queryFn(` + UPDATE users u + LEFT JOIN ( + SELECT user_id, MIN(balance_date) AS first_date + FROM transactions + GROUP BY user_id + ) t ON t.user_id = u.id + SET u.created_at = DATE_SUB(t.first_date, INTERVAL 1 SECOND) + WHERE u.created_at >= t.first_date; + ;`) + + // linked user also, but we need to use gradido_id as index, because on cross group transactions linked_user_id is empty + await queryFn(` + UPDATE users u + LEFT JOIN ( + SELECT linked_user_gradido_id , MIN(balance_date) AS first_date + FROM transactions + GROUP BY linked_user_gradido_id + ) t ON t.linked_user_gradido_id = u.gradido_id + SET u.created_at = DATE_SUB(t.first_date, INTERVAL 1 SECOND) + WHERE u.created_at >= t.first_date; + ;`) + + /** + * Fix 4: Ensure all transaction memos meet the minimum length requirement. + * + * Background: + * In early Gradido production data, some transactions have a `memo` field + * shorter than the current rule of 5 characters. This can cause issues in + * reporting, display, or blockchain validation processes that expect + * a minimum memo length. + * + * What this query does: + * - For memos with 0 characters, sets the value to 'empty empty'. + * - For memos with 1-4 characters, pads the memo on the left with spaces + * until it reaches 5 characters. + * - Memos that are already 5 characters or longer are left unchanged. + * + * Notes: + * - This ensures all memos are at least 5 characters long. + * - The padding uses spaces. + * - Only memos shorter than 5 characters are affected. + */ + await queryFn(` + UPDATE transactions t + SET t.memo = CASE + WHEN CHAR_LENGTH(t.memo) = 0 THEN 'empty empty' + WHEN CHAR_LENGTH(t.memo) < 5 THEN LPAD(t.memo, 5, ' ') + ELSE t.memo + END + WHERE CHAR_LENGTH(t.memo) < 5 + ;`) + + await queryFn(` + UPDATE contributions t + SET t.memo = CASE + WHEN CHAR_LENGTH(t.memo) = 0 THEN 'empty empty' + WHEN CHAR_LENGTH(t.memo) < 5 THEN LPAD(t.memo, 5, ' ') + ELSE t.memo + END + WHERE CHAR_LENGTH(t.memo) < 5 + ;`) + + await queryFn(` + UPDATE transaction_links t + SET t.memo = CASE + WHEN CHAR_LENGTH(t.memo) = 0 THEN 'empty empty' + WHEN CHAR_LENGTH(t.memo) < 5 THEN LPAD(t.memo, 5, ' ') + ELSE t.memo + END + WHERE CHAR_LENGTH(t.memo) < 5 + ;`) + + /** + * Fix 5: Insert missing 'ADMIN_CONTRIBUTION_LINK_CREATE' events for contribution_links. + * + * Background: + * Each contribution in the blockchain requires a confirmation by a user. + * In the current DB version, there is no information about who confirmed contributions based on contribution_links. + * Recently, functionality was added to create an 'ADMIN_CONTRIBUTION_LINK_CREATE' event + * for newly created contribution_links, but existing contribution_links were not updated. + * + * This query inserts an 'ADMIN_CONTRIBUTION_LINK_CREATE' event for every contribution_link + * that does not already have such an event. + * The acting_user_id is set to the first Admin, and affected_user_id is set to 0. + */ + await queryFn(` + INSERT INTO \`events\`(acting_user_id, affected_user_id, \`type\`, involved_contribution_link_id) + SELECT ( + SELECT u.id + FROM users u + JOIN user_roles r ON r.user_id = u.id + WHERE r.role = 'ADMIN' + ORDER BY r.id ASC + LIMIT 1 + ) AS acting_user_id, 0 as affected_user_id, 'ADMIN_CONTRIBUTION_LINK_CREATE' AS \`type\`, c.id AS involved_contribution_link_id + FROM contribution_links c + LEFT JOIN \`events\` e ON e.involved_contribution_link_id = c.id AND e.type = 'ADMIN_CONTRIBUTION_LINK_CREATE' + WHERE e.id IS NULL + ;`) +} + +export async function downgrade(queryFn: (query: string, values?: any[]) => Promise>) { + // downgrade not possible +} diff --git a/database/src/queries/communities.ts b/database/src/queries/communities.ts index 83e8933d0..6d1064f9b 100644 --- a/database/src/queries/communities.ts +++ b/database/src/queries/communities.ts @@ -84,7 +84,7 @@ export async function getReachableCommunities( federatedCommunities: { verifiedAt: MoreThanOrEqual(new Date(Date.now() - authenticationTimeoutMs)), }, - }, + }, // or { foreign: false }, ], order, @@ -99,3 +99,16 @@ export async function getNotReachableCommunities( order, }) } + +// return the home community and all communities which had at least once make it through the first handshake +export async function getAuthorizedCommunities( + order?: FindOptionsOrder, +): Promise { + return await DbCommunity.find({ + where: [ + { authenticatedAt: Not(IsNull()) }, // or + { foreign: false }, + ], + order, + }) +} diff --git a/dlt-connector/.gitignore b/dlt-connector/.gitignore index 4c6422640..c1c4be080 100644 --- a/dlt-connector/.gitignore +++ b/dlt-connector/.gitignore @@ -3,6 +3,8 @@ /.env.bak /build/ /locales/ +lib +.zigar-cache package-json.lock coverage # emacs diff --git a/dlt-connector/bun.lock b/dlt-connector/bun.lock index 1d12dcaf3..793c3d00c 100644 --- a/dlt-connector/bun.lock +++ b/dlt-connector/bun.lock @@ -4,7 +4,9 @@ "": { "name": "dlt-connector", "dependencies": { - "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#f265dbb1780a912cf8b0418dfe3eaf5cdc5b51cf", + "bun-zigar": "^0.15.2", + "cross-env": "^7.0.3", + "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#785fd766289726d41ae01f1e80a274aed871a7fb", }, "devDependencies": { "@biomejs/biome": "2.0.0", @@ -18,6 +20,7 @@ "@types/uuid": "^8.3.4", "adm-zip": "^0.5.16", "async-mutex": "^0.5.0", + "decimal.js-light": "^2.5.1", "dotenv": "^10.0.0", "drizzle-orm": "^0.44.7", "elysia": "1.3.8", @@ -389,6 +392,8 @@ "bun-types": ["bun-types@1.3.1", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-NMrcy7smratanWJ2mMXdpatalovtxVggkj11bScuWuiOoXTiKIu2eVS1/7qbyI/4yHedtsn175n4Sm4JcdHLXw=="], + "bun-zigar": ["bun-zigar@0.15.2", "", { "dependencies": { "node-zigar-addon": "0.15.2", "zigar-compiler": "^0.15.2" }, "bin": { "zigar": "bin/cli.js", "bun-zigar": "bin/cli.js" } }, "sha512-slEHTEapQEIqB86OeiToPuuFXe39DCIYISTPzbIMBTZL34vRzCIa5wFn5ATudauHFFwl5/y5JYv8tluk2QL9Eg=="], + "call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="], "camelcase": ["camelcase@6.3.0", "", {}, "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA=="], @@ -431,6 +436,8 @@ "cookie": ["cookie@1.0.2", "", {}, "sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA=="], + "cross-env": ["cross-env@7.0.3", "", { "dependencies": { "cross-spawn": "^7.0.1" }, "bin": { "cross-env": "src/bin/cross-env.js", "cross-env-shell": "src/bin/cross-env-shell.js" } }, "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw=="], + "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], "crypto-js": ["crypto-js@4.2.0", "", {}, "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q=="], @@ -443,6 +450,8 @@ "debug": ["debug@4.4.1", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ=="], + "decimal.js-light": ["decimal.js-light@2.5.1", "", {}, "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg=="], + "deep-extend": ["deep-extend@0.6.0", "", {}, "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="], "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="], @@ -575,7 +584,7 @@ "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], - "gradido-blockchain-js": ["gradido-blockchain-js@github:gradido/gradido-blockchain-js#f265dbb", { "dependencies": { "bindings": "^1.5.0", "nan": "^2.20.0", "node-addon-api": "^7.1.1", "node-gyp-build": "^4.8.1", "prebuildify": "git+https://github.com/einhornimmond/prebuildify#65d94455fab86b902c0d59bb9c06ac70470e56b2" } }, "gradido-gradido-blockchain-js-f265dbb"], + "gradido-blockchain-js": ["gradido-blockchain-js@github:gradido/gradido-blockchain-js#785fd76", { "dependencies": { "bindings": "^1.5.0", "nan": "^2.20.0", "node-addon-api": "^7.1.1", "node-gyp-build": "^4.8.1", "prebuildify": "git+https://github.com/einhornimmond/prebuildify#65d94455fab86b902c0d59bb9c06ac70470e56b2" } }, "gradido-gradido-blockchain-js-785fd76"], "graphql": ["graphql@16.11.0", "", {}, "sha512-mS1lbMsxgQj6hge1XZ6p7GPhbrtFwUFYi3wRzXAC/FmYnyXMTvvI3td3rjmQ2u8ewXueaSvRPWaEcgVVOT9Jnw=="], @@ -777,7 +786,7 @@ "node-addon-api": ["node-addon-api@7.1.1", "", {}, "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ=="], - "node-api-headers": ["node-api-headers@1.6.0", "", {}, "sha512-81T99+mWLZnxX0LlZPYuafyFlxVVaWKQ0BDAbSrOqLO+v+gzCzu0GTAVNeVK8lucqjqo9L/1UcK9cpkem8Py4Q=="], + "node-api-headers": ["node-api-headers@1.8.0", "", {}, "sha512-jfnmiKWjRAGbdD1yQS28bknFM1tbHC1oucyuMPjmkEs+kpiu76aRs40WlTmBmyEgzDM76ge1DQ7XJ3R5deiVjQ=="], "node-gyp-build": ["node-gyp-build@4.8.4", "", { "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", "node-gyp-build-test": "build-test.js" } }, "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ=="], @@ -785,6 +794,8 @@ "node-releases": ["node-releases@2.0.26", "", {}, "sha512-S2M9YimhSjBSvYnlr5/+umAnPHE++ODwt5e2Ij6FoX45HA/s4vHdkDx1eax2pAPeAOqu4s9b7ppahsyEFdVqQA=="], + "node-zigar-addon": ["node-zigar-addon@0.15.2", "", { "dependencies": { "node-api-headers": "^1.7.0" } }, "sha512-QjJcPRtUZLkULaFXapAvTzLKKRddgaupr7wQqgDUQo541FMCXAhgWdZJtNcIgCNykJG0bG0Fza5VTKBdSvyavQ=="], + "normalize-path": ["normalize-path@3.0.0", "", {}, "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="], "npm-path": ["npm-path@2.0.4", "", { "dependencies": { "which": "^1.2.10" }, "bin": { "npm-path": "bin/npm-path" } }, "sha512-IFsj0R9C7ZdR5cP+ET342q77uSRdtWOlWpih5eC+lu29tIDbNEgDbzgVJ5UFvYHWhxDZ5TFkJafFioO0pPQjCw=="], @@ -1053,6 +1064,8 @@ "yoctocolors-cjs": ["yoctocolors-cjs@2.1.3", "", {}, "sha512-U/PBtDf35ff0D8X8D0jfdzHYEPFxAI7jJlxZXwCSez5M3190m+QobIfh+sWDWSHMCWWJN2AWamkegn6vr6YBTw=="], + "zigar-compiler": ["zigar-compiler@0.15.2", "", {}, "sha512-zlJ8kUwndwrLl4iRlIWEcidC2rcSsfeWM0jvbSoxUVf+SEKd4bVik3z4YDivuyX3SUiUjpCMNyp65etD6BKRmQ=="], + "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], "@babel/core/debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], @@ -1097,6 +1110,8 @@ "cmake-js/fs-extra": ["fs-extra@11.3.2", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" } }, "sha512-Xr9F6z6up6Ws+NjzMCZc6WXg2YFRlrLP9NQDO3VQrWrfiojdhS56TzueT88ze0uBdCTwEIhQ3ptnmKeWGFAe0A=="], + "cmake-js/node-api-headers": ["node-api-headers@1.6.0", "", {}, "sha512-81T99+mWLZnxX0LlZPYuafyFlxVVaWKQ0BDAbSrOqLO+v+gzCzu0GTAVNeVK8lucqjqo9L/1UcK9cpkem8Py4Q=="], + "connect/debug": ["debug@2.6.9", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="], "elliptic/bn.js": ["bn.js@4.12.2", "", {}, "sha512-n4DSx829VRTRByMRGdjQ9iqsN0Bh4OolPsFnaZBLcbi8iXcB+kJ9s7EnRt4wILZNV3kPLHkRVfOc/HvhC3ovDw=="], diff --git a/dlt-connector/bunfig.toml b/dlt-connector/bunfig.toml new file mode 100644 index 000000000..4299d9c62 --- /dev/null +++ b/dlt-connector/bunfig.toml @@ -0,0 +1 @@ +preload = ["bun-zigar"] diff --git a/dlt-connector/package.json b/dlt-connector/package.json index 4c90e7b5b..e8e28936a 100644 --- a/dlt-connector/package.json +++ b/dlt-connector/package.json @@ -7,18 +7,20 @@ "license": "Apache-2.0", "private": true, "scripts": { - "start": "bun run src/index.ts", + "start": "cross-env TZ=UTC bun run src/index.ts", "build": "bun build src/index.ts --outdir=build --target=bun --external=gradido-blockchain-js --minify", - "dev": "bun run --watch src/index.ts", - "migrate": "bun src/migrations/db-v2.7.0_to_blockchain-v3.5", - "test": "bun test", - "test:debug": "bun test --inspect-brk", + "dev": "cross-env TZ=UTC bun run --watch src/index.ts", + "migrate": "cross-env TZ=UTC bun src/migrations/db-v2.7.0_to_blockchain-v3.7", + "test": "cross-env TZ=UTC bun test", + "test:debug": "cross-env TZ=UTC bun test --inspect-brk", "typecheck": "tsc --noEmit", "lint": "biome check --error-on-warnings .", "lint:fix": "biome check --error-on-warnings . --write" }, "dependencies": { - "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#f265dbb1780a912cf8b0418dfe3eaf5cdc5b51cf" + "bun-zigar": "^0.15.2", + "cross-env": "^7.0.3", + "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#785fd766289726d41ae01f1e80a274aed871a7fb" }, "devDependencies": { "@biomejs/biome": "2.0.0", @@ -32,6 +34,7 @@ "@types/uuid": "^8.3.4", "adm-zip": "^0.5.16", "async-mutex": "^0.5.0", + "decimal.js-light": "^2.5.1", "dotenv": "^10.0.0", "drizzle-orm": "^0.44.7", "elysia": "1.3.8", diff --git a/dlt-connector/src/bootstrap/init.ts b/dlt-connector/src/bootstrap/init.ts index 13b77783c..3911b4af7 100644 --- a/dlt-connector/src/bootstrap/init.ts +++ b/dlt-connector/src/bootstrap/init.ts @@ -1,9 +1,11 @@ import { readFileSync } from 'node:fs' -import { loadCryptoKeys, MemoryBlock } from 'gradido-blockchain-js' +import { InMemoryBlockchainProvider, loadCryptoKeys, MemoryBlock } from 'gradido-blockchain-js' import { configure, getLogger, Logger } from 'log4js' import * as v from 'valibot' import { CONFIG } from '../config' import { MIN_TOPIC_EXPIRE_MILLISECONDS_FOR_UPDATE } from '../config/const' +import { KeyPairIdentifierLogic } from '../data/KeyPairIdentifier.logic' +import { ResolveKeyPair } from '../interactions/resolveKeyPair/ResolveKeyPair.context' import { SendToHieroContext } from '../interactions/sendToHiero/SendToHiero.context' import { Community, communitySchema } from '../schemas/transaction.schema' import { isPortOpenRetry } from '../utils/network' @@ -33,11 +35,18 @@ export async function checkHieroAccount(logger: Logger, clients: AppContextClien export async function checkHomeCommunity( appContext: AppContext, logger: Logger, -): Promise { +): Promise { const { backend, hiero } = appContext.clients // wait for backend server - await isPortOpenRetry(backend.url) + try { + logger.info(`Waiting for backend server to become available at ${backend.url}`) + await isPortOpenRetry(backend.url) + } catch (e) { + logger.error(`Backend server at ${backend.url} is not reachable (${e})`) + return + } + // ask backend for home community let homeCommunity = await backend.getHomeCommunityDraft() // on missing topicId, create one @@ -56,7 +65,7 @@ export async function checkHomeCommunity( await hiero.updateTopic(homeCommunity.hieroTopicId) topicInfo = await hiero.getTopicInfo(homeCommunity.hieroTopicId) logger.info( - `updated topic info, new expiration time: ${topicInfo.expirationTime.toLocaleDateString()}`, + `Topic expiration extended. New expiration time: ${topicInfo.expirationTime.toLocaleDateString()}`, ) } } @@ -64,9 +73,16 @@ export async function checkHomeCommunity( throw new Error('still no topic id, after creating topic and update community in backend.') } appContext.cache.setHomeCommunityTopicId(homeCommunity.hieroTopicId) - logger.info(`home community topic: ${homeCommunity.hieroTopicId}`) - logger.info(`gradido node server: ${appContext.clients.gradidoNode.url}`) - logger.info(`gradido backend server: ${appContext.clients.backend.url}`) + logger.info(`Home community topic id: ${homeCommunity.hieroTopicId}`) + logger.info(`Gradido node server: ${appContext.clients.gradidoNode.url}`) + logger.info(`Gradido backend server: ${appContext.clients.backend.url}`) + + await ResolveKeyPair( + new KeyPairIdentifierLogic({ + communityTopicId: homeCommunity.hieroTopicId, + communityId: homeCommunity.uuid, + }), + ) return v.parse(communitySchema, homeCommunity) } @@ -80,10 +96,11 @@ export async function checkGradidoNode( // ask gradido node if community blockchain was created try { + InMemoryBlockchainProvider.getInstance().getBlockchain(homeCommunity.uuid) if ( !(await clients.gradidoNode.getTransaction({ transactionId: 1, - topic: homeCommunity.hieroTopicId, + communityId: homeCommunity.uuid, })) ) { // if not exist, create community root transaction diff --git a/dlt-connector/src/bootstrap/initGradidoNode.ts b/dlt-connector/src/bootstrap/initGradidoNode.ts index 7c652a667..183b3c731 100644 --- a/dlt-connector/src/bootstrap/initGradidoNode.ts +++ b/dlt-connector/src/bootstrap/initGradidoNode.ts @@ -7,11 +7,7 @@ import { exportCommunities } from '../client/GradidoNode/communities' import { GradidoNodeProcess } from '../client/GradidoNode/GradidoNodeProcess' import { HieroClient } from '../client/hiero/HieroClient' import { CONFIG } from '../config' -import { - GRADIDO_NODE_HOME_FOLDER_NAME, - GRADIDO_NODE_RUNTIME_PATH, - LOG4JS_BASE_CATEGORY, -} from '../config/const' +import { GRADIDO_NODE_HOME_FOLDER_NAME, LOG4JS_BASE_CATEGORY } from '../config/const' import { checkFileExist, checkPathExist } from '../utils/filesystem' import { isPortOpen } from '../utils/network' import { AppContextClients } from './appContext' @@ -37,7 +33,7 @@ export async function initGradidoNode(clients: AppContextClients): Promise // write Hedera Address Book exportHederaAddressbooks(gradidoNodeHomeFolder, clients.hiero), // check GradidoNode Runtime, download when missing - ensureGradidoNodeRuntimeAvailable(GRADIDO_NODE_RUNTIME_PATH), + ensureGradidoNodeRuntimeAvailable(GradidoNodeProcess.getRuntimePathFileName()), // export communities to GradidoNode Folder exportCommunities(gradidoNodeHomeFolder, clients.backend), ]) @@ -57,11 +53,22 @@ async function exportHederaAddressbooks( async function ensureGradidoNodeRuntimeAvailable(runtimeFileName: string): Promise { const runtimeFolder = path.dirname(runtimeFileName) + const wantedVersion = `v${CONFIG.DLT_GRADIDO_NODE_SERVER_VERSION}` checkPathExist(runtimeFolder, true) - if (!checkFileExist(runtimeFileName)) { + let versionMatch = false + const isFileExist = checkFileExist(runtimeFileName) + if (isFileExist) { + const foundVersion = await GradidoNodeProcess.checkRuntimeVersion() + if (wantedVersion !== foundVersion) { + logger.info(`GradidoNode version detected: ${foundVersion}, required: ${wantedVersion}`) + } else { + versionMatch = true + } + } + if (!isFileExist || !versionMatch) { const runtimeArchiveFilename = createGradidoNodeRuntimeArchiveFilename() const downloadUrl = new URL( - `https://github.com/gradido/gradido_node/releases/download/v${CONFIG.DLT_GRADIDO_NODE_SERVER_VERSION}/${runtimeArchiveFilename}`, + `https://github.com/gradido/gradido_node/releases/download/${wantedVersion}/${runtimeArchiveFilename}`, ) logger.debug(`download GradidoNode Runtime from ${downloadUrl}`) const archive = await fetch(downloadUrl) diff --git a/dlt-connector/src/cache/KeyPairCacheManager.ts b/dlt-connector/src/cache/KeyPairCacheManager.ts index 8d7c3bf56..21b24cae1 100644 --- a/dlt-connector/src/cache/KeyPairCacheManager.ts +++ b/dlt-connector/src/cache/KeyPairCacheManager.ts @@ -76,4 +76,14 @@ export class KeyPairCacheManager { } return keyPair } + + public getKeyPairSync(input: string, createKeyPair: () => KeyPairEd25519): KeyPairEd25519 { + const keyPair = this.cache.get(input) + if (!keyPair) { + const keyPair = createKeyPair() + this.cache.set(input, keyPair) + return keyPair + } + return keyPair + } } diff --git a/dlt-connector/src/client/GradidoNode/GradidoNodeClient.ts b/dlt-connector/src/client/GradidoNode/GradidoNodeClient.ts index d66a58887..9d1515bfc 100644 --- a/dlt-connector/src/client/GradidoNode/GradidoNodeClient.ts +++ b/dlt-connector/src/client/GradidoNode/GradidoNodeClient.ts @@ -8,7 +8,7 @@ import { LOG4JS_BASE_CATEGORY } from '../../config/const' import { AddressType } from '../../data/AddressType.enum' import { Uuidv4Hash } from '../../data/Uuidv4Hash' import { addressTypeSchema, confirmedTransactionSchema } from '../../schemas/typeConverter.schema' -import { Hex32, Hex32Input, HieroId, hex32Schema } from '../../schemas/typeGuard.schema' +import { Hex32, Hex32Input, hex32Schema, Uuidv4 } from '../../schemas/typeGuard.schema' import { isPortOpenRetry } from '../../utils/network' import { GradidoNodeErrorCodes } from './GradidoNodeErrorCodes' import { @@ -75,7 +75,10 @@ export class GradidoNodeClient { const response = await this.rpcCall<{ transaction: string }>('getTransaction', parameter) if (response.isSuccess()) { // this.logger.debug('result: ', response.result.transaction) - return v.parse(confirmedTransactionSchema, response.result.transaction) + return v.parse(confirmedTransactionSchema, { + base64: response.result.transaction, + communityId: parameter.communityId, + }) } if (response.isError()) { if (response.error.code === GradidoNodeErrorCodes.TRANSACTION_NOT_FOUND) { @@ -88,19 +91,22 @@ export class GradidoNodeClient { /** * getLastTransaction * get the last confirmed transaction from a specific community - * @param hieroTopic the community hiero topic id + * @param communityId the community id * @returns the last confirmed transaction or undefined if blockchain for community is empty or not found * @throws GradidoNodeRequestError */ - public async getLastTransaction(hieroTopic: HieroId): Promise { + public async getLastTransaction(communityId: Uuidv4): Promise { const parameter = { format: 'base64', - topic: hieroTopic, + communityId, } const response = await this.rpcCall<{ transaction: string }>('getLastTransaction', parameter) if (response.isSuccess()) { - return v.parse(confirmedTransactionSchema, response.result.transaction) + return v.parse(confirmedTransactionSchema, { + base64: response.result.transaction, + communityId: parameter.communityId, + }) } if (response.isError()) { if (response.error.code === GradidoNodeErrorCodes.GRADIDO_NODE_ERROR) { @@ -115,7 +121,7 @@ export class GradidoNodeClient { * get list of confirmed transactions from a specific community * @param input fromTransactionId is the id of the first transaction to return * @param input maxResultCount is the max number of transactions to return - * @param input topic is the community hiero topic id + * @param input communityId is the community id * @returns list of confirmed transactions * @throws GradidoNodeRequestError * @example @@ -123,7 +129,7 @@ export class GradidoNodeClient { * const transactions = await getTransactions({ * fromTransactionId: 1, * maxResultCount: 100, - * topic: communityUuid, + * communityId: communityUuid, * }) * ``` */ @@ -137,7 +143,10 @@ export class GradidoNodeClient { parameter, ) return result.transactions.map((transactionBase64) => - v.parse(confirmedTransactionSchema, transactionBase64), + v.parse(confirmedTransactionSchema, { + base64: transactionBase64, + communityId: parameter.communityId, + }), ) } @@ -163,7 +172,10 @@ export class GradidoNodeClient { parameter, ) return response.transactions.map((transactionBase64) => - v.parse(confirmedTransactionSchema, transactionBase64), + v.parse(confirmedTransactionSchema, { + base64: transactionBase64, + communityId: parameter.communityId, + }), ) } @@ -173,15 +185,15 @@ export class GradidoNodeClient { * can be used to check if user/account exists on blockchain * look also for gmw, auf and deferred transfer accounts * @param pubkey the public key of the user or account - * @param hieroTopic the community hiero topic id + * @param communityId the community id * @returns the address type of the user/account, AddressType.NONE if not found * @throws GradidoNodeRequestError */ - public async getAddressType(pubkey: Hex32Input, hieroTopic: HieroId): Promise { + public async getAddressType(pubkey: Hex32Input, communityId: Uuidv4): Promise { const parameter = { pubkey: v.parse(hex32Schema, pubkey), - topic: hieroTopic, + communityId, } const response = await this.rpcCallResolved<{ addressType: string }>( 'getAddressType', @@ -194,17 +206,17 @@ export class GradidoNodeClient { * findUserByNameHash * find a user by name hash * @param nameHash the name hash of the user - * @param hieroTopic the community hiero topic id + * @param communityId the community id * @returns the public key of the user as hex32 string or undefined if user is not found * @throws GradidoNodeRequestError */ public async findUserByNameHash( nameHash: Uuidv4Hash, - hieroTopic: HieroId, + communityId: Uuidv4, ): Promise { const parameter = { nameHash: nameHash.getAsHexString(), - topic: hieroTopic, + communityId, } const response = await this.rpcCall<{ pubkey: string; timeUsed: string }>( 'findUserByNameHash', diff --git a/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts b/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts index 0eae2f37e..35c56f4b1 100644 --- a/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts +++ b/dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts @@ -1,12 +1,12 @@ +import path from 'node:path' import { Mutex } from 'async-mutex' -import { Subprocess, spawn } from 'bun' +import { $, Subprocess, spawn } from 'bun' import { getLogger, Logger } from 'log4js' import { CONFIG } from '../../config' import { GRADIDO_NODE_KILL_TIMEOUT_MILLISECONDS, GRADIDO_NODE_MIN_RUNTIME_BEFORE_EXIT_MILLISECONDS, GRADIDO_NODE_MIN_RUNTIME_BEFORE_RESTART_MILLISECONDS, - GRADIDO_NODE_RUNTIME_PATH, LOG4JS_BASE_CATEGORY, } from '../../config/const' import { delay } from '../../utils/time' @@ -43,20 +43,33 @@ export class GradidoNodeProcess { return GradidoNodeProcess.instance } + public static getRuntimePathFileName(): string { + const isWindows = process.platform === 'win32' + const binaryName = isWindows ? 'GradidoNode.exe' : 'GradidoNode' + + return path.join(CONFIG.DLT_GRADIDO_NODE_SERVER_HOME_FOLDER, 'bin', binaryName) + } + + public static async checkRuntimeVersion(): Promise { + return (await $`${GradidoNodeProcess.getRuntimePathFileName()} --version`.text()).trim() + } + public start() { if (this.proc) { this.logger.warn('GradidoNodeProcess already running.') return } - this.logger.info(`starting GradidoNodeProcess with path: ${GRADIDO_NODE_RUNTIME_PATH}`) + const gradidoNodeRuntimePath = GradidoNodeProcess.getRuntimePathFileName() + this.logger.info(`starting GradidoNodeProcess with path: ${gradidoNodeRuntimePath}`) this.lastStarted = new Date() const logger = this.logger - this.proc = spawn([GRADIDO_NODE_RUNTIME_PATH], { + this.proc = spawn([gradidoNodeRuntimePath], { env: { CLIENTS_HIERO_NETWORKTYPE: CONFIG.HIERO_HEDERA_NETWORK, SERVER_JSON_RPC_PORT: CONFIG.DLT_NODE_SERVER_PORT.toString(), USERPROFILE: CONFIG.DLT_GRADIDO_NODE_SERVER_HOME_FOLDER, HOME: CONFIG.DLT_GRADIDO_NODE_SERVER_HOME_FOLDER, + UNSECURE_ALLOW_CORS_ALL: CONFIG.DLT_GRADIDO_NODE_SERVER_ALLOW_CORS ? '1' : '0', }, onExit(_proc, exitCode, signalCode, error) { logger.warn(`GradidoNodeProcess exited with code ${exitCode} and signalCode ${signalCode}`) diff --git a/dlt-connector/src/client/GradidoNode/communities.ts b/dlt-connector/src/client/GradidoNode/communities.ts index a418255f7..8af18e3ef 100644 --- a/dlt-connector/src/client/GradidoNode/communities.ts +++ b/dlt-connector/src/client/GradidoNode/communities.ts @@ -5,7 +5,7 @@ import { getLogger } from 'log4js' import { CONFIG } from '../../config' import { GRADIDO_NODE_HOME_FOLDER_NAME, LOG4JS_BASE_CATEGORY } from '../../config/const' import { HieroId } from '../../schemas/typeGuard.schema' -import { checkFileExist, checkPathExist } from '../../utils/filesystem' +import { checkFileExist, checkPathExist, toFolderName } from '../../utils/filesystem' import { BackendClient } from '../backend/BackendClient' import { GradidoNodeProcess } from './GradidoNodeProcess' @@ -15,7 +15,7 @@ const ensureCommunitiesAvailableMutex: Mutex = new Mutex() // prototype, later add api call to gradido dlt node server for adding/updating communities type CommunityForDltNodeServer = { communityId: string - hieroTopicId: string + hieroTopicId?: string | null alias: string folder: string } @@ -38,28 +38,20 @@ export async function ensureCommunitiesAvailable(communityTopicIds: HieroId[]): } export async function exportCommunities(homeFolder: string, client: BackendClient): Promise { - const communities = await client.getReachableCommunities() + const communities = await client.getAuthorizedCommunities() const communitiesPath = path.join(homeFolder, 'communities.json') checkPathExist(path.dirname(communitiesPath), true) - // make sure communityName is unique - const communityName = new Set() const communitiesForDltNodeServer: CommunityForDltNodeServer[] = [] for (const com of communities) { - if (!com.uuid || !com.hieroTopicId) { + if (!com.uuid) { continue } - // use name as alias if not empty and unique, otherwise use uuid - let alias = com.name - if (!alias || communityName.has(alias)) { - alias = com.uuid - } - communityName.add(alias) communitiesForDltNodeServer.push({ communityId: com.uuid, hieroTopicId: com.hieroTopicId, - alias, + alias: com.name, // use only alpha-numeric chars for folder name - folder: alias.replace(/[^a-zA-Z0-9]/g, '_'), + folder: toFolderName(com.uuid), }) } fs.writeFileSync(communitiesPath, JSON.stringify(communitiesForDltNodeServer, null, 2)) diff --git a/dlt-connector/src/client/GradidoNode/input.schema.test.ts b/dlt-connector/src/client/GradidoNode/input.schema.test.ts index fbe63dabb..0b7b4dc71 100644 --- a/dlt-connector/src/client/GradidoNode/input.schema.test.ts +++ b/dlt-connector/src/client/GradidoNode/input.schema.test.ts @@ -1,18 +1,20 @@ import { beforeAll, describe, expect, it } from 'bun:test' +import { v4 as uuidv4 } from 'uuid' import * as v from 'valibot' import { - HieroId, HieroTransactionIdString, hieroIdSchema, hieroTransactionIdStringSchema, + Uuidv4, + uuidv4Schema, } from '../../schemas/typeGuard.schema' import { transactionIdentifierSchema } from './input.schema' -let topic: HieroId -const topicString = '0.0.261' +let communityId: Uuidv4 +const uuidv4String = uuidv4() let hieroTransactionId: HieroTransactionIdString beforeAll(() => { - topic = v.parse(hieroIdSchema, topicString) + communityId = v.parse(uuidv4Schema, uuidv4String) hieroTransactionId = v.parse(hieroTransactionIdStringSchema, '0.0.261-1755348116-1281621') }) @@ -21,39 +23,39 @@ describe('transactionIdentifierSchema ', () => { expect( v.parse(transactionIdentifierSchema, { transactionId: 1, - topic: topicString, + communityId, }), ).toEqual({ transactionId: 1, hieroTransactionId: undefined, - topic, + communityId, }) }) it('valid, transaction identified by hieroTransactionId and topic', () => { expect( v.parse(transactionIdentifierSchema, { hieroTransactionId: '0.0.261-1755348116-1281621', - topic: topicString, + communityId, }), ).toEqual({ hieroTransactionId, - topic, + communityId, }) }) - it('invalid, missing topic', () => { + it('invalid, missing communityId', () => { expect(() => v.parse(transactionIdentifierSchema, { transactionId: 1, hieroTransactionId: '0.0.261-1755348116-1281621', }), - ).toThrowError(new Error('Invalid key: Expected "topic" but received undefined')) + ).toThrowError(new Error('Invalid key: Expected "communityId" but received undefined')) }) it('invalid, transactionNr and iotaMessageId set', () => { expect(() => v.parse(transactionIdentifierSchema, { transactionId: 1, hieroTransactionId: '0.0.261-1755348116-1281621', - topic, + communityId, }), ).toThrowError(new Error('expect transactionNr or hieroTransactionId not both')) }) diff --git a/dlt-connector/src/client/GradidoNode/input.schema.ts b/dlt-connector/src/client/GradidoNode/input.schema.ts index 16a8643f5..61c9887d5 100644 --- a/dlt-connector/src/client/GradidoNode/input.schema.ts +++ b/dlt-connector/src/client/GradidoNode/input.schema.ts @@ -1,12 +1,12 @@ import * as v from 'valibot' -import { hieroIdSchema, hieroTransactionIdStringSchema } from '../../schemas/typeGuard.schema' +import { hieroTransactionIdStringSchema, uuidv4Schema } from '../../schemas/typeGuard.schema' export const transactionsRangeSchema = v.object({ // default value is 1, from first transactions fromTransactionId: v.nullish(v.pipe(v.number(), v.minValue(1, 'expect number >= 1')), 1), // default value is 100, max 100 transactions maxResultCount: v.nullish(v.pipe(v.number(), v.minValue(1, 'expect number >= 1')), 100), - topic: hieroIdSchema, + communityId: uuidv4Schema, }) export type TransactionsRangeInput = v.InferInput @@ -19,7 +19,7 @@ export const transactionIdentifierSchema = v.pipe( undefined, ), hieroTransactionId: v.nullish(hieroTransactionIdStringSchema, undefined), - topic: hieroIdSchema, + communityId: uuidv4Schema, }), v.custom((value: any) => { const setFieldsCount = diff --git a/dlt-connector/src/client/backend/BackendClient.ts b/dlt-connector/src/client/backend/BackendClient.ts index ce9660e36..7829e0e25 100644 --- a/dlt-connector/src/client/backend/BackendClient.ts +++ b/dlt-connector/src/client/backend/BackendClient.ts @@ -6,6 +6,7 @@ import { CONFIG } from '../../config' import { LOG4JS_BASE_CATEGORY } from '../../config/const' import { HieroId, Uuidv4 } from '../../schemas/typeGuard.schema' import { + getAuthorizedCommunities, getReachableCommunities, homeCommunityGraphqlQuery, setHomeCommunityTopicId, @@ -101,6 +102,19 @@ export class BackendClient { return v.parse(v.array(communitySchema), data.reachableCommunities) } + public async getAuthorizedCommunities(): Promise { + this.logger.info('get authorized communities on backend') + const { data, errors } = await this.client.rawRequest<{ authorizedCommunities: Community[] }>( + getAuthorizedCommunities, + {}, + await this.getRequestHeader(), + ) + if (errors) { + throw errors[0] + } + return v.parse(v.array(communitySchema), data.authorizedCommunities) + } + private async getRequestHeader(): Promise<{ authorization: string }> { diff --git a/dlt-connector/src/client/backend/graphql.ts b/dlt-connector/src/client/backend/graphql.ts index fafd77fcd..466a16c16 100644 --- a/dlt-connector/src/client/backend/graphql.ts +++ b/dlt-connector/src/client/backend/graphql.ts @@ -44,3 +44,12 @@ export const getReachableCommunities = gql` } ${communityFragment} ` + +export const getAuthorizedCommunities = gql` + query { + authorizedCommunities { + ...Community_common + } + } + ${communityFragment} +` diff --git a/dlt-connector/src/client/hiero/HieroClient.ts b/dlt-connector/src/client/hiero/HieroClient.ts index c32c4cbd7..088df0b4a 100644 --- a/dlt-connector/src/client/hiero/HieroClient.ts +++ b/dlt-connector/src/client/hiero/HieroClient.ts @@ -20,7 +20,7 @@ import { getLogger, Logger } from 'log4js' import * as v from 'valibot' import { CONFIG } from '../../config' import { LOG4JS_BASE_CATEGORY } from '../../config/const' -import { HieroId, hieroIdSchema } from '../../schemas/typeGuard.schema' +import { HieroId, hieroIdSchema, Uuidv4 } from '../../schemas/typeGuard.schema' import { durationInMinutesFromDates, printTimeDuration } from '../../utils/time' import { GradidoNodeClient } from '../GradidoNode/GradidoNodeClient' import { GradidoNodeProcess } from '../GradidoNode/GradidoNodeProcess' @@ -72,6 +72,7 @@ export class HieroClient { public async sendMessage( topicId: HieroId, + communityId: Uuidv4, transaction: GradidoTransaction, ): Promise { const timeUsed = new Profiler() @@ -99,10 +100,10 @@ export class HieroClient { ) // TODO: fix issue in GradidoNode // hot fix, when gradido node is running some time, the hiero listener stop working, so we check if our new transaction is received - // after 10 seconds, else restart GradidoNode + // after 20 seconds, else restart GradidoNode setTimeout(async () => { const transaction = await GradidoNodeClient.getInstance().getTransaction({ - topic: topicId, + communityId, hieroTransactionId: sendResponse.transactionId.toString(), }) if (!transaction) { @@ -121,7 +122,7 @@ export class HieroClient { GradidoNodeProcess.getInstance().start() } } - }, 10000) + }, 20000) if (logger.isInfoEnabled()) { // only for logging sendResponse.getReceiptWithSigner(this.wallet).then((receipt) => { diff --git a/dlt-connector/src/config/const.ts b/dlt-connector/src/config/const.ts index 5657f65ee..5dcd3d293 100644 --- a/dlt-connector/src/config/const.ts +++ b/dlt-connector/src/config/const.ts @@ -17,6 +17,6 @@ export const GRADIDO_NODE_RUNTIME_PATH = path.join( // if last start was less than this time, do not restart export const GRADIDO_NODE_MIN_RUNTIME_BEFORE_RESTART_MILLISECONDS = 1000 * 30 export const GRADIDO_NODE_MIN_RUNTIME_BEFORE_EXIT_MILLISECONDS = 1000 * 2 -export const GRADIDO_NODE_KILL_TIMEOUT_MILLISECONDS = 10000 +export const GRADIDO_NODE_KILL_TIMEOUT_MILLISECONDS = 60 * 1000 // currently hard coded in gradido node, update in future export const GRADIDO_NODE_HOME_FOLDER_NAME = '.gradido' diff --git a/dlt-connector/src/config/schema.ts b/dlt-connector/src/config/schema.ts index ef4558545..c89db3cc3 100644 --- a/dlt-connector/src/config/schema.ts +++ b/dlt-connector/src/config/schema.ts @@ -82,14 +82,21 @@ export const configSchema = v.object({ DLT_GRADIDO_NODE_SERVER_VERSION: v.optional( v.pipe( v.string('The version of the DLT node server, for example: 0.9.0'), - v.regex(/^\d+\.\d+\.\d+$/), + v.regex(/^\d+\.\d+\.\d+(.\d+)?$/), ), - '0.9.2', + '0.9.6.10', ), DLT_GRADIDO_NODE_SERVER_HOME_FOLDER: v.optional( v.string('The home folder for the gradido dlt node server'), path.join(__dirname, '..', '..', 'gradido_node'), ), + DLT_GRADIDO_NODE_SERVER_ALLOW_CORS: v.optional( + v.pipe( + v.string('Whether to allow CORS for the gradido dlt node server'), + v.transform((input: string) => input === 'true'), + ), + 'false', + ), BACKEND_PORT: v.optional( v.pipe( v.string('A valid port on which the backend server is running'), diff --git a/dlt-connector/src/data/KeyPairIdentifier.logic.ts b/dlt-connector/src/data/KeyPairIdentifier.logic.ts index 3b6b71c6e..0ee541572 100644 --- a/dlt-connector/src/data/KeyPairIdentifier.logic.ts +++ b/dlt-connector/src/data/KeyPairIdentifier.logic.ts @@ -54,6 +54,9 @@ export class KeyPairIdentifierLogic { return this.identifier.seed } + getCommunityId(): Uuidv4 { + return this.identifier.communityId + } getCommunityTopicId(): HieroId { return this.identifier.communityTopicId } @@ -76,7 +79,7 @@ export class KeyPairIdentifierLogic { return this.getSeed() } getCommunityKey(): string { - return this.getCommunityTopicId() + return this.getCommunityId() } getCommunityUserKey(): string { return this.deriveCommunityUserHash(0) @@ -107,7 +110,7 @@ export class KeyPairIdentifierLogic { ) } const resultString = - this.identifier.communityTopicId + + this.identifier.communityId + this.identifier.account.userUuid.replace(/-/g, '') + accountNr.toString() return new MemoryBlock(resultString).calculateHash().convertToHex() diff --git a/dlt-connector/src/data/deriveKeyPair.ts b/dlt-connector/src/data/deriveKeyPair.ts new file mode 100644 index 000000000..f4a805eb3 --- /dev/null +++ b/dlt-connector/src/data/deriveKeyPair.ts @@ -0,0 +1,46 @@ +import { KeyPairEd25519, MemoryBlock } from 'gradido-blockchain-js' +import { GradidoBlockchainCryptoError, ParameterError } from '../errors' +import { Hex32, Uuidv4 } from '../schemas/typeGuard.schema' +import { hardenDerivationIndex } from '../utils/derivationHelper' + +export function deriveFromSeed(seed: Hex32): KeyPairEd25519 { + const keyPair = KeyPairEd25519.create(MemoryBlock.fromHex(seed)) + if (!keyPair) { + throw new Error(`couldn't create keyPair from seed: ${seed}`) + } + return keyPair +} + +export function deriveFromCode(code: string): KeyPairEd25519 { + // code is expected to be 24 bytes long, but we need 32 + // so hash the seed with blake2 and we have 32 Bytes + const hash = new MemoryBlock(code).calculateHash() + const keyPair = KeyPairEd25519.create(hash) + if (!keyPair) { + throw new ParameterError(`error creating Ed25519 KeyPair from seed: ${code.substring(0, 5)}...`) + } + return keyPair +} + +export function deriveFromKeyPairAndUuid(keyPair: KeyPairEd25519, uuid: Uuidv4): KeyPairEd25519 { + const wholeHex = Buffer.from(uuid.replace(/-/g, ''), 'hex') + const parts = [] + for (let i = 0; i < 4; i++) { + parts[i] = hardenDerivationIndex(wholeHex.subarray(i * 4, (i + 1) * 4).readUInt32BE()) + } + // parts: [2206563009, 2629978174, 2324817329, 2405141782] + return parts.reduce( + (keyPair: KeyPairEd25519, node: number) => deriveFromKeyPairAndIndex(keyPair, node), + keyPair, + ) +} + +export function deriveFromKeyPairAndIndex(keyPair: KeyPairEd25519, index: number): KeyPairEd25519 { + const localKeyPair = keyPair.deriveChild(index) + if (!localKeyPair) { + throw new GradidoBlockchainCryptoError( + `KeyPairEd25519 child derivation failed, has private key: ${keyPair.hasPrivateKey()}, index: ${index}`, + ) + } + return localKeyPair +} diff --git a/dlt-connector/src/index.ts b/dlt-connector/src/index.ts index 1d4513e83..3f0081ffb 100644 --- a/dlt-connector/src/index.ts +++ b/dlt-connector/src/index.ts @@ -21,6 +21,9 @@ async function main() { // get home community, create topic if not exist, or check topic expiration and update it if needed const homeCommunity = await checkHomeCommunity(appContext, logger) + if (!homeCommunity) { + process.exit(1) + } // ask gradido node if community blockchain was created // if not exist, create community root transaction diff --git a/dlt-connector/src/interactions/resolveKeyPair/AbstractRemoteKeyPair.role.ts b/dlt-connector/src/interactions/resolveKeyPair/AbstractRemoteKeyPair.role.ts index 30b824d86..5b99a6c19 100644 --- a/dlt-connector/src/interactions/resolveKeyPair/AbstractRemoteKeyPair.role.ts +++ b/dlt-connector/src/interactions/resolveKeyPair/AbstractRemoteKeyPair.role.ts @@ -1,10 +1,10 @@ import { KeyPairEd25519 } from 'gradido-blockchain-js' -import { HieroId } from '../../schemas/typeGuard.schema' +import { Uuidv4 } from '../../schemas/typeGuard.schema' export abstract class AbstractRemoteKeyPairRole { - protected topic: HieroId - public constructor(communityTopicId: HieroId) { - this.topic = communityTopicId + protected communityId: Uuidv4 + public constructor(communityId: Uuidv4) { + this.communityId = communityId } public abstract retrieveKeyPair(): Promise } diff --git a/dlt-connector/src/interactions/resolveKeyPair/ForeignCommunityKeyPair.role.ts b/dlt-connector/src/interactions/resolveKeyPair/ForeignCommunityKeyPair.role.ts index 8911b8851..88db253a7 100644 --- a/dlt-connector/src/interactions/resolveKeyPair/ForeignCommunityKeyPair.role.ts +++ b/dlt-connector/src/interactions/resolveKeyPair/ForeignCommunityKeyPair.role.ts @@ -10,7 +10,7 @@ export class ForeignCommunityKeyPairRole extends AbstractRemoteKeyPairRole { public async retrieveKeyPair(): Promise { const transactionIdentifier = { transactionId: 1, - topic: this.topic, + communityId: this.communityId, } const firstTransaction = await GradidoNodeClient.getInstance().getTransaction(transactionIdentifier) diff --git a/dlt-connector/src/interactions/resolveKeyPair/RemoteAccountKeyPair.role.ts b/dlt-connector/src/interactions/resolveKeyPair/RemoteAccountKeyPair.role.ts index 524c3dc1a..43e1fa5f6 100644 --- a/dlt-connector/src/interactions/resolveKeyPair/RemoteAccountKeyPair.role.ts +++ b/dlt-connector/src/interactions/resolveKeyPair/RemoteAccountKeyPair.role.ts @@ -7,7 +7,7 @@ import { AbstractRemoteKeyPairRole } from './AbstractRemoteKeyPair.role' export class RemoteAccountKeyPairRole extends AbstractRemoteKeyPairRole { public constructor(private identifier: IdentifierAccount) { - super(identifier.communityTopicId) + super(identifier.communityId) } public async retrieveKeyPair(): Promise { @@ -17,7 +17,7 @@ export class RemoteAccountKeyPairRole extends AbstractRemoteKeyPairRole { const accountPublicKey = await GradidoNodeClient.getInstance().findUserByNameHash( new Uuidv4Hash(this.identifier.account.userUuid), - this.topic, + this.communityId, ) if (accountPublicKey) { return new KeyPairEd25519(MemoryBlock.createPtr(MemoryBlock.fromHex(accountPublicKey))) diff --git a/dlt-connector/src/interactions/resolveKeyPair/ResolveKeyPair.context.test.ts b/dlt-connector/src/interactions/resolveKeyPair/ResolveKeyPair.context.test.ts index 16a444cd8..3541ae943 100644 --- a/dlt-connector/src/interactions/resolveKeyPair/ResolveKeyPair.context.test.ts +++ b/dlt-connector/src/interactions/resolveKeyPair/ResolveKeyPair.context.test.ts @@ -33,6 +33,7 @@ mock.module('../../config', () => ({ })) const topicId = '0.0.21732' +const communityId = '1e88a0f4-d4fc-4cae-a7e8-a88e613ce324' const userUuid = 'aa25cf6f-2879-4745-b2ea-6d3c37fb44b0' afterAll(() => { @@ -45,7 +46,7 @@ describe('KeyPairCalculation', () => { }) it('community key pair', async () => { const identifier = new KeyPairIdentifierLogic( - v.parse(identifierKeyPairSchema, { communityTopicId: topicId }), + v.parse(identifierKeyPairSchema, { communityId, communityTopicId: topicId }), ) const keyPair = await ResolveKeyPair(identifier) expect(keyPair.getPublicKey()?.convertToHex()).toBe( @@ -55,6 +56,7 @@ describe('KeyPairCalculation', () => { it('user key pair', async () => { const identifier = new KeyPairIdentifierLogic( v.parse(identifierKeyPairSchema, { + communityId, communityTopicId: topicId, account: { userUuid }, }), @@ -70,6 +72,7 @@ describe('KeyPairCalculation', () => { it('account key pair', async () => { const identifier = new KeyPairIdentifierLogic( v.parse(identifierKeyPairSchema, { + communityId, communityTopicId: topicId, account: { userUuid, accountNr: 1 }, }), diff --git a/dlt-connector/src/interactions/resolveKeyPair/ResolveKeyPair.context.ts b/dlt-connector/src/interactions/resolveKeyPair/ResolveKeyPair.context.ts index 406463c4c..551997e9d 100644 --- a/dlt-connector/src/interactions/resolveKeyPair/ResolveKeyPair.context.ts +++ b/dlt-connector/src/interactions/resolveKeyPair/ResolveKeyPair.context.ts @@ -45,7 +45,7 @@ export async function ResolveKeyPair(input: KeyPairIdentifierLogic): Promise { const builder = new GradidoTransactionBuilder() const communityKeyPair = await ResolveKeyPair( - new KeyPairIdentifierLogic({ communityTopicId: this.community.hieroTopicId }), + new KeyPairIdentifierLogic({ + communityTopicId: this.community.hieroTopicId, + communityId: this.community.uuid, + }), ) const gmwKeyPair = communityKeyPair.deriveChild( hardenDerivationIndex(GMW_ACCOUNT_DERIVATION_INDEX), @@ -47,6 +50,7 @@ export class CommunityRootTransactionRole extends AbstractTransactionRole { } builder .setCreatedAt(this.community.creationDate) + .setSenderCommunity(this.community.uuid) .setCommunityRoot( communityKeyPair.getPublicKey(), gmwKeyPair.getPublicKey(), diff --git a/dlt-connector/src/interactions/sendToHiero/CreationTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/CreationTransaction.role.ts index 4b0f7aefd..25f55f145 100644 --- a/dlt-connector/src/interactions/sendToHiero/CreationTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/CreationTransaction.role.ts @@ -36,7 +36,7 @@ export class CreationTransactionRole extends AbstractTransactionRole { } getRecipientCommunityTopicId(): HieroId { - throw new Error('creation: cannot be used as cross group transaction') + return this.creationTransaction.user.communityTopicId } public async getGradidoTransactionBuilder(): Promise { @@ -52,6 +52,7 @@ export class CreationTransactionRole extends AbstractTransactionRole { const homeCommunityKeyPair = await ResolveKeyPair( new KeyPairIdentifierLogic({ communityTopicId: this.homeCommunityTopicId, + communityId: this.creationTransaction.user.communityId, }), ) // Memo: encrypted, home community and recipient can decrypt it @@ -64,8 +65,13 @@ export class CreationTransactionRole extends AbstractTransactionRole { new AuthenticatedEncryption(recipientKeyPair), ), ) + .setRecipientCommunity(this.creationTransaction.user.communityId) .setTransactionCreation( - new TransferAmount(recipientKeyPair.getPublicKey(), this.creationTransaction.amount), + new TransferAmount( + recipientKeyPair.getPublicKey(), + this.creationTransaction.amount, + this.creationTransaction.user.communityId, + ), this.creationTransaction.targetDate, ) .sign(signerKeyPair) diff --git a/dlt-connector/src/interactions/sendToHiero/DeferredTransferTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/DeferredTransferTransaction.role.ts index 95459a8b6..4cad8f0b8 100644 --- a/dlt-connector/src/interactions/sendToHiero/DeferredTransferTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/DeferredTransferTransaction.role.ts @@ -41,6 +41,7 @@ export class DeferredTransferTransactionRole extends AbstractTransactionRole { const recipientKeyPair = await ResolveKeyPair( new KeyPairIdentifierLogic({ communityTopicId: this.deferredTransferTransaction.linkedUser.communityTopicId, + communityId: this.deferredTransferTransaction.linkedUser.communityId, seed: this.seed, }), ) @@ -54,6 +55,7 @@ export class DeferredTransferTransactionRole extends AbstractTransactionRole { new AuthenticatedEncryption(recipientKeyPair), ), ) + .setSenderCommunity(this.deferredTransferTransaction.user.communityId) .setDeferredTransfer( new GradidoTransfer( new TransferAmount( @@ -61,6 +63,7 @@ export class DeferredTransferTransactionRole extends AbstractTransactionRole { this.deferredTransferTransaction.amount.calculateCompoundInterest( this.deferredTransferTransaction.timeoutDuration.getSeconds(), ), + this.deferredTransferTransaction.user.communityId, ), recipientKeyPair.getPublicKey(), ), diff --git a/dlt-connector/src/interactions/sendToHiero/RedeemDeferredTransferTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/RedeemDeferredTransferTransaction.role.ts index 4615a4707..5053e0211 100644 --- a/dlt-connector/src/interactions/sendToHiero/RedeemDeferredTransferTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/RedeemDeferredTransferTransaction.role.ts @@ -59,12 +59,15 @@ export class RedeemDeferredTransferTransactionRole extends AbstractTransactionRo builder .setCreatedAt(this.redeemDeferredTransferTransaction.createdAt) + .setSenderCommunity(this.redeemDeferredTransferTransaction.user.communityId) + .setRecipientCommunity(this.linkedUser.communityId) .setRedeemDeferredTransfer( this.parentDeferredTransaction.getId(), new GradidoTransfer( new TransferAmount( senderKeyPair.getPublicKey(), this.redeemDeferredTransferTransaction.amount, + this.redeemDeferredTransferTransaction.user.communityId, ), recipientKeyPair.getPublicKey(), ), @@ -73,12 +76,6 @@ export class RedeemDeferredTransferTransactionRole extends AbstractTransactionRo for (let i = 0; i < memos.size(); i++) { builder.addMemo(memos.get(i)) } - const senderCommunity = this.redeemDeferredTransferTransaction.user.communityTopicId - const recipientCommunity = this.linkedUser.communityTopicId - if (senderCommunity !== recipientCommunity) { - // we have a cross group transaction - builder.setSenderCommunity(senderCommunity).setRecipientCommunity(recipientCommunity) - } builder.sign(senderKeyPair) return builder } diff --git a/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.test.ts b/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.test.ts index ab07a2f83..c1be4d10e 100644 --- a/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.test.ts +++ b/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.test.ts @@ -1,14 +1,20 @@ import { describe, expect, it } from 'bun:test' -import { InteractionValidate, ValidateType_SINGLE } from 'gradido-blockchain-js' +import { + InMemoryBlockchainProvider, + InteractionValidate, + ValidateType_SINGLE, +} from 'gradido-blockchain-js' import * as v from 'valibot' import { transactionSchema } from '../../schemas/transaction.schema' import { hieroIdSchema } from '../../schemas/typeGuard.schema' import { RegisterAddressTransactionRole } from './RegisterAddressTransaction.role' const userUuid = '408780b2-59b3-402a-94be-56a4f4f4e8ec' +const communityId = '1e88a0f4-d4fc-4cae-a7e8-a88e613ce324' const transaction = { user: { communityTopicId: '0.0.21732', + communityId, account: { userUuid, accountNr: 0, @@ -18,6 +24,8 @@ const transaction = { accountType: 'COMMUNITY_HUMAN', createdAt: '2022-01-01T00:00:00.000Z', } +// create blockchain in native module +InMemoryBlockchainProvider.getInstance().getBlockchain(communityId) describe('RegisterAddressTransaction.role', () => { it('get correct prepared builder', async () => { diff --git a/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.ts index acb40975c..b85b6482d 100644 --- a/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.ts @@ -35,7 +35,12 @@ export class RegisterAddressTransactionRole extends AbstractTransactionRole { public async getGradidoTransactionBuilder(): Promise { const builder = new GradidoTransactionBuilder() const communityTopicId = this.registerAddressTransaction.user.communityTopicId - const communityKeyPair = await ResolveKeyPair(new KeyPairIdentifierLogic({ communityTopicId })) + const communityKeyPair = await ResolveKeyPair( + new KeyPairIdentifierLogic({ + communityTopicId, + communityId: this.registerAddressTransaction.user.communityId, + }), + ) const keyPairIdentifier = this.registerAddressTransaction.user // when accountNr is 0 it is the user account keyPairIdentifier.account.accountNr = 0 @@ -45,6 +50,7 @@ export class RegisterAddressTransactionRole extends AbstractTransactionRole { builder .setCreatedAt(this.registerAddressTransaction.createdAt) + .setSenderCommunity(this.registerAddressTransaction.user.communityId) .setRegisterAddress( userKeyPair.getPublicKey(), this.registerAddressTransaction.accountType as AddressType, diff --git a/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts b/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts index 622addb28..bacc01401 100644 --- a/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts +++ b/dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts @@ -1,8 +1,8 @@ import { GradidoTransaction, HieroTransactionId, - InteractionSerialize, InteractionValidate, + LedgerAnchor, ValidateType_SINGLE, } from 'gradido-blockchain-js' import { getLogger } from 'log4js' @@ -23,6 +23,8 @@ import { HieroTransactionIdString, hieroTransactionIdStringSchema, identifierSeedSchema, + Uuidv4, + uuidv4Schema, } from '../../schemas/typeGuard.schema' import { isTopicStillOpen } from '../../utils/hiero' import { LinkedTransactionKeyPairRole } from '../resolveKeyPair/LinkedTransactionKeyPair.role' @@ -59,20 +61,24 @@ export async function SendToHieroContext( const outboundHieroTransactionIdString = await sendViaHiero( outboundTransaction, role.getSenderCommunityTopicId(), + v.parse(uuidv4Schema, outboundTransaction.getCommunityId()), ) - // serialize Hiero transaction ID and attach it to the builder for the inbound transaction - const transactionIdSerializer = new InteractionSerialize( - new HieroTransactionId(outboundHieroTransactionIdString), + // attach Hiero transaction ID to the builder for the inbound transaction + builder.setParentLedgerAnchor( + new LedgerAnchor(new HieroTransactionId(outboundHieroTransactionIdString)), ) - builder.setParentMessageId(transactionIdSerializer.run()) // build and validate inbound transaction const inboundTransaction = builder.buildInbound() validate(inboundTransaction) // send inbound transaction to hiero - await sendViaHiero(inboundTransaction, role.getRecipientCommunityTopicId()) + await sendViaHiero( + inboundTransaction, + role.getRecipientCommunityTopicId(), + v.parse(uuidv4Schema, inboundTransaction.getCommunityId()), + ) return outboundHieroTransactionIdString } else { // build and validate local transaction @@ -83,6 +89,7 @@ export async function SendToHieroContext( const hieroTransactionIdString = await sendViaHiero( transaction, role.getSenderCommunityTopicId(), + v.parse(uuidv4Schema, transaction.getCommunityId()), ) return hieroTransactionIdString } @@ -98,9 +105,10 @@ function validate(transaction: GradidoTransaction): void { async function sendViaHiero( gradidoTransaction: GradidoTransaction, topic: HieroId, + communityId: Uuidv4, ): Promise { const client = HieroClient.getInstance() - const transactionId = await client.sendMessage(topic, gradidoTransaction) + const transactionId = await client.sendMessage(topic, communityId, gradidoTransaction) if (!transactionId) { throw new Error('missing transaction id from hiero') } @@ -156,7 +164,7 @@ async function chooseCorrectRole( throw new Error("redeem deferred transfer: couldn't generate seed public key") } const transactions = await GradidoNodeClient.getInstance().getTransactionsForAccount( - { maxResultCount: 2, topic: transaction.user.communityTopicId }, + { maxResultCount: 2, communityId: transaction.user.communityId }, seedPublicKey.convertToHex(), ) if (!transactions || transactions.length !== 1) { diff --git a/dlt-connector/src/interactions/sendToHiero/TransferTransaction.role.ts b/dlt-connector/src/interactions/sendToHiero/TransferTransaction.role.ts index f0a1314cb..eb2d19972 100644 --- a/dlt-connector/src/interactions/sendToHiero/TransferTransaction.role.ts +++ b/dlt-connector/src/interactions/sendToHiero/TransferTransaction.role.ts @@ -40,7 +40,6 @@ export class TransferTransactionRole extends AbstractTransactionRole { const recipientKeyPair = await ResolveKeyPair( new KeyPairIdentifierLogic(this.transferTransaction.linkedUser), ) - builder .setCreatedAt(this.transferTransaction.createdAt) .addMemo( @@ -50,16 +49,16 @@ export class TransferTransactionRole extends AbstractTransactionRole { new AuthenticatedEncryption(recipientKeyPair), ), ) + .setSenderCommunity(this.transferTransaction.user.communityId) + .setRecipientCommunity(this.transferTransaction.linkedUser.communityId) .setTransactionTransfer( - new TransferAmount(senderKeyPair.getPublicKey(), this.transferTransaction.amount), + new TransferAmount( + senderKeyPair.getPublicKey(), + this.transferTransaction.amount, + this.transferTransaction.user.communityId, + ), recipientKeyPair.getPublicKey(), ) - const senderCommunity = this.transferTransaction.user.communityTopicId - const recipientCommunity = this.transferTransaction.linkedUser.communityTopicId - if (senderCommunity !== recipientCommunity) { - // we have a cross group transaction - builder.setSenderCommunity(senderCommunity).setRecipientCommunity(recipientCommunity) - } builder.sign(senderKeyPair) return builder } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts deleted file mode 100644 index fd89d5ea0..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts +++ /dev/null @@ -1,181 +0,0 @@ -import { - Filter, - GradidoTransactionBuilder, - HieroAccountId, - HieroTransactionId, - InMemoryBlockchain, - InteractionSerialize, - Timestamp, -} from 'gradido-blockchain-js' -import { getLogger } from 'log4js' -import * as v from 'valibot' -import { LOG4JS_BASE_CATEGORY } from '../../config/const' -import { InputTransactionType } from '../../data/InputTransactionType.enum' -import { LinkedTransactionKeyPairRole } from '../../interactions/resolveKeyPair/LinkedTransactionKeyPair.role' -import { CommunityRootTransactionRole } from '../../interactions/sendToHiero/CommunityRootTransaction.role' -import { CreationTransactionRole } from '../../interactions/sendToHiero/CreationTransaction.role' -import { DeferredTransferTransactionRole } from '../../interactions/sendToHiero/DeferredTransferTransaction.role' -import { RedeemDeferredTransferTransactionRole } from '../../interactions/sendToHiero/RedeemDeferredTransferTransaction.role' -import { RegisterAddressTransactionRole } from '../../interactions/sendToHiero/RegisterAddressTransaction.role' -import { TransferTransactionRole } from '../../interactions/sendToHiero/TransferTransaction.role' -import { Community, Transaction } from '../../schemas/transaction.schema' -import { identifierSeedSchema } from '../../schemas/typeGuard.schema' - -const logger = getLogger( - `${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`, -) -export const defaultHieroAccount = new HieroAccountId(0, 0, 2) - -function addToBlockchain( - builder: GradidoTransactionBuilder, - blockchain: InMemoryBlockchain, - createdAtTimestamp: Timestamp, -): boolean { - const transaction = builder.build() - // TOD: use actual transaction id if exist in dlt_transactions table - const transactionId = new HieroTransactionId(createdAtTimestamp, defaultHieroAccount) - const interactionSerialize = new InteractionSerialize(transactionId) - - try { - const result = blockchain.createAndAddConfirmedTransaction( - transaction, - interactionSerialize.run(), - createdAtTimestamp, - ) - return result - } catch (error) { - logger.error(`Transaction ${transaction.toJson(true)} not added: ${error}`) - return false - } -} - -export async function addCommunityRootTransaction( - blockchain: InMemoryBlockchain, - community: Community, -): Promise { - const communityRootTransactionRole = new CommunityRootTransactionRole(community) - if ( - addToBlockchain( - await communityRootTransactionRole.getGradidoTransactionBuilder(), - blockchain, - new Timestamp(community.creationDate), - ) - ) { - logger.info(`Community Root Transaction added`) - } else { - throw new Error(`Community Root Transaction not added`) - } -} - -export async function addRegisterAddressTransaction( - blockchain: InMemoryBlockchain, - transaction: Transaction, -): Promise { - const registerAddressRole = new RegisterAddressTransactionRole(transaction) - if ( - addToBlockchain( - await registerAddressRole.getGradidoTransactionBuilder(), - blockchain, - new Timestamp(transaction.createdAt), - ) - ) { - logger.debug( - `Register Address Transaction added for user ${transaction.user.account!.userUuid}`, - ) - } else { - throw new Error( - `Register Address Transaction not added for user ${transaction.user.account!.userUuid}`, - ) - } -} - -export async function addTransaction( - senderBlockchain: InMemoryBlockchain, - _recipientBlockchain: InMemoryBlockchain, - transaction: Transaction, -): Promise { - const createdAtTimestamp = new Timestamp(transaction.createdAt) - if (transaction.type === InputTransactionType.GRADIDO_CREATION) { - const creationTransactionRole = new CreationTransactionRole(transaction) - if ( - addToBlockchain( - await creationTransactionRole.getGradidoTransactionBuilder(), - senderBlockchain, - createdAtTimestamp, - ) - ) { - logger.debug(`Creation Transaction added for user ${transaction.user.account!.userUuid}`) - } else { - throw new Error( - `Creation Transaction not added for user ${transaction.user.account!.userUuid}`, - ) - } - } else if (transaction.type === InputTransactionType.GRADIDO_TRANSFER) { - const transferTransactionRole = new TransferTransactionRole(transaction) - // will crash with cross group transaction - if ( - addToBlockchain( - await transferTransactionRole.getGradidoTransactionBuilder(), - senderBlockchain, - createdAtTimestamp, - ) - ) { - logger.debug(`Transfer Transaction added for user ${transaction.user.account!.userUuid}`) - } else { - throw new Error( - `Transfer Transaction not added for user ${transaction.user.account!.userUuid}`, - ) - } - } else if (transaction.type === InputTransactionType.GRADIDO_DEFERRED_TRANSFER) { - const transferTransactionRole = new DeferredTransferTransactionRole(transaction) - if ( - addToBlockchain( - await transferTransactionRole.getGradidoTransactionBuilder(), - senderBlockchain, - createdAtTimestamp, - ) - ) { - logger.debug( - `Deferred Transfer Transaction added for user ${transaction.user.account!.userUuid}`, - ) - } else { - throw new Error( - `Deferred Transfer Transaction not added for user ${transaction.user.account!.userUuid}`, - ) - } - } else if (transaction.type === InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER) { - const seedKeyPairRole = new LinkedTransactionKeyPairRole( - v.parse(identifierSeedSchema, transaction.user.seed), - ) - const f = new Filter() - f.involvedPublicKey = seedKeyPairRole.generateKeyPair().getPublicKey() - const deferredTransaction = senderBlockchain.findOne(f) - if (!deferredTransaction) { - throw new Error( - `redeem deferred transfer: couldn't find parent deferred transfer on Gradido Node for ${JSON.stringify(transaction, null, 2)} and public key from seed: ${f.involvedPublicKey?.convertToHex()}`, - ) - } - const confirmedDeferredTransaction = deferredTransaction.getConfirmedTransaction() - if (!confirmedDeferredTransaction) { - throw new Error('redeem deferred transfer: invalid TransactionEntry') - } - const redeemTransactionRole = new RedeemDeferredTransferTransactionRole( - transaction, - confirmedDeferredTransaction, - ) - const involvedUser = transaction.user.account - ? transaction.user.account.userUuid - : transaction.linkedUser?.account?.userUuid - if ( - addToBlockchain( - await redeemTransactionRole.getGradidoTransactionBuilder(), - senderBlockchain, - createdAtTimestamp, - ) - ) { - logger.debug(`Redeem Deferred Transfer Transaction added for user ${involvedUser}`) - } else { - throw new Error(`Redeem Deferred Transfer Transaction not added for user ${involvedUser}`) - } - } -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts deleted file mode 100644 index 45eff5ffd..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { InMemoryBlockchainProvider } from 'gradido-blockchain-js' -import * as v from 'valibot' -import { HieroId, hieroIdSchema } from '../../schemas/typeGuard.schema' -import { addCommunityRootTransaction } from './blockchain' -import { Context } from './Context' -import { communityDbToCommunity } from './convert' -import { loadCommunities } from './database' -import { generateKeyPairCommunity } from './keyPair' -import { CommunityContext } from './valibot.schema' - -export async function bootstrap(): Promise { - const context = await Context.create() - context.communities = await bootstrapCommunities(context) - return context -} - -async function bootstrapCommunities(context: Context): Promise> { - const communities = new Map() - const communitiesDb = await loadCommunities(context.db) - const topicIds = new Set() - - for (const communityDb of communitiesDb) { - const blockchain = InMemoryBlockchainProvider.getInstance().findBlockchain( - communityDb.uniqueAlias, - ) - if (!blockchain) { - throw new Error(`Couldn't create Blockchain for community ${communityDb.communityUuid}`) - } - context.logger.info(`Blockchain for community '${communityDb.uniqueAlias}' created`) - // make sure topic id is unique - let topicId: HieroId - do { - topicId = v.parse(hieroIdSchema, '0.0.' + Math.floor(Math.random() * 10000)) - } while (topicIds.has(topicId)) - topicIds.add(topicId) - - communities.set(communityDb.communityUuid, { - communityId: communityDb.uniqueAlias, - blockchain, - topicId, - folder: communityDb.uniqueAlias.replace(/[^a-zA-Z0-9]/g, '_'), - }) - - generateKeyPairCommunity(communityDb, context.cache, topicId) - let creationDate = communityDb.creationDate - if (communityDb.userMinCreatedAt && communityDb.userMinCreatedAt < communityDb.creationDate) { - // create community root transaction 1 minute before first user - creationDate = new Date(new Date(communityDb.userMinCreatedAt).getTime() - 1000 * 60) - } - // community from db to community format the dlt connector normally uses - const community = communityDbToCommunity(topicId, communityDb, creationDate) - await addCommunityRootTransaction(blockchain, community) - } - return communities -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/convert.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/convert.ts deleted file mode 100644 index e60f98dcf..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/convert.ts +++ /dev/null @@ -1,136 +0,0 @@ -import * as v from 'valibot' -import { AccountType } from '../../data/AccountType.enum' -import { InputTransactionType } from '../../data/InputTransactionType.enum' -import { - Community, - communitySchema, - Transaction, - TransactionInput, - transactionSchema, -} from '../../schemas/transaction.schema' -import { - gradidoAmountSchema, - HieroId, - memoSchema, - timeoutDurationSchema, -} from '../../schemas/typeGuard.schema' -import { TransactionTypeId } from './TransactionTypeId' -import { CommunityDb, CreatedUserDb, TransactionDb, TransactionLinkDb } from './valibot.schema' - -export function getInputTransactionTypeFromTypeId(typeId: TransactionTypeId): InputTransactionType { - switch (typeId) { - case TransactionTypeId.CREATION: - return InputTransactionType.GRADIDO_CREATION - case TransactionTypeId.SEND: - return InputTransactionType.GRADIDO_TRANSFER - case TransactionTypeId.RECEIVE: - throw new Error('not used') - default: - throw new Error('not implemented') - } -} - -export function communityDbToCommunity( - topicId: HieroId, - communityDb: CommunityDb, - creationDate: Date, -): Community { - return v.parse(communitySchema, { - hieroTopicId: topicId, - uuid: communityDb.communityUuid, - foreign: communityDb.foreign, - creationDate, - }) -} - -export function userDbToTransaction(userDb: CreatedUserDb, communityTopicId: HieroId): Transaction { - return v.parse(transactionSchema, { - user: { - communityTopicId: communityTopicId, - account: { userUuid: userDb.gradidoId }, - }, - type: InputTransactionType.REGISTER_ADDRESS, - accountType: AccountType.COMMUNITY_HUMAN, - createdAt: userDb.createdAt, - }) -} - -export function transactionDbToTransaction( - transactionDb: TransactionDb, - communityTopicId: HieroId, - recipientCommunityTopicId: HieroId, -): Transaction { - if ( - transactionDb.typeId !== TransactionTypeId.CREATION && - transactionDb.typeId !== TransactionTypeId.SEND && - transactionDb.typeId !== TransactionTypeId.RECEIVE - ) { - throw new Error('not implemented') - } - - const user = { - communityTopicId: communityTopicId, - account: { userUuid: transactionDb.user.gradidoId }, - } - const linkedUser = { - communityTopicId: recipientCommunityTopicId, - account: { userUuid: transactionDb.linkedUser.gradidoId }, - } - const transaction: TransactionInput = { - user, - linkedUser, - amount: v.parse(gradidoAmountSchema, transactionDb.amount), - memo: v.parse(memoSchema, transactionDb.memo), - type: InputTransactionType.GRADIDO_TRANSFER, - createdAt: transactionDb.balanceDate, - } - if (transactionDb.typeId === TransactionTypeId.CREATION) { - if (!transactionDb.creationDate) { - throw new Error('contribution transaction without creation date') - } - transaction.targetDate = transactionDb.creationDate - transaction.type = InputTransactionType.GRADIDO_CREATION - } else if (transactionDb.typeId === TransactionTypeId.RECEIVE) { - transaction.user = linkedUser - transaction.linkedUser = user - } - if (transactionDb.transactionLinkCode) { - if (transactionDb.typeId !== TransactionTypeId.RECEIVE) { - throw new Error( - "linked transaction which isn't receive, send will taken care of on link creation", - ) - } - transaction.user = { - communityTopicId: recipientCommunityTopicId, - seed: transactionDb.transactionLinkCode, - } - transaction.type = InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER - } - return v.parse(transactionSchema, transaction) -} - -export function transactionLinkDbToTransaction( - transactionLinkDb: TransactionLinkDb, - communityTopicId: HieroId, -): Transaction { - return v.parse(transactionSchema, { - user: { - communityTopicId: communityTopicId, - account: { userUuid: transactionLinkDb.user.gradidoId }, - }, - linkedUser: { - communityTopicId: communityTopicId, - seed: transactionLinkDb.code, - }, - type: InputTransactionType.GRADIDO_DEFERRED_TRANSFER, - amount: v.parse(gradidoAmountSchema, transactionLinkDb.amount), - memo: v.parse(memoSchema, transactionLinkDb.memo), - createdAt: transactionLinkDb.createdAt, - timeoutDuration: v.parse( - timeoutDurationSchema, - Math.round( - (transactionLinkDb.validUntil.getTime() - transactionLinkDb.createdAt.getTime()) / 1000, - ), - ), - }) -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts deleted file mode 100644 index 87cb3eb4a..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts +++ /dev/null @@ -1,189 +0,0 @@ -import { asc, eq, inArray, isNotNull, sql } from 'drizzle-orm' -import { alias } from 'drizzle-orm/mysql-core' -import { MySql2Database } from 'drizzle-orm/mysql2' -import { GradidoUnit } from 'gradido-blockchain-js' -import { getLogger } from 'log4js' -import * as v from 'valibot' -import { LOG4JS_BASE_CATEGORY } from '../../config/const' -import { - communitiesTable, - transactionLinksTable, - transactionsTable, - usersTable, -} from './drizzle.schema' -import { TransactionTypeId } from './TransactionTypeId' -import { - CommunityDb, - CreatedUserDb, - communityDbSchema, - createdUserDbSchema, - TransactionDb, - TransactionLinkDb, - transactionDbSchema, - transactionLinkDbSchema, -} from './valibot.schema' - -const logger = getLogger( - `${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`, -) - -// queries -export async function loadCommunities(db: MySql2Database): Promise { - const result = await db - .select({ - foreign: communitiesTable.foreign, - communityUuid: communitiesTable.communityUuid, - name: communitiesTable.name, - creationDate: communitiesTable.creationDate, - userMinCreatedAt: sql`MIN(${usersTable.createdAt})`, - }) - .from(communitiesTable) - .leftJoin(usersTable, eq(communitiesTable.communityUuid, usersTable.communityUuid)) - .where(isNotNull(communitiesTable.communityUuid)) - .orderBy(asc(communitiesTable.id)) - .groupBy(communitiesTable.communityUuid) - - const communityNames = new Set() - return result.map((row: any) => { - let alias = row.name - if (communityNames.has(row.name)) { - alias = row.community_uuid - } else { - communityNames.add(row.name) - } - return v.parse(communityDbSchema, { - ...row, - uniqueAlias: alias, - }) - }) -} - -export async function loadUsers( - db: MySql2Database, - offset: number, - count: number, -): Promise { - const result = await db - .select() - .from(usersTable) - .orderBy(asc(usersTable.createdAt), asc(usersTable.id)) - .limit(count) - .offset(offset) - - return result.map((row: any) => { - return v.parse(createdUserDbSchema, row) - }) -} - -export async function loadTransactions( - db: MySql2Database, - offset: number, - count: number, -): Promise { - const linkedUsers = alias(usersTable, 'linkedUser') - - const result = await db - .select({ - transaction: transactionsTable, - user: usersTable, - linkedUser: linkedUsers, - transactionLink: transactionLinksTable, - }) - .from(transactionsTable) - .where( - inArray(transactionsTable.typeId, [TransactionTypeId.CREATION, TransactionTypeId.RECEIVE]), - ) - .leftJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) - .leftJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id)) - .leftJoin( - transactionLinksTable, - eq(transactionsTable.transactionLinkId, transactionLinksTable.id), - ) - .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) - .limit(count) - .offset(offset) - - return result.map((row: any) => { - // console.log(row) - try { - // check for consistent data beforehand - const userCreatedAt = new Date(row.user.createdAt) - const linkedUserCreatedAd = new Date(row.linkedUser.createdAt) - const balanceDate = new Date(row.transaction.balanceDate) - if ( - userCreatedAt.getTime() > balanceDate.getTime() || - linkedUserCreatedAd.getTime() > balanceDate.getTime() - ) { - logger.error(`table row: `, row) - throw new Error( - 'at least one user was created after transaction balance date, logic error!', - ) - } - - let amount = GradidoUnit.fromString(row.transaction.amount) - if (row.transaction.typeId === TransactionTypeId.SEND) { - amount = amount.mul(new GradidoUnit(-1)) - } - return v.parse(transactionDbSchema, { - ...row.transaction, - transactionLinkCode: row.transactionLink ? row.transactionLink.code : null, - user: row.user, - linkedUser: row.linkedUser, - }) - } catch (e) { - logger.error(`table row: ${JSON.stringify(row, null, 2)}`) - if (e instanceof v.ValiError) { - logger.error(v.flatten(e.issues)) - } - throw e - } - }) -} - -export async function loadTransactionLinks( - db: MySql2Database, - offset: number, - count: number, -): Promise { - const result = await db - .select() - .from(transactionLinksTable) - .leftJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id)) - .orderBy(asc(transactionLinksTable.createdAt), asc(transactionLinksTable.id)) - .limit(count) - .offset(offset) - - return result.map((row: any) => { - return v.parse(transactionLinkDbSchema, { - ...row.transaction_links, - user: row.users, - }) - }) -} - -export async function loadDeletedTransactionLinks( - db: MySql2Database, - offset: number, - count: number, -): Promise { - const result = await db - .select() - .from(transactionLinksTable) - .leftJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id)) - .where(isNotNull(transactionLinksTable.deletedAt)) - .orderBy(asc(transactionLinksTable.deletedAt), asc(transactionLinksTable.id)) - .limit(count) - .offset(offset) - - return result.map((row: any) => { - return v.parse(transactionDbSchema, { - typeId: TransactionTypeId.RECEIVE, - amount: row.transaction_links.amount, - balanceDate: new Date(row.transaction_links.deletedAt), - memo: row.transaction_links.memo, - transactionLinkCode: row.transaction_links.code, - user: row.users, - linkedUser: row.users, - }) - }) -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts deleted file mode 100644 index 875c59eba..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { Filter } from 'gradido-blockchain-js' -import { onShutdown } from '../../../../shared/src/helper/onShutdown' -import { exportAllCommunities } from './binaryExport' -import { bootstrap } from './bootstrap' -import { syncDbWithBlockchainContext } from './interaction/syncDbWithBlockchain/syncDbWithBlockchain.context' - -const BATCH_SIZE = 100 - -async function main() { - // prepare in memory blockchains - const context = await bootstrap() - onShutdown(async (reason, error) => { - context.logger.info(`shutdown reason: ${reason}`) - if (error) { - context.logger.error(error) - } - }) - - // synchronize to in memory blockchain - await syncDbWithBlockchainContext(context, BATCH_SIZE) - - // write as binary file for GradidoNode - exportAllCommunities(context, BATCH_SIZE) - - // log runtime statistics - context.logRuntimeStatistics() - - // needed because of shutdown handler (TODO: fix shutdown handler) - process.exit(0) -} - -main().catch((e) => { - // biome-ignore lint/suspicious/noConsole: maybe logger isn't initialized here - console.error(e) - process.exit(1) -}) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts deleted file mode 100644 index 90523fa1b..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { Profiler } from 'gradido-blockchain-js' -import { getLogger, Logger } from 'log4js' -import { LOG4JS_BASE_CATEGORY } from '../../../../config/const' -import { Context } from '../../Context' - -export abstract class AbstractSyncRole { - private items: T[] = [] - private offset = 0 - protected logger: Logger - - constructor(protected readonly context: Context) { - this.logger = getLogger( - `${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5.interaction.syncDbWithBlockchain`, - ) - } - - abstract getDate(): Date - abstract loadFromDb(offset: number, count: number): Promise - abstract pushToBlockchain(item: T): Promise - abstract itemTypeName(): string - - // return count of new loaded items - async ensureFilled(batchSize: number): Promise { - if (this.items.length === 0) { - let timeUsed: Profiler | undefined - if (this.logger.isDebugEnabled()) { - timeUsed = new Profiler() - } - this.items = await this.loadFromDb(this.offset, batchSize) - this.offset += this.items.length - if (timeUsed && this.items.length) { - this.logger.debug( - `${timeUsed.string()} for loading ${this.items.length} ${this.itemTypeName()} from db`, - ) - } - return this.items.length - } - return 0 - } - - async toBlockchain(): Promise { - if (this.isEmpty()) { - throw new Error(`[toBlockchain] No items, please call this only if isEmpty returns false`) - } - await this.pushToBlockchain(this.shift()) - } - - peek(): T { - if (this.isEmpty()) { - throw new Error(`[peek] No items, please call this only if isEmpty returns false`) - } - return this.items[0] - } - - shift(): T { - const item = this.items.shift() - if (!item) { - throw new Error(`[shift] No items, shift return undefined`) - } - return item - } - - get length(): number { - return this.items.length - } - - isEmpty(): boolean { - return this.items.length === 0 - } -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts deleted file mode 100644 index fd1ae225f..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { loadDeletedTransactionLinks } from '../../database' -import { TransactionDb } from '../../valibot.schema' -import { TransactionsSyncRole } from './TransactionsSync.role' - -export class DeletedTransactionLinksSyncRole extends TransactionsSyncRole { - itemTypeName(): string { - return 'deletedTransactionLinks' - } - - async loadFromDb(offset: number, count: number): Promise { - return await loadDeletedTransactionLinks(this.context.db, offset, count) - } -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinksSync.role.ts deleted file mode 100644 index 847be89ba..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinksSync.role.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { addTransaction } from '../../blockchain' -import { transactionLinkDbToTransaction } from '../../convert' -import { loadTransactionLinks } from '../../database' -import { TransactionLinkDb } from '../../valibot.schema' -import { AbstractSyncRole } from './AbstractSync.role' - -export class TransactionLinksSyncRole extends AbstractSyncRole { - getDate(): Date { - return this.peek().createdAt - } - - itemTypeName(): string { - return 'transactionLinks' - } - - async loadFromDb(offset: number, count: number): Promise { - return await loadTransactionLinks(this.context.db, offset, count) - } - - async pushToBlockchain(item: TransactionLinkDb): Promise { - const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) - const transaction = transactionLinkDbToTransaction(item, communityContext.topicId) - await addTransaction(communityContext.blockchain, communityContext.blockchain, transaction) - } -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionsSync.role.ts deleted file mode 100644 index df8709ac8..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionsSync.role.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { addTransaction } from '../../blockchain' -import { transactionDbToTransaction } from '../../convert' -import { loadTransactions } from '../../database' -import { TransactionDb } from '../../valibot.schema' -import { AbstractSyncRole } from './AbstractSync.role' - -export class TransactionsSyncRole extends AbstractSyncRole { - getDate(): Date { - return this.peek().balanceDate - } - - itemTypeName(): string { - return 'transactions' - } - - async loadFromDb(offset: number, count: number): Promise { - return await loadTransactions(this.context.db, offset, count) - } - - async pushToBlockchain(item: TransactionDb): Promise { - const senderCommunityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) - const recipientCommunityContext = this.context.getCommunityContextByUuid( - item.linkedUser.communityUuid, - ) - this.context.cache.setHomeCommunityTopicId(senderCommunityContext.topicId) - const transaction = transactionDbToTransaction( - item, - senderCommunityContext.topicId, - recipientCommunityContext.topicId, - ) - await addTransaction( - senderCommunityContext.blockchain, - recipientCommunityContext.blockchain, - transaction, - ) - } -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts deleted file mode 100644 index d6b40938f..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { addRegisterAddressTransaction } from '../../blockchain' -import { userDbToTransaction } from '../../convert' -import { loadUsers } from '../../database' -import { generateKeyPairUserAccount } from '../../keyPair' -import { CreatedUserDb } from '../../valibot.schema' -import { AbstractSyncRole } from './AbstractSync.role' - -export class UsersSyncRole extends AbstractSyncRole { - getDate(): Date { - return this.peek().createdAt - } - - itemTypeName(): string { - return 'users' - } - - async loadFromDb(offset: number, count: number): Promise { - const users = await loadUsers(this.context.db, offset, count) - for (const user of users) { - const communityContext = this.context.getCommunityContextByUuid(user.communityUuid) - await generateKeyPairUserAccount(user, this.context.cache, communityContext.topicId) - } - return users - } - - async pushToBlockchain(item: CreatedUserDb): Promise { - const communityContext = this.context.getCommunityContextByUuid(item.communityUuid) - const transaction = userDbToTransaction(item, communityContext.topicId) - return await addRegisterAddressTransaction(communityContext.blockchain, transaction) - } -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts deleted file mode 100644 index e67cd70da..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { Profiler } from 'gradido-blockchain-js' -import { Context } from '../../Context' -import { DeletedTransactionLinksSyncRole } from './DeletedTransactionLinksSync.role' -import { TransactionLinksSyncRole } from './TransactionLinksSync.role' -import { TransactionsSyncRole } from './TransactionsSync.role' -import { UsersSyncRole } from './UsersSync.role' - -export async function syncDbWithBlockchainContext(context: Context, batchSize: number) { - const timeUsed = new Profiler() - const containers = [ - new UsersSyncRole(context), - new TransactionsSyncRole(context), - new DeletedTransactionLinksSyncRole(context), - new TransactionLinksSyncRole(context), - ] - - while (true) { - timeUsed.reset() - const results = await Promise.all(containers.map((c) => c.ensureFilled(batchSize))) - const loadedItemsCount = results.reduce((acc, c) => acc + c, 0) - // log only, if at least one new item was loaded - if (loadedItemsCount && context.logger.isInfoEnabled()) { - context.logger.info(`${loadedItemsCount} new items loaded from db in ${timeUsed.string()}`) - } - - // remove empty containers - const available = containers.filter((c) => !c.isEmpty()) - if (available.length === 0) { - break - } - - // sort by date, to ensure container on index 0 is the one with the smallest date - if (available.length > 0) { - available.sort((a, b) => a.getDate().getTime() - b.getDate().getTime()) - } - await available[0].toBlockchain() - } -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts deleted file mode 100644 index c9b4eccb9..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { crypto_generichash_batch, crypto_generichash_KEYBYTES } from 'sodium-native' - -export function bytesToMbyte(bytes: number): string { - return (bytes / 1024 / 1024).toFixed(4) -} - -export function bytesToKbyte(bytes: number): string { - return (bytes / 1024).toFixed(0) -} - -export function calculateOneHashStep(hash: Buffer, data: Buffer): Buffer { - const outputHash = Buffer.alloc(crypto_generichash_KEYBYTES, 0) - crypto_generichash_batch(outputHash, [hash, data]) - return outputHash -} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts deleted file mode 100644 index 26fc985c7..000000000 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { InMemoryBlockchain } from 'gradido-blockchain-js' -import * as v from 'valibot' -import { booleanSchema, dateSchema } from '../../schemas/typeConverter.schema' -import { - gradidoAmountSchema, - hieroIdSchema, - identifierSeedSchema, - memoSchema, - uuidv4Schema, -} from '../../schemas/typeGuard.schema' -import { TransactionTypeId } from './TransactionTypeId' - -export const createdUserDbSchema = v.object({ - gradidoId: uuidv4Schema, - communityUuid: uuidv4Schema, - createdAt: dateSchema, -}) - -export const userDbSchema = v.object({ - gradidoId: uuidv4Schema, - communityUuid: uuidv4Schema, -}) - -export const transactionDbSchema = v.object({ - typeId: v.enum(TransactionTypeId), - amount: gradidoAmountSchema, - balanceDate: dateSchema, - memo: memoSchema, - creationDate: v.nullish(dateSchema), - user: userDbSchema, - linkedUser: userDbSchema, - transactionLinkCode: v.nullish(identifierSeedSchema), -}) - -export const transactionLinkDbSchema = v.object({ - user: userDbSchema, - code: identifierSeedSchema, - amount: gradidoAmountSchema, - memo: memoSchema, - createdAt: dateSchema, - validUntil: dateSchema, -}) - -export const communityDbSchema = v.object({ - foreign: booleanSchema, - communityUuid: uuidv4Schema, - name: v.string(), - creationDate: dateSchema, - userMinCreatedAt: v.nullish(dateSchema), - uniqueAlias: v.string(), -}) - -export const communityContextSchema = v.object({ - communityId: v.string(), - blockchain: v.instance(InMemoryBlockchain, 'expect InMemoryBlockchain type'), - topicId: hieroIdSchema, - folder: v.pipe( - v.string(), - v.minLength(1, 'expect string length >= 1'), - v.maxLength(255, 'expect string length <= 255'), - v.regex(/^[a-zA-Z0-9-_]+$/, 'expect string to be a valid (alphanumeric, _, -) folder name'), - ), -}) - -export type TransactionDb = v.InferOutput -export type UserDb = v.InferOutput -export type CreatedUserDb = v.InferOutput -export type TransactionLinkDb = v.InferOutput -export type CommunityDb = v.InferOutput -export type CommunityContext = v.InferOutput diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts similarity index 90% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts index 58400a615..cffccc044 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts @@ -1,4 +1,5 @@ import { heapStats } from 'bun:jsc' +import dotenv from 'dotenv' import { drizzle, MySql2Database } from 'drizzle-orm/mysql2' import { Filter, Profiler, SearchDirection_ASC } from 'gradido-blockchain-js' import { getLogger, Logger } from 'log4js' @@ -11,6 +12,8 @@ import { Uuidv4 } from '../../schemas/typeGuard.schema' import { bytesToMbyte } from './utils' import { CommunityContext } from './valibot.schema' +dotenv.config() + export class Context { public logger: Logger public db: MySql2Database @@ -36,11 +39,9 @@ export class Context { database: CONFIG.MYSQL_DATABASE, port: CONFIG.MYSQL_PORT, }) - return new Context( - getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5`), - drizzle({ client: connection }), - KeyPairCacheManager.getInstance(), - ) + const db = drizzle({ client: connection }) + const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5`) + return new Context(logger, db, KeyPairCacheManager.getInstance()) } getCommunityContextByUuid(communityUuid: Uuidv4): CommunityContext { diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/binaryExport.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts similarity index 65% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/binaryExport.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts index 43459372b..6870740eb 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/binaryExport.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts @@ -9,12 +9,13 @@ import { } from 'gradido-blockchain-js' import { CONFIG } from '../../config' import { Context } from './Context' -import { bytesToKbyte, calculateOneHashStep } from './utils' +import { bytesString, calculateOneHashStep } from './utils' import { CommunityContext } from './valibot.schema' export function exportAllCommunities(context: Context, batchSize: number) { const timeUsed = new Profiler() for (const communityContext of context.communities.values()) { + context.logger.info(`exporting community ${communityContext.communityId} to binary file`) exportCommunity(communityContext, context, batchSize) } context.logger.info(`time used for exporting communities to binary file: ${timeUsed.string()}`) @@ -25,36 +26,77 @@ export function exportCommunity( context: Context, batchSize: number, ) { + const timeUsed = new Profiler() + const timeSinceLastPrint = new Profiler() // write as binary file for GradidoNode const f = new Filter() f.pagination.size = batchSize f.pagination.page = 1 f.searchDirection = SearchDirection_ASC const binFilePath = prepareFolder(communityContext) + let count = 0 + let printCount = 0 let lastTransactionCount = 0 + let triggeredTransactionsCount = 0 let hash = Buffer.alloc(32, 0) + const isDebug = context.logger.isDebugEnabled() + const printConsole = () => { + if (triggeredTransactionsCount > 0) { + process.stdout.write( + `exported ${count} transactions + ${triggeredTransactionsCount} triggered from timeouted transaction links\r`, + ) + } else { + process.stdout.write(`exported ${count} transactions\r`) + } + } do { const transactions = communityContext.blockchain.findAll(f) lastTransactionCount = transactions.size() for (let i = 0; i < lastTransactionCount; i++) { const confirmedTransaction = transactions.get(i)?.getConfirmedTransaction() - const transactionNr = f.pagination.page * batchSize + i + const transactionNr = (f.pagination.page - 2) * batchSize + i if (!confirmedTransaction) { throw new Error(`invalid TransactionEntry at index: ${transactionNr} `) } hash = exportTransaction(confirmedTransaction, hash, binFilePath) + if ( + confirmedTransaction + ?.getGradidoTransaction() + ?.getTransactionBody() + ?.isTimeoutDeferredTransfer() + ) { + triggeredTransactionsCount++ + } else { + count++ + } + if (isDebug) { + if (timeSinceLastPrint.millis() > 100) { + printConsole() + timeSinceLastPrint.reset() + } + } else { + printCount++ + if (printCount >= 100) { + printConsole() + printCount = 0 + } + } } f.pagination.page++ } while (lastTransactionCount === batchSize) + printConsole() + process.stdout.write(`\n`) fs.appendFileSync(binFilePath, hash!) context.logger.info( `binary file for community ${communityContext.communityId} written to ${binFilePath}`, ) + const sumTransactionsCount = (f.pagination.page - 2) * batchSize + lastTransactionCount + const fileSize = fs.statSync(binFilePath).size context.logger.info( - `transactions count: ${(f.pagination.page - 1) * batchSize + lastTransactionCount}, size: ${bytesToKbyte(fs.statSync(binFilePath).size)} KByte`, + `exported ${sumTransactionsCount} transactions (${bytesString(fileSize)}) in ${timeUsed.string()}`, ) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts new file mode 100644 index 000000000..edf752788 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts @@ -0,0 +1,50 @@ +import { + AccountBalances, + Filter, + GradidoTransaction, + HieroAccountId, + InMemoryBlockchain, + LedgerAnchor, + Profiler, +} from 'gradido-blockchain-js' +import { NotEnoughGradidoBalanceError } from './errors' + +export const defaultHieroAccount = new HieroAccountId(0, 0, 2) +export let callTime: number = 0 +const timeUsed = new Profiler() + +export function addToBlockchain( + transaction: GradidoTransaction, + blockchain: InMemoryBlockchain, + ledgerAnchor: LedgerAnchor, + accountBalances: AccountBalances, +): boolean { + try { + timeUsed.reset() + const result = blockchain.createAndAddConfirmedTransactionExternFast( + transaction, + ledgerAnchor, + accountBalances, + ) + callTime += timeUsed.nanos() + return result + } catch (error) { + if (error instanceof Error) { + const matches = error.message.match( + /not enough Gradido Balance for (send coins|operation), needed: -?(\d+\.\d+), exist: (\d+\.\d+)/, + ) + if (matches) { + const needed = parseFloat(matches[2]) + const exist = parseFloat(matches[3]) + throw new NotEnoughGradidoBalanceError(needed, exist) + } + } + // const wekingheim = InMemoryBlockchainProvider.getInstance().getBlockchain('wekingheim') + // const lastTransactionw = wekingheim?.findOne(Filter.LAST_TRANSACTION) + + const lastTransaction = blockchain.findOne(Filter.LAST_TRANSACTION) + throw new Error( + `Transaction ${transaction.toJson(true)} not added: ${error}, last transaction was: ${lastTransaction?.getConfirmedTransaction()?.toJson(true)}`, + ) + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts new file mode 100644 index 000000000..a95cc136a --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts @@ -0,0 +1,105 @@ +import { randomBytes } from 'node:crypto' +import { + AccountBalances, + GradidoTransactionBuilder, + InMemoryBlockchainProvider, + LedgerAnchor, +} from 'gradido-blockchain-js' +import * as v from 'valibot' +import { CONFIG } from '../../config' +import { deriveFromSeed } from '../../data/deriveKeyPair' +import { Hex32, hex32Schema } from '../../schemas/typeGuard.schema' +import { + AUF_ACCOUNT_DERIVATION_INDEX, + GMW_ACCOUNT_DERIVATION_INDEX, + hardenDerivationIndex, +} from '../../utils/derivationHelper' +import { toFolderName } from '../../utils/filesystem' +import { addToBlockchain } from './blockchain' +import { Context } from './Context' +import { Balance } from './data/Balance' +import { + loadAdminUsersCache, + loadCommunities, + loadContributionLinkModeratorCache, +} from './database' +import { CommunityContext } from './valibot.schema' + +export async function bootstrap(): Promise { + const context = await Context.create() + context.communities = await bootstrapCommunities(context) + await Promise.all([ + loadContributionLinkModeratorCache(context.db), + loadAdminUsersCache(context.db), + ]) + return context +} + +async function bootstrapCommunities(context: Context): Promise> { + const communities = new Map() + const communitiesDb = await loadCommunities(context.db) + + for (const communityDb of communitiesDb) { + const communityId = communityDb.communityUuid + const blockchain = InMemoryBlockchainProvider.getInstance().getBlockchain(communityId) + if (!blockchain) { + throw new Error(`Couldn't create Blockchain for community ${communityId}`) + } + context.logger.info(`Blockchain for community '${communityId}' created`) + let seed: Hex32 + if (!communityDb.foreign) { + seed = v.parse(hex32Schema, CONFIG.HOME_COMMUNITY_SEED.convertToHex()) + } else { + seed = v.parse(hex32Schema, randomBytes(32).toString('hex')) + } + + let creationDate = communityDb.creationDate + if (communityDb.userMinCreatedAt && communityDb.userMinCreatedAt < communityDb.creationDate) { + // create community root transaction 1 minute before first user + creationDate = new Date(new Date(communityDb.userMinCreatedAt).getTime() - 1000 * 60) + } + const communityKeyPair = deriveFromSeed(seed) + const gmwKeyPair = communityKeyPair.deriveChild( + hardenDerivationIndex(GMW_ACCOUNT_DERIVATION_INDEX), + ) + const aufKeyPair = communityKeyPair.deriveChild( + hardenDerivationIndex(AUF_ACCOUNT_DERIVATION_INDEX), + ) + if (!communityKeyPair || !gmwKeyPair || !aufKeyPair) { + throw new Error( + `Error on creating key pair for community ${JSON.stringify(communityDb, null, 2)}`, + ) + } + const builder = new GradidoTransactionBuilder() + builder + .setCreatedAt(creationDate) + .setSenderCommunity(communityId) + .setCommunityRoot( + communityKeyPair.getPublicKey(), + gmwKeyPair.getPublicKey(), + aufKeyPair.getPublicKey(), + ) + .sign(communityKeyPair) + + const communityContext: CommunityContext = { + communityId, + foreign: communityDb.foreign, + blockchain, + keyPair: communityKeyPair, + folder: toFolderName(communityId), + gmwBalance: new Balance(gmwKeyPair.getPublicKey()!, communityId), + aufBalance: new Balance(aufKeyPair.getPublicKey()!, communityId), + } + communities.set(communityId, communityContext) + const accountBalances = new AccountBalances() + accountBalances.add(communityContext.aufBalance.getAccountBalance()) + accountBalances.add(communityContext.gmwBalance.getAccountBalance()) + addToBlockchain( + builder.build(), + blockchain, + new LedgerAnchor(communityDb.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_COMMUNITY_ID), + accountBalances, + ) + } + return communities +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/Balance.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/Balance.ts new file mode 100644 index 000000000..d140f1c93 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/Balance.ts @@ -0,0 +1,117 @@ +import Decimal from 'decimal.js-light' +import { AccountBalance, GradidoUnit, MemoryBlockPtr } from 'gradido-blockchain-js' +import { NegativeBalanceError } from '../errors' +import { legacyCalculateDecay } from '../utils' + +export class Balance { + private balance: GradidoUnit + private date: Date + private publicKey: MemoryBlockPtr + private communityId: string + + constructor(publicKey: MemoryBlockPtr, communityId: string) { + this.balance = new GradidoUnit(0) + this.date = new Date() + this.publicKey = publicKey + this.communityId = communityId + } + + static fromAccountBalance( + accountBalance: AccountBalance, + confirmedAt: Date, + communityId: string, + ): Balance { + const balance = new Balance(accountBalance.getPublicKey()!, communityId) + balance.update(accountBalance.getBalance(), confirmedAt) + return balance + } + + getBalance(): GradidoUnit { + return this.balance + } + + getDate(): Date { + return this.date + } + + updateLegacyDecay(amount: GradidoUnit, date: Date) { + // make sure to copy instead of referencing + const previousBalanceString = this.balance.toString() + const previousDate = new Date(this.date.getTime()) + + if (this.balance.equal(GradidoUnit.zero())) { + this.balance = amount + this.date = date + } else { + const decayedBalance = legacyCalculateDecay( + new Decimal(this.balance.toString()), + this.date, + date, + ).toDecimalPlaces(4, Decimal.ROUND_CEIL) + const newBalance = decayedBalance.add(new Decimal(amount.toString())) + this.balance = GradidoUnit.fromString(newBalance.toString()) + this.date = date + } + if (this.balance.lt(GradidoUnit.zero())) { + if (this.balance.lt(GradidoUnit.fromGradidoCent(100).negated())) { + const previousDecayedBalance = legacyCalculateDecay( + new Decimal(previousBalanceString), + previousDate, + date, + ) + throw new NegativeBalanceError( + `negative Gradido amount detected in Balance.updateLegacyDecay`, + previousBalanceString, + amount.toString(), + previousDecayedBalance.toString(), + ) + } else { + this.balance = GradidoUnit.zero() + } + } + } + + update(amount: GradidoUnit, date: Date) { + const previousBalance = new GradidoUnit(this.balance.toString()) + const previousDate = new Date(this.date.getTime()) + + if (this.balance.equal(GradidoUnit.zero())) { + this.balance = amount + this.date = date + } else { + this.balance = this.balance.calculateDecay(this.date, date).add(amount) + this.date = date + } + if (this.balance.lt(GradidoUnit.zero())) { + // ignore diffs less than a gradido cent + if (this.balance.lt(GradidoUnit.fromGradidoCent(100).negated())) { + const previousDecayedBalance = this.balance.calculateDecay(previousDate, date) + throw new NegativeBalanceError( + `negative Gradido amount detected in Balance.update`, + previousBalance.toString(), + amount.toString(), + previousDecayedBalance.toString(), + ) + } else { + this.balance = GradidoUnit.zero() + } + } + } + + getAccountBalance(): AccountBalance { + return new AccountBalance(this.publicKey, this.balance, this.communityId) + } + + toString(): string { + return JSON.stringify( + { + balance: this.balance.toString(), + date: this.date, + publicKey: this.publicKey.convertToHex(), + communityId: this.communityId, + }, + null, + 2, + ) + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/ContributionStatus.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/ContributionStatus.ts new file mode 100644 index 000000000..a4bbc55f3 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/ContributionStatus.ts @@ -0,0 +1,7 @@ +export enum ContributionStatus { + PENDING = 'PENDING', + DELETED = 'DELETED', + IN_PROGRESS = 'IN_PROGRESS', + DENIED = 'DENIED', + CONFIRMED = 'CONFIRMED', +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/TransactionTypeId.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/TransactionTypeId.ts similarity index 100% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/TransactionTypeId.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/TransactionTypeId.ts diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/keyPair.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/keyPair.ts similarity index 74% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/keyPair.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/keyPair.ts index b3432da80..bb00e7b87 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/keyPair.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/keyPair.ts @@ -1,13 +1,13 @@ import { KeyPairEd25519, MemoryBlock, MemoryBlockPtr } from 'gradido-blockchain-js' import { getLogger } from 'log4js' -import { KeyPairCacheManager } from '../../cache/KeyPairCacheManager' -import { CONFIG } from '../../config' -import { LOG4JS_BASE_CATEGORY } from '../../config/const' -import { KeyPairIdentifierLogic } from '../../data/KeyPairIdentifier.logic' -import { AccountKeyPairRole } from '../../interactions/resolveKeyPair/AccountKeyPair.role' -import { UserKeyPairRole } from '../../interactions/resolveKeyPair/UserKeyPair.role' -import { HieroId } from '../../schemas/typeGuard.schema' -import { CommunityDb, UserDb } from './valibot.schema' +import { KeyPairCacheManager } from '../../../cache/KeyPairCacheManager' +import { CONFIG } from '../../../config' +import { LOG4JS_BASE_CATEGORY } from '../../../config/const' +import { KeyPairIdentifierLogic } from '../../../data/KeyPairIdentifier.logic' +import { AccountKeyPairRole } from '../../../interactions/resolveKeyPair/AccountKeyPair.role' +import { UserKeyPairRole } from '../../../interactions/resolveKeyPair/UserKeyPair.role' +import { HieroId } from '../../../schemas/typeGuard.schema' +import { CommunityDb, UserDb } from '../valibot.schema' const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.keyPair`) @@ -30,7 +30,10 @@ export function generateKeyPairCommunity( if (!keyPair) { throw new Error(`Couldn't create key pair for community ${community.communityUuid}`) } - const communityKeyPairKey = new KeyPairIdentifierLogic({ communityTopicId: topicId }).getKey() + const communityKeyPairKey = new KeyPairIdentifierLogic({ + communityTopicId: topicId, + communityId: community.communityUuid, + }).getKey() cache.addKeyPair(communityKeyPairKey, keyPair) logger.info(`Community Key Pair added with key: ${communityKeyPairKey}`) } @@ -44,6 +47,7 @@ export async function generateKeyPairUserAccount( const userKeyPairRole = new UserKeyPairRole(user.gradidoId, communityKeyPair) const userKeyPairKey = new KeyPairIdentifierLogic({ communityTopicId: communityTopicId, + communityId: user.communityUuid, account: { userUuid: user.gradidoId, accountNr: 0, @@ -56,6 +60,7 @@ export async function generateKeyPairUserAccount( const accountKeyPairRole = new AccountKeyPairRole(1, userKeyPair) const accountKeyPairKey = new KeyPairIdentifierLogic({ communityTopicId: communityTopicId, + communityId: user.communityUuid, account: { userUuid: user.gradidoId, accountNr: 1, diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/database.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/database.ts new file mode 100644 index 000000000..1dccb7609 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/database.ts @@ -0,0 +1,65 @@ +import { and, asc, eq, isNotNull, isNull, or, sql } from 'drizzle-orm' +import { MySql2Database } from 'drizzle-orm/mysql2' +import * as v from 'valibot' +import { communitiesTable, eventsTable, userRolesTable, usersTable } from './drizzle.schema' +import { CommunityDb, communityDbSchema, UserDb, userDbSchema } from './valibot.schema' + +export const contributionLinkModerators = new Map() +export const adminUsers = new Map() + +export async function loadContributionLinkModeratorCache(db: MySql2Database): Promise { + const result = await db + .select({ + event: eventsTable, + user: usersTable, + }) + .from(eventsTable) + .innerJoin(usersTable, eq(eventsTable.actingUserId, usersTable.id)) + .where(eq(eventsTable.type, 'ADMIN_CONTRIBUTION_LINK_CREATE')) + .orderBy(asc(eventsTable.id)) + + result.map((row: any) => { + contributionLinkModerators.set( + row.event.involvedContributionLinkId, + v.parse(userDbSchema, row.user), + ) + }) +} + +export async function loadAdminUsersCache(db: MySql2Database): Promise { + const result = await db + .select({ + user: usersTable, + }) + .from(userRolesTable) + .where(eq(userRolesTable.role, 'ADMIN')) + .innerJoin(usersTable, eq(userRolesTable.userId, usersTable.id)) + + result.map((row: any) => { + adminUsers.set(row.gradidoId, v.parse(userDbSchema, row.user)) + }) +} + +// queries +export async function loadCommunities(db: MySql2Database): Promise { + const result = await db + .select({ + id: communitiesTable.id, + foreign: communitiesTable.foreign, + communityUuid: communitiesTable.communityUuid, + name: communitiesTable.name, + creationDate: communitiesTable.creationDate, + userMinCreatedAt: sql`MIN(${usersTable.createdAt})`, + }) + .from(communitiesTable) + .innerJoin(usersTable, eq(communitiesTable.communityUuid, usersTable.communityUuid)) + .where( + and(isNotNull(communitiesTable.communityUuid), sql`${usersTable.createdAt} > '2000-01-01'`), + ) + .orderBy(asc(communitiesTable.id)) + .groupBy(communitiesTable.communityUuid) + + return result.map((row: any) => { + return v.parse(communityDbSchema, row) + }) +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/drizzle.schema.ts similarity index 55% rename from dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts rename to dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/drizzle.schema.ts index e08231f4d..71c5bd08a 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/drizzle.schema.ts @@ -24,12 +24,34 @@ export const communitiesTable = mysqlTable( (table) => [unique('uuid_key').on(table.communityUuid)], ) +export const contributionsTable = mysqlTable('contributions', { + id: int().autoincrement().notNull(), + userId: int('user_id').default(sql`NULL`), + contributionDate: datetime('contribution_date', { mode: 'string' }).default(sql`NULL`), + memo: varchar({ length: 512 }).notNull(), + amount: decimal({ precision: 40, scale: 20 }).notNull(), + contributionLinkId: int('contribution_link_id').default(sql`NULL`), + confirmedBy: int('confirmed_by').default(sql`NULL`), + confirmedAt: datetime('confirmed_at', { mode: 'string' }).default(sql`NULL`), + contributionStatus: varchar('contribution_status', { length: 12 }).default("'PENDING'").notNull(), + transactionId: int('transaction_id').default(sql`NULL`), +}) + +export const eventsTable = mysqlTable('events', { + id: int().autoincrement().notNull(), + type: varchar({ length: 100 }).notNull(), + actingUserId: int('acting_user_id').notNull(), + involvedContributionLinkId: int('involved_contribution_link_id').default(sql`NULL`), +}) + export const usersTable = mysqlTable( 'users', { id: int().autoincrement().notNull(), + foreign: tinyint().default(0).notNull(), gradidoId: char('gradido_id', { length: 36 }).notNull(), communityUuid: varchar('community_uuid', { length: 36 }).default(sql`NULL`), + deletedAt: datetime('deleted_at', { mode: 'string', fsp: 3 }).default(sql`NULL`), createdAt: datetime('created_at', { mode: 'string', fsp: 3 }) .default(sql`current_timestamp(3)`) .notNull(), @@ -37,6 +59,16 @@ export const usersTable = mysqlTable( (table) => [unique('uuid_key').on(table.gradidoId, table.communityUuid)], ) +export const userRolesTable = mysqlTable( + 'user_roles', + { + id: int().autoincrement().notNull(), + userId: int('user_id').notNull(), + role: varchar({ length: 40 }).notNull(), + }, + (table) => [index('user_id').on(table.userId)], +) + export const transactionsTable = mysqlTable( 'transactions', { @@ -44,6 +76,7 @@ export const transactionsTable = mysqlTable( typeId: int('type_id').default(sql`NULL`), transactionLinkId: int('transaction_link_id').default(sql`NULL`), amount: decimal({ precision: 40, scale: 20 }).default(sql`NULL`), + balance: decimal({ precision: 40, scale: 20 }).default(sql`NULL`), balanceDate: datetime('balance_date', { mode: 'string', fsp: 3 }) .default(sql`current_timestamp(3)`) .notNull(), @@ -51,6 +84,9 @@ export const transactionsTable = mysqlTable( creationDate: datetime('creation_date', { mode: 'string', fsp: 3 }).default(sql`NULL`), userId: int('user_id').notNull(), linkedUserId: int('linked_user_id').default(sql`NULL`), + linkedUserCommunityUuid: char('linked_user_community_uuid', { length: 36 }).default(sql`NULL`), + linkedUserGradidoId: char('linked_user_gradido_id', { length: 36 }).default(sql`NULL`), + linkedTransactionId: int('linked_transaction_id').default(sql`NULL`), }, (table) => [index('user_id').on(table.userId)], ) @@ -59,9 +95,12 @@ export const transactionLinksTable = mysqlTable('transaction_links', { id: int().autoincrement().notNull(), userId: int().notNull(), amount: decimal({ precision: 40, scale: 20 }).notNull(), + holdAvailableAmount: decimal('hold_available_amount', { precision: 40, scale: 20 }).notNull(), memo: varchar({ length: 255 }).notNull(), code: varchar({ length: 24 }).notNull(), createdAt: datetime({ mode: 'string' }).notNull(), deletedAt: datetime({ mode: 'string' }).default(sql`NULL`), validUntil: datetime({ mode: 'string' }).notNull(), + redeemedAt: datetime({ mode: 'string' }).default(sql`NULL`), + redeemedBy: int().default(sql`NULL`), }) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/errors.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/errors.ts new file mode 100644 index 000000000..8e09c6995 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/errors.ts @@ -0,0 +1,67 @@ +import * as v from 'valibot' + +export class NotEnoughGradidoBalanceError extends Error { + constructor( + public needed: number, + public exist: number, + ) { + super( + `Not enough Gradido Balance for send coins, needed: ${needed} Gradido, exist: ${exist} Gradido`, + ) + this.name = 'NotEnoughGradidoBalanceError' + } +} + +export class DatabaseError extends Error { + constructor(message: string, rows: any, originalError: Error) { + const parts: string[] = [`DatabaseError in ${message}`] + + // Valibot-specific + if (originalError instanceof v.ValiError) { + const flattened = v.flatten(originalError.issues) + parts.push('Validation errors:') + parts.push(JSON.stringify(flattened, null, 2)) + } else { + parts.push(`Original error: ${originalError.message}`) + } + + parts.push('Rows:') + parts.push(JSON.stringify(rows, null, 2)) + + super(parts.join('\n\n')) + + this.name = 'DatabaseError' + this.cause = originalError + } +} + +export class BlockchainError extends Error { + constructor(message: string, item: any, originalError: Error) { + const parts: string[] = [`BlockchainError in ${message}`] + + parts.push(`Original error: ${originalError.message}`) + parts.push('Item:') + parts.push(JSON.stringify(item, null, 2)) + + super(parts.join('\n\n')) + + this.name = 'BlockchainError' + this.cause = originalError + } +} + +export class NegativeBalanceError extends Error { + constructor( + message: string, + previousBalanceString: string, + amount: string, + previousDecayedBalance: string, + ) { + const parts: string[] = [`NegativeBalanceError in ${message}`] + parts.push(`Previous balance: ${previousBalanceString}`) + parts.push(`Amount: ${amount}`) + parts.push(`Previous decayed balance: ${previousDecayedBalance}`) + super(parts.join('\n')) + this.name = 'NegativeBalanceError' + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts new file mode 100644 index 000000000..1576f9770 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts @@ -0,0 +1,62 @@ +import { Filter, Profiler, ThreadingPolicy_Half, verifySignatures } from 'gradido-blockchain-js' +import { onShutdown } from '../../../../shared/src/helper/onShutdown' +import { exportAllCommunities } from './binaryExport' +import { bootstrap } from './bootstrap' +import { syncDbWithBlockchainContext } from './interaction/syncDbWithBlockchain/syncDbWithBlockchain.context' + +// import { hello } from '../../../zig/hello.zig' + +const BATCH_SIZE = 1000 + +async function main() { + // hello() + // return + // prepare in memory blockchains + const context = await bootstrap() + onShutdown(async (reason, error) => { + context.logger.info(`shutdown reason: ${reason}`) + if (error) { + context.logger.error(error) + } + }) + + // synchronize to in memory blockchain + try { + await syncDbWithBlockchainContext(context, BATCH_SIZE) + } catch (e) { + context.logger.error(e) + //context.logBlogchain(v.parse(uuidv4Schema, 'e70da33e-5976-4767-bade-aa4e4fa1c01a')) + } + + const timeUsed = new Profiler() + // bulk verify transaction signatures + for (const communityContext of context.communities.values()) { + // verifySignatures(Filter.ALL_TRANSACTIONS, ThreadingPolicy_Half) + const result = verifySignatures( + Filter.ALL_TRANSACTIONS, + communityContext.communityId, + ThreadingPolicy_Half, + ) + if (!result.isEmpty()) { + throw new Error( + `Verification of signatures failed for community ${communityContext.communityId}`, + ) + } + } + context.logger.info(`verified in ${timeUsed.string()}`) + + // write as binary file for GradidoNode + exportAllCommunities(context, BATCH_SIZE) + + // log runtime statistics + context.logRuntimeStatistics() + + // needed because of shutdown handler (TODO: fix shutdown handler) + process.exit(0) +} + +main().catch((e) => { + // biome-ignore lint/suspicious/noConsole: maybe logger isn't initialized here + console.error(e) + process.exit(1) +}) diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts new file mode 100644 index 000000000..9a2cfe660 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts @@ -0,0 +1,170 @@ +import { + AccountBalances, + Filter, + GradidoTransactionBuilder, + InMemoryBlockchain, + KeyPairEd25519, + MemoryBlockPtr, + Profiler, + SearchDirection_DESC, +} from 'gradido-blockchain-js' +import { getLogger, Logger } from 'log4js' +import { LOG4JS_BASE_CATEGORY } from '../../../../config/const' +import { deriveFromKeyPairAndIndex, deriveFromKeyPairAndUuid } from '../../../../data/deriveKeyPair' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' +import { Context } from '../../Context' +import { Balance } from '../../data/Balance' +import { CommunityContext } from '../../valibot.schema' + +export type IndexType = { + date: Date + id: number +} +export let nanosBalanceForUser = 0 +const lastBalanceOfUserTimeUsed = new Profiler() + +export abstract class AbstractSyncRole { + private items: ItemType[] = [] + protected lastIndex: IndexType = { date: new Date(0), id: 0 } + protected logger: Logger + protected transactionBuilder: GradidoTransactionBuilder + protected accountBalances: AccountBalances + + constructor(protected readonly context: Context) { + this.logger = getLogger( + `${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5.interaction.syncDbWithBlockchain`, + ) + this.transactionBuilder = new GradidoTransactionBuilder() + this.accountBalances = new AccountBalances() + } + + getAccountKeyPair(communityContext: CommunityContext, gradidoId: Uuidv4): KeyPairEd25519 { + return this.context.cache.getKeyPairSync(gradidoId, () => { + return deriveFromKeyPairAndIndex( + deriveFromKeyPairAndUuid(communityContext.keyPair, gradidoId), + 1, + ) + }) + } + + getLastBalanceForUser( + publicKey: MemoryBlockPtr, + blockchain: InMemoryBlockchain, + communityId: string, + ): Balance { + lastBalanceOfUserTimeUsed.reset() + if (publicKey.isEmpty()) { + throw new Error('publicKey is empty') + } + const f = Filter.lastBalanceFor(publicKey) + f.setCommunityId(communityId) + const lastSenderTransaction = blockchain.findOne(f) + if (!lastSenderTransaction) { + return new Balance(publicKey, communityId) + } + const lastConfirmedTransaction = lastSenderTransaction.getConfirmedTransaction() + if (!lastConfirmedTransaction) { + throw new Error( + `invalid transaction, getConfirmedTransaction call failed for transaction nr: ${lastSenderTransaction.getTransactionNr()}`, + ) + } + const senderLastAccountBalance = lastConfirmedTransaction.getAccountBalance( + publicKey, + communityId, + ) + if (!senderLastAccountBalance) { + return new Balance(publicKey, communityId) + } + const result = Balance.fromAccountBalance( + senderLastAccountBalance, + lastConfirmedTransaction.getConfirmedAt().getDate(), + communityId, + ) + nanosBalanceForUser += lastBalanceOfUserTimeUsed.nanos() + return result + } + + logLastBalanceChangingTransactions( + publicKey: MemoryBlockPtr, + blockchain: InMemoryBlockchain, + transactionCount: number = 5, + ) { + if (!this.context.logger.isDebugEnabled()) { + return + } + const f = new Filter() + f.updatedBalancePublicKey = publicKey + f.searchDirection = SearchDirection_DESC + f.pagination.size = transactionCount + const lastTransactions = blockchain.findAll(f) + for (let i = lastTransactions.size() - 1; i >= 0; i--) { + const tx = lastTransactions.get(i) + this.context.logger.debug(`${i}: ${tx?.getConfirmedTransaction()!.toJson(true)}`) + } + } + + abstract getDate(): Date + // for using seek rather than offset pagination approach + abstract getLastIndex(): IndexType + abstract loadFromDb(lastIndex: IndexType, count: number): Promise + abstract pushToBlockchain(item: ItemType): void + abstract itemTypeName(): string + abstract getCommunityUuids(): Uuidv4[] + + // return count of new loaded items + async ensureFilled(batchSize: number): Promise { + if (this.items.length === 0) { + let timeUsed: Profiler | undefined + if (this.logger.isDebugEnabled()) { + timeUsed = new Profiler() + } + this.items = await this.loadFromDb(this.lastIndex, batchSize) + if (this.length > 0) { + this.lastIndex = this.getLastIndex() + if (timeUsed) { + this.logger.debug( + `${timeUsed.string()} for loading ${this.items.length} ${this.itemTypeName()} from db`, + ) + } + } + return this.items.length + } + return 0 + } + + toBlockchain(): void { + if (this.isEmpty()) { + throw new Error(`[toBlockchain] No items, please call this only if isEmpty returns false`) + } + this.pushToBlockchain(this.shift()) + } + + peek(): ItemType { + if (this.isEmpty()) { + throw new Error(`[peek] No items, please call this only if isEmpty returns false`) + } + return this.items[0] + } + peekLast(): ItemType { + if (this.isEmpty()) { + throw new Error(`[peekLast] No items, please call this only if isEmpty returns false`) + } + return this.items[this.items.length - 1] + } + + shift(): ItemType { + const item = this.items.shift() + if (!item) { + throw new Error(`[shift] No items, shift return undefined`) + } + return item + } + + get length(): number { + return this.items.length + } + + isEmpty(): boolean { + return this.items.length === 0 + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts new file mode 100644 index 000000000..4e149f583 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts @@ -0,0 +1,69 @@ +import { and, asc, eq, gt, isNotNull, or } from 'drizzle-orm' +import * as v from 'valibot' +import { Context } from '../../Context' +import { ContributionStatus } from '../../data/ContributionStatus' +import { contributionLinkModerators } from '../../database' +import { contributionsTable, usersTable } from '../../drizzle.schema' +import { DatabaseError } from '../../errors' +import { toMysqlDateTime } from '../../utils' +import { CreationTransactionDb, creationTransactionDbSchema } from '../../valibot.schema' +import { IndexType } from './AbstractSync.role' +import { CreationsSyncRole } from './CreationsSync.role' + +export class ContributionLinkTransactionSyncRole extends CreationsSyncRole { + constructor(readonly context: Context) { + super(context) + } + itemTypeName(): string { + return 'contributionLinkTransaction' + } + + async loadFromDb(lastIndex: IndexType, count: number): Promise { + const result = await this.context.db + .select({ + contribution: contributionsTable, + user: usersTable, + }) + .from(contributionsTable) + .where( + and( + isNotNull(contributionsTable.contributionLinkId), + eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED), + or( + gt(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), + and( + eq(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), + gt(contributionsTable.transactionId, lastIndex.id), + ), + ), + ), + ) + .innerJoin(usersTable, eq(contributionsTable.userId, usersTable.id)) + .orderBy(asc(contributionsTable.confirmedAt), asc(contributionsTable.transactionId)) + .limit(count) + + const verifiedCreationTransactions: CreationTransactionDb[] = [] + for (const row of result) { + if (!row.contribution.contributionLinkId) { + throw new Error( + `expect contributionLinkId to be set: ${JSON.stringify(row.contribution, null, 2)}`, + ) + } + const item = { + ...row.contribution, + user: row.user, + confirmedByUser: contributionLinkModerators.get(row.contribution.contributionLinkId), + } + if (!item.confirmedByUser || item.userId === item.confirmedByUser.id) { + this.context.logger.warn(`skipped Contribution Link Transaction ${row.contribution.id}`) + continue + } + try { + verifiedCreationTransactions.push(v.parse(creationTransactionDbSchema, item)) + } catch (e) { + throw new DatabaseError('load contributions with contribution link id', item, e as Error) + } + } + return verifiedCreationTransactions + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts new file mode 100644 index 000000000..683953a06 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts @@ -0,0 +1,181 @@ +import { and, asc, eq, gt, isNull, or } from 'drizzle-orm' +import { alias } from 'drizzle-orm/mysql-core' +import { + AccountBalances, + AuthenticatedEncryption, + EncryptedMemo, + Filter, + GradidoTransactionBuilder, + KeyPairEd25519, + LedgerAnchor, + MemoryBlockPtr, + SearchDirection_DESC, + TransactionType_CREATION, + TransferAmount, +} from 'gradido-blockchain-js' +import * as v from 'valibot' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' +import { addToBlockchain } from '../../blockchain' +import { Context } from '../../Context' +import { ContributionStatus } from '../../data/ContributionStatus' +import { contributionsTable, usersTable } from '../../drizzle.schema' +import { BlockchainError, DatabaseError } from '../../errors' +import { toMysqlDateTime } from '../../utils' +import { + CommunityContext, + CreationTransactionDb, + creationTransactionDbSchema, +} from '../../valibot.schema' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' + +export class CreationsSyncRole extends AbstractSyncRole { + constructor(context: Context) { + super(context) + this.accountBalances.reserve(3) + } + + getDate(): Date { + return this.peek().confirmedAt + } + getCommunityUuids(): Uuidv4[] { + return [this.peek().user.communityUuid] + } + + getLastIndex(): IndexType { + const lastItem = this.peekLast() + return { date: lastItem.confirmedAt, id: lastItem.transactionId } + } + + itemTypeName(): string { + return 'creationTransactions' + } + + async loadFromDb(lastIndex: IndexType, count: number): Promise { + const confirmedByUsers = alias(usersTable, 'confirmedByUser') + const result = await this.context.db + .select({ + contribution: contributionsTable, + user: usersTable, + confirmedByUser: confirmedByUsers, + }) + .from(contributionsTable) + .where( + and( + isNull(contributionsTable.contributionLinkId), + eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED), + or( + gt(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), + and( + eq(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)), + gt(contributionsTable.transactionId, lastIndex.id), + ), + ), + ), + ) + .innerJoin(usersTable, eq(contributionsTable.userId, usersTable.id)) + .innerJoin(confirmedByUsers, eq(contributionsTable.confirmedBy, confirmedByUsers.id)) + .orderBy(asc(contributionsTable.confirmedAt), asc(contributionsTable.transactionId)) + .limit(count) + + return result.map((row) => { + const item = { + ...row.contribution, + user: row.user, + confirmedByUser: row.confirmedByUser, + } + try { + return v.parse(creationTransactionDbSchema, item) + } catch (e) { + throw new DatabaseError('loadCreations', item, e as Error) + } + }) + } + + buildTransaction( + item: CreationTransactionDb, + communityContext: CommunityContext, + recipientKeyPair: KeyPairEd25519, + signerKeyPair: KeyPairEd25519, + ): GradidoTransactionBuilder { + return this.transactionBuilder + .setCreatedAt(item.confirmedAt) + .setRecipientCommunity(communityContext.communityId) + .addMemo( + new EncryptedMemo( + item.memo, + new AuthenticatedEncryption(communityContext.keyPair), + new AuthenticatedEncryption(recipientKeyPair), + ), + ) + .setTransactionCreation( + new TransferAmount( + recipientKeyPair.getPublicKey(), + item.amount, + communityContext.communityId, + ), + item.contributionDate, + ) + .sign(signerKeyPair) + } + + calculateAccountBalances( + item: CreationTransactionDb, + communityContext: CommunityContext, + recipientPublicKey: MemoryBlockPtr, + ): AccountBalances { + this.accountBalances.clear() + const balance = this.getLastBalanceForUser( + recipientPublicKey, + communityContext.blockchain, + communityContext.communityId, + ) + + // calculate decay since last balance with legacy calculation method + balance.updateLegacyDecay(item.amount, item.confirmedAt) + communityContext.aufBalance.updateLegacyDecay(item.amount, item.confirmedAt) + communityContext.gmwBalance.updateLegacyDecay(item.amount, item.confirmedAt) + + this.accountBalances.add(balance.getAccountBalance()) + this.accountBalances.add(communityContext.aufBalance.getAccountBalance()) + this.accountBalances.add(communityContext.gmwBalance.getAccountBalance()) + return this.accountBalances + } + + pushToBlockchain(item: CreationTransactionDb): void { + const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) + const blockchain = communityContext.blockchain + if (item.confirmedByUser.communityUuid !== item.user.communityUuid) { + throw new Error( + `contribution was confirmed from other community: ${JSON.stringify(item, null, 2)}`, + ) + } + + const recipientKeyPair = this.getAccountKeyPair(communityContext, item.user.gradidoId) + const recipientPublicKey = recipientKeyPair.getPublicKey() + const signerKeyPair = this.getAccountKeyPair(communityContext, item.confirmedByUser.gradidoId) + if (!recipientKeyPair || !signerKeyPair || !recipientPublicKey) { + throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`) + } + + try { + addToBlockchain( + this.buildTransaction(item, communityContext, recipientKeyPair, signerKeyPair).build(), + blockchain, + new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_CONTRIBUTION_ID), + this.calculateAccountBalances(item, communityContext, recipientPublicKey), + ) + } catch (e) { + const f = new Filter() + f.transactionType = TransactionType_CREATION + f.searchDirection = SearchDirection_DESC + f.pagination.size = 1 + const lastContribution = blockchain.findOne(f) + if (lastContribution) { + this.context.logger.warn( + `last contribution: ${lastContribution.getConfirmedTransaction()?.toJson(true)}`, + ) + } + throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) + } + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts new file mode 100644 index 000000000..05aeaa5a8 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts @@ -0,0 +1,205 @@ +import { and, asc, eq, gt, isNotNull, lt, or } from 'drizzle-orm' +import { + AccountBalance, + AccountBalances, + Filter, + GradidoDeferredTransfer, + GradidoTransactionBuilder, + GradidoTransfer, + GradidoUnit, + KeyPairEd25519, + LedgerAnchor, + MemoryBlockPtr, + TransferAmount, +} from 'gradido-blockchain-js' +import * as v from 'valibot' +import { deriveFromCode } from '../../../../data/deriveKeyPair' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' +import { addToBlockchain } from '../../blockchain' +import { Context } from '../../Context' +import { Balance } from '../../data/Balance' +import { transactionLinksTable, usersTable } from '../../drizzle.schema' +import { BlockchainError, DatabaseError } from '../../errors' +import { toMysqlDateTime } from '../../utils' +import { + CommunityContext, + DeletedTransactionLinkDb, + deletedTransactionLinKDbSchema, +} from '../../valibot.schema' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' + +export class DeletedTransactionLinksSyncRole extends AbstractSyncRole { + constructor(context: Context) { + super(context) + this.accountBalances.reserve(2) + } + + getDate(): Date { + return this.peek().deletedAt + } + getCommunityUuids(): Uuidv4[] { + return [this.peek().user.communityUuid] + } + + getLastIndex(): IndexType { + const lastItem = this.peekLast() + return { date: lastItem.deletedAt, id: lastItem.id } + } + + itemTypeName(): string { + return 'deletedTransactionLinks' + } + + async loadFromDb(lastIndex: IndexType, count: number): Promise { + const result = await this.context.db + .select({ + transactionLink: transactionLinksTable, + user: usersTable, + }) + .from(transactionLinksTable) + .where( + and( + isNotNull(transactionLinksTable.deletedAt), + lt(transactionLinksTable.deletedAt, transactionLinksTable.validUntil), + or( + gt(transactionLinksTable.deletedAt, toMysqlDateTime(lastIndex.date)), + and( + eq(transactionLinksTable.deletedAt, toMysqlDateTime(lastIndex.date)), + gt(transactionLinksTable.id, lastIndex.id), + ), + ), + ), + ) + .innerJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id)) + .orderBy(asc(transactionLinksTable.deletedAt), asc(transactionLinksTable.id)) + .limit(count) + + return result.map((row) => { + const item = { + ...row.transactionLink, + user: row.user, + } + try { + return v.parse(deletedTransactionLinKDbSchema, item) + } catch (e) { + throw new DatabaseError('loadDeletedTransactionLinks', item, e as Error) + } + }) + } + + buildTransaction( + communityContext: CommunityContext, + item: DeletedTransactionLinkDb, + linkFundingTransactionNr: number, + restAmount: GradidoUnit, + senderKeyPair: KeyPairEd25519, + linkFundingPublicKey: MemoryBlockPtr, + ): GradidoTransactionBuilder { + return this.transactionBuilder + .setCreatedAt(item.deletedAt) + .setSenderCommunity(communityContext.communityId) + .setRedeemDeferredTransfer( + linkFundingTransactionNr, + new GradidoTransfer( + new TransferAmount( + senderKeyPair.getPublicKey(), + restAmount, + communityContext.communityId, + ), + linkFundingPublicKey, + ), + ) + .sign(senderKeyPair) + } + + calculateBalances( + item: DeletedTransactionLinkDb, + fundingTransaction: GradidoDeferredTransfer, + senderLastBalance: Balance, + communityContext: CommunityContext, + senderPublicKey: MemoryBlockPtr, + ): AccountBalances { + this.accountBalances.clear() + + const fundingUserLastBalance = this.getLastBalanceForUser( + fundingTransaction.getSenderPublicKey()!, + communityContext.blockchain, + communityContext.communityId, + ) + fundingUserLastBalance.updateLegacyDecay(senderLastBalance.getBalance(), item.deletedAt) + + // account of link is set to zero, gdd will be send back to initiator + this.accountBalances.add( + new AccountBalance(senderPublicKey, GradidoUnit.zero(), communityContext.communityId), + ) + this.accountBalances.add(fundingUserLastBalance.getAccountBalance()) + return this.accountBalances + } + + pushToBlockchain(item: DeletedTransactionLinkDb): void { + const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) + const blockchain = communityContext.blockchain + + const senderKeyPair = deriveFromCode(item.code) + const senderPublicKey = senderKeyPair.getPublicKey() + + if (!senderKeyPair || !senderPublicKey) { + throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`) + } + + const transaction = blockchain.findOne(Filter.lastBalanceFor(senderPublicKey)) + if (!transaction) { + throw new Error(`expect transaction for code: ${item.code}`) + } + // should be funding transaction + if (!transaction.isDeferredTransfer()) { + throw new Error( + `expect funding transaction: ${transaction.getConfirmedTransaction()?.toJson(true)}`, + ) + } + const body = transaction + .getConfirmedTransaction() + ?.getGradidoTransaction() + ?.getTransactionBody() + const deferredTransfer = body?.getDeferredTransfer() + if (!deferredTransfer || !deferredTransfer.getRecipientPublicKey()?.equal(senderPublicKey)) { + throw new Error( + `expect funding transaction to belong to code: ${item.code}: ${transaction.getConfirmedTransaction()?.toJson(true)}`, + ) + } + const linkFundingPublicKey = deferredTransfer.getSenderPublicKey() + if (!linkFundingPublicKey) { + throw new Error(`missing sender public key of transaction link founder`) + } + const senderLastBalance = this.getLastBalanceForUser( + senderPublicKey, + communityContext.blockchain, + communityContext.communityId, + ) + senderLastBalance.updateLegacyDecay(GradidoUnit.zero(), item.deletedAt) + + try { + addToBlockchain( + this.buildTransaction( + communityContext, + item, + transaction.getTransactionNr(), + senderLastBalance.getBalance(), + senderKeyPair, + linkFundingPublicKey, + ).build(), + blockchain, + new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_LINK_ID), + this.calculateBalances( + item, + deferredTransfer, + senderLastBalance, + communityContext, + senderPublicKey, + ), + ) + } catch (e) { + throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) + } + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts new file mode 100644 index 000000000..d85921a4b --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts @@ -0,0 +1,186 @@ +import { and, asc, eq, gt, isNotNull, isNull, or } from 'drizzle-orm' +import { alias } from 'drizzle-orm/mysql-core' +import { + AccountBalances, + AuthenticatedEncryption, + EncryptedMemo, + GradidoTransactionBuilder, + KeyPairEd25519, + LedgerAnchor, + MemoryBlockPtr, + TransferAmount, +} from 'gradido-blockchain-js' +import * as v from 'valibot' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' +import { addToBlockchain } from '../../blockchain' +import { Context } from '../../Context' +import { TransactionTypeId } from '../../data/TransactionTypeId' +import { transactionsTable, usersTable } from '../../drizzle.schema' +import { + BlockchainError, + DatabaseError, + NegativeBalanceError, + NotEnoughGradidoBalanceError, +} from '../../errors' +import { toMysqlDateTime } from '../../utils' +import { CommunityContext, TransactionDb, transactionDbSchema } from '../../valibot.schema' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' + +export class LocalTransactionsSyncRole extends AbstractSyncRole { + constructor(context: Context) { + super(context) + this.accountBalances.reserve(2) + } + + getDate(): Date { + return this.peek().balanceDate + } + + getCommunityUuids(): Uuidv4[] { + return [this.peek().user.communityUuid] + } + + getLastIndex(): IndexType { + const lastItem = this.peekLast() + return { date: lastItem.balanceDate, id: lastItem.id } + } + + itemTypeName(): string { + return 'localTransactions' + } + + async loadFromDb(lastIndex: IndexType, count: number): Promise { + const linkedUsers = alias(usersTable, 'linkedUser') + const result = await this.context.db + .select({ + transaction: transactionsTable, + user: usersTable, + linkedUser: linkedUsers, + }) + .from(transactionsTable) + .where( + and( + eq(transactionsTable.typeId, TransactionTypeId.RECEIVE), + isNull(transactionsTable.transactionLinkId), + isNotNull(transactionsTable.linkedUserId), + eq(usersTable.foreign, 0), + eq(linkedUsers.foreign, 0), + or( + gt(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)), + and( + eq(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)), + gt(transactionsTable.id, lastIndex.id), + ), + ), + ), + ) + .innerJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) + .innerJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id)) + .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) + .limit(count) + + return result.map((row) => { + const item = { + ...row.transaction, + user: row.user, + linkedUser: row.linkedUser, + } + try { + return v.parse(transactionDbSchema, item) + } catch (e) { + throw new DatabaseError('loadLocalTransferTransactions', item, e as Error) + } + }) + } + + buildTransaction( + communityContext: CommunityContext, + item: TransactionDb, + senderKeyPair: KeyPairEd25519, + recipientKeyPair: KeyPairEd25519, + ): GradidoTransactionBuilder { + return this.transactionBuilder + .setCreatedAt(item.balanceDate) + .addMemo( + new EncryptedMemo( + item.memo, + new AuthenticatedEncryption(senderKeyPair), + new AuthenticatedEncryption(recipientKeyPair), + ), + ) + .setSenderCommunity(communityContext.communityId) + .setTransactionTransfer( + new TransferAmount(senderKeyPair.getPublicKey(), item.amount, communityContext.communityId), + recipientKeyPair.getPublicKey(), + ) + .sign(senderKeyPair) + } + + calculateBalances( + item: TransactionDb, + communityContext: CommunityContext, + senderPublicKey: MemoryBlockPtr, + recipientPublicKey: MemoryBlockPtr, + ): AccountBalances { + this.accountBalances.clear() + + const senderLastBalance = this.getLastBalanceForUser( + senderPublicKey, + communityContext.blockchain, + communityContext.communityId, + ) + const recipientLastBalance = this.getLastBalanceForUser( + recipientPublicKey, + communityContext.blockchain, + communityContext.communityId, + ) + + try { + senderLastBalance.updateLegacyDecay(item.amount.negated(), item.balanceDate) + } catch (e) { + if (e instanceof NegativeBalanceError) { + this.logLastBalanceChangingTransactions(senderPublicKey, communityContext.blockchain) + throw e + } + } + recipientLastBalance.updateLegacyDecay(item.amount, item.balanceDate) + + this.accountBalances.add(senderLastBalance.getAccountBalance()) + this.accountBalances.add(recipientLastBalance.getAccountBalance()) + return this.accountBalances + } + + pushToBlockchain(item: TransactionDb): void { + const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) + const blockchain = communityContext.blockchain + if (item.linkedUser.communityUuid !== item.user.communityUuid) { + throw new Error( + `transfer between user from different communities: ${JSON.stringify(item, null, 2)}`, + ) + } + + // I use the received transaction so user and linked user are swapped and user is recipient and linkedUser ist sender + const senderKeyPair = this.getAccountKeyPair(communityContext, item.linkedUser.gradidoId) + const senderPublicKey = senderKeyPair.getPublicKey() + const recipientKeyPair = this.getAccountKeyPair(communityContext, item.user.gradidoId) + const recipientPublicKey = recipientKeyPair.getPublicKey() + + if (!senderKeyPair || !senderPublicKey || !recipientKeyPair || !recipientPublicKey) { + throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`) + } + + try { + addToBlockchain( + this.buildTransaction(communityContext, item, senderKeyPair, recipientKeyPair).build(), + blockchain, + new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_ID), + this.calculateBalances(item, communityContext, senderPublicKey, recipientPublicKey), + ) + } catch (e) { + if (e instanceof NotEnoughGradidoBalanceError) { + this.logLastBalanceChangingTransactions(senderPublicKey, blockchain) + } + throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) + } + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts new file mode 100644 index 000000000..012a07b08 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts @@ -0,0 +1,230 @@ +import { and, asc, eq, gt, isNotNull, isNull, or } from 'drizzle-orm' +import { alias } from 'drizzle-orm/mysql-core' +import { + AccountBalance, + AccountBalances, + AuthenticatedEncryption, + EncryptedMemo, + Filter, + GradidoDeferredTransfer, + GradidoTransactionBuilder, + GradidoTransfer, + GradidoUnit, + KeyPairEd25519, + LedgerAnchor, + MemoryBlockPtr, + TransferAmount, +} from 'gradido-blockchain-js' +import * as v from 'valibot' +import { deriveFromCode } from '../../../../data/deriveKeyPair' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' +import { addToBlockchain } from '../../blockchain' +import { Context } from '../../Context' +import { transactionLinksTable, usersTable } from '../../drizzle.schema' +import { BlockchainError, DatabaseError } from '../../errors' +import { toMysqlDateTime } from '../../utils' +import { + CommunityContext, + RedeemedTransactionLinkDb, + redeemedTransactionLinkDbSchema, +} from '../../valibot.schema' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' + +export class RedeemTransactionLinksSyncRole extends AbstractSyncRole { + constructor(context: Context) { + super(context) + this.accountBalances.reserve(3) + } + + getDate(): Date { + return this.peek().redeemedAt + } + + getCommunityUuids(): Uuidv4[] { + return [this.peek().user.communityUuid] + } + + getLastIndex(): IndexType { + const lastItem = this.peekLast() + return { date: lastItem.redeemedAt, id: lastItem.id } + } + + itemTypeName(): string { + return 'redeemTransactionLinks' + } + + async loadFromDb(lastIndex: IndexType, count: number): Promise { + const redeemedByUser = alias(usersTable, 'redeemedByUser') + const result = await this.context.db + .select({ + transactionLink: transactionLinksTable, + user: usersTable, + redeemedBy: redeemedByUser, + }) + .from(transactionLinksTable) + .where( + and( + isNull(transactionLinksTable.deletedAt), + isNotNull(transactionLinksTable.redeemedAt), + eq(usersTable.foreign, 0), + eq(redeemedByUser.foreign, 0), + or( + gt(transactionLinksTable.redeemedAt, toMysqlDateTime(lastIndex.date)), + and( + eq(transactionLinksTable.redeemedAt, toMysqlDateTime(lastIndex.date)), + gt(transactionLinksTable.id, lastIndex.id), + ), + ), + ), + ) + .innerJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id)) + .innerJoin(redeemedByUser, eq(transactionLinksTable.redeemedBy, redeemedByUser.id)) + .orderBy(asc(transactionLinksTable.redeemedAt), asc(transactionLinksTable.id)) + .limit(count) + + return result.map((row) => { + const item = { + ...row.transactionLink, + redeemedBy: row.redeemedBy, + user: row.user, + } + try { + return v.parse(redeemedTransactionLinkDbSchema, item) + } catch (e) { + throw new DatabaseError('loadRedeemTransactionLinks', item, e as Error) + } + }) + } + + buildTransaction( + communityContext: CommunityContext, + item: RedeemedTransactionLinkDb, + linkFundingTransactionNr: number, + senderKeyPair: KeyPairEd25519, + recipientKeyPair: KeyPairEd25519, + ): GradidoTransactionBuilder { + return this.transactionBuilder + .setCreatedAt(item.redeemedAt) + .addMemo( + new EncryptedMemo( + item.memo, + new AuthenticatedEncryption(senderKeyPair), + new AuthenticatedEncryption(recipientKeyPair), + ), + ) + .setSenderCommunity(communityContext.communityId) + .setRedeemDeferredTransfer( + linkFundingTransactionNr, + new GradidoTransfer( + new TransferAmount( + senderKeyPair.getPublicKey(), + item.amount, + communityContext.communityId, + ), + recipientKeyPair.getPublicKey(), + ), + ) + .sign(senderKeyPair) + } + + calculateBalances( + item: RedeemedTransactionLinkDb, + fundingTransaction: GradidoDeferredTransfer, + communityContext: CommunityContext, + senderPublicKey: MemoryBlockPtr, + recipientPublicKey: MemoryBlockPtr, + ): AccountBalances { + this.accountBalances.clear() + + const senderLastBalance = this.getLastBalanceForUser( + senderPublicKey, + communityContext.blockchain, + communityContext.communityId, + ) + const fundingUserLastBalance = this.getLastBalanceForUser( + fundingTransaction.getSenderPublicKey()!, + communityContext.blockchain, + communityContext.communityId, + ) + const recipientLastBalance = this.getLastBalanceForUser( + recipientPublicKey, + communityContext.blockchain, + communityContext.communityId, + ) + + if (senderLastBalance.getAccountBalance().getBalance().lt(item.amount)) { + throw new Error( + `sender has not enough balance (${senderLastBalance.getAccountBalance().getBalance().toString()}) to send ${item.amount.toString()} to ${recipientPublicKey.convertToHex()}`, + ) + } + senderLastBalance.updateLegacyDecay(item.amount.negated(), item.redeemedAt) + fundingUserLastBalance.updateLegacyDecay(senderLastBalance.getBalance(), item.redeemedAt) + recipientLastBalance.updateLegacyDecay(item.amount, item.redeemedAt) + + // account of link is set to zero, and change send back to link creator + this.accountBalances.add( + new AccountBalance(senderPublicKey, GradidoUnit.zero(), communityContext.communityId), + ) + this.accountBalances.add(recipientLastBalance.getAccountBalance()) + this.accountBalances.add(fundingUserLastBalance.getAccountBalance()) + return this.accountBalances + } + + pushToBlockchain(item: RedeemedTransactionLinkDb): void { + const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) + const blockchain = communityContext.blockchain + + const senderKeyPair = deriveFromCode(item.code) + const senderPublicKey = senderKeyPair.getPublicKey() + const recipientKeyPair = this.getAccountKeyPair(communityContext, item.redeemedBy.gradidoId) + const recipientPublicKey = recipientKeyPair.getPublicKey() + + if (!senderKeyPair || !senderPublicKey || !recipientKeyPair || !recipientPublicKey) { + throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`) + } + + const transaction = blockchain.findOne(Filter.lastBalanceFor(senderPublicKey)) + if (!transaction) { + throw new Error(`expect transaction for code: ${item.code}`) + } + // should be funding transaction + if (!transaction.isDeferredTransfer()) { + throw new Error( + `expect funding transaction: ${transaction.getConfirmedTransaction()?.toJson(true)}`, + ) + } + const body = transaction + .getConfirmedTransaction() + ?.getGradidoTransaction() + ?.getTransactionBody() + const deferredTransfer = body?.getDeferredTransfer() + if (!deferredTransfer || !deferredTransfer.getRecipientPublicKey()?.equal(senderPublicKey)) { + throw new Error( + `expect funding transaction to belong to code: ${item.code}: ${transaction.getConfirmedTransaction()?.toJson(true)}`, + ) + } + + try { + addToBlockchain( + this.buildTransaction( + communityContext, + item, + transaction.getTransactionNr(), + senderKeyPair, + recipientKeyPair, + ).build(), + blockchain, + new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_LINK_ID), + this.calculateBalances( + item, + deferredTransfer, + communityContext, + senderPublicKey, + recipientPublicKey, + ), + ) + } catch (e) { + throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) + } + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts new file mode 100644 index 000000000..b951641ab --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts @@ -0,0 +1,266 @@ +import { Decimal } from 'decimal.js-light' +import { and, asc, eq, gt, inArray, isNull, ne, or } from 'drizzle-orm' +import { alias } from 'drizzle-orm/mysql-core' +import { + AccountBalance, + AccountBalances, + AuthenticatedEncryption, + EncryptedMemo, + GradidoTransactionBuilder, + GradidoUnit, + KeyPairEd25519, + LedgerAnchor, + MemoryBlockPtr, + TransferAmount, +} from 'gradido-blockchain-js' +import * as v from 'valibot' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' +import { addToBlockchain } from '../../blockchain' +import { Context } from '../../Context' +import { TransactionTypeId } from '../../data/TransactionTypeId' +import { transactionsTable, usersTable } from '../../drizzle.schema' +import { + BlockchainError, + DatabaseError, + NegativeBalanceError, + NotEnoughGradidoBalanceError, +} from '../../errors' +import { toMysqlDateTime } from '../../utils' +import { CommunityContext, TransactionDb, transactionDbSchema, UserDb } from '../../valibot.schema' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' + +export class RemoteTransactionsSyncRole extends AbstractSyncRole { + constructor(context: Context) { + super(context) + this.accountBalances.reserve(1) + } + + getDate(): Date { + return this.peek().balanceDate + } + + getCommunityUuids(): Uuidv4[] { + const currentItem = this.peek() + return [currentItem.user.communityUuid, currentItem.linkedUser.communityUuid] + } + + getLastIndex(): IndexType { + const lastItem = this.peekLast() + return { date: lastItem.balanceDate, id: lastItem.id } + } + + itemTypeName(): string { + return 'remoteTransactions' + } + + async loadFromDb(lastIndex: IndexType, count: number): Promise { + const linkedUsers = alias(usersTable, 'linkedUser') + const result = await this.context.db + .select({ + transaction: transactionsTable, + user: usersTable, + linkedUser: linkedUsers, + }) + .from(transactionsTable) + .where( + and( + inArray(transactionsTable.typeId, [TransactionTypeId.RECEIVE, TransactionTypeId.SEND]), + isNull(transactionsTable.transactionLinkId), + ne(usersTable.communityUuid, linkedUsers.communityUuid), + or( + gt(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)), + and( + eq(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)), + gt(transactionsTable.id, lastIndex.id), + ), + ), + ), + ) + .innerJoin(usersTable, eq(transactionsTable.userId, usersTable.id)) + .innerJoin(linkedUsers, eq(transactionsTable.linkedUserGradidoId, linkedUsers.gradidoId)) + .orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id)) + .limit(count) + + return result.map((row) => { + const item = { + ...row.transaction, + user: row.user, + linkedUser: row.linkedUser, + } + if (item.typeId === TransactionTypeId.SEND && item.amount) { + item.amount = new Decimal(item.amount).neg().toString() + } + try { + return v.parse(transactionDbSchema, item) + } catch (e) { + throw new DatabaseError('loadRemoteTransferTransactions', item, e as Error) + } + }) + } + + buildTransaction( + item: TransactionDb, + senderKeyPair: KeyPairEd25519, + recipientKeyPair: KeyPairEd25519, + senderCommunityId: string, + recipientCommunityId: string, + ): GradidoTransactionBuilder { + return this.transactionBuilder + .setCreatedAt(item.balanceDate) + .addMemo( + new EncryptedMemo( + item.memo, + new AuthenticatedEncryption(senderKeyPair), + new AuthenticatedEncryption(recipientKeyPair), + ), + ) + .setSenderCommunity(senderCommunityId) + .setRecipientCommunity(recipientCommunityId) + .setTransactionTransfer( + new TransferAmount(senderKeyPair.getPublicKey(), item.amount, senderCommunityId), + recipientKeyPair.getPublicKey(), + ) + .sign(senderKeyPair) + } + + calculateBalances( + item: TransactionDb, + communityContext: CommunityContext, + coinCommunityId: string, + amount: GradidoUnit, + publicKey: MemoryBlockPtr, + ): AccountBalances { + this.accountBalances.clear() + + // try to use same coins from this community + let lastBalance = this.getLastBalanceForUser( + publicKey, + communityContext.blockchain, + coinCommunityId, + ) + if ( + coinCommunityId !== communityContext.communityId && + (lastBalance.getBalance().equal(GradidoUnit.zero()) || + lastBalance.getBalance().calculateDecay(lastBalance.getDate(), item.balanceDate).lt(amount)) + ) { + // don't work, so we use or own coins + lastBalance = this.getLastBalanceForUser( + publicKey, + communityContext.blockchain, + communityContext.communityId, + ) + } + if ( + lastBalance + .getBalance() + .calculateDecay(lastBalance.getDate(), item.balanceDate) + .add(amount) + .lt(GradidoUnit.zero()) && + communityContext.foreign + ) { + this.accountBalances.add(new AccountBalance(publicKey, GradidoUnit.zero(), coinCommunityId)) + return this.accountBalances + } + + try { + lastBalance.updateLegacyDecay(amount, item.balanceDate) + } catch (e) { + if (e instanceof NegativeBalanceError) { + this.logLastBalanceChangingTransactions(publicKey, communityContext.blockchain, 1) + throw e + } + } + this.accountBalances.add(lastBalance.getAccountBalance()) + return this.accountBalances + } + + getUser(item: TransactionDb): { senderUser: UserDb; recipientUser: UserDb } { + return item.typeId === TransactionTypeId.RECEIVE + ? { senderUser: item.linkedUser, recipientUser: item.user } + : { senderUser: item.user, recipientUser: item.linkedUser } + } + + pushToBlockchain(item: TransactionDb): void { + const { senderUser, recipientUser } = this.getUser(item) + const ledgerAnchor = new LedgerAnchor( + item.id, + LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_ID, + ) + + if (senderUser.communityUuid === recipientUser.communityUuid) { + throw new Error( + `transfer between user from same community: ${JSON.stringify(item, null, 2)}, check db query`, + ) + } + const senderCommunityContext = this.context.getCommunityContextByUuid(senderUser.communityUuid) + const recipientCommunityContext = this.context.getCommunityContextByUuid( + recipientUser.communityUuid, + ) + const senderBlockchain = senderCommunityContext.blockchain + const recipientBlockchain = recipientCommunityContext.blockchain + + // I use the received transaction so user and linked user are swapped and user is recipient and linkedUser ist sender + const senderKeyPair = this.getAccountKeyPair(senderCommunityContext, senderUser.gradidoId) + const senderPublicKey = senderKeyPair.getPublicKey() + const recipientKeyPair = this.getAccountKeyPair( + recipientCommunityContext, + recipientUser.gradidoId, + ) + const recipientPublicKey = recipientKeyPair.getPublicKey() + + if (!senderKeyPair || !senderPublicKey || !recipientKeyPair || !recipientPublicKey) { + throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`) + } + const transactionBuilder = this.buildTransaction( + item, + senderKeyPair, + recipientKeyPair, + senderCommunityContext.communityId, + recipientCommunityContext.communityId, + ) + const outboundTransaction = transactionBuilder.buildOutbound() + + try { + addToBlockchain( + outboundTransaction, + senderBlockchain, + ledgerAnchor, + this.calculateBalances( + item, + senderCommunityContext, + senderCommunityContext.communityId, + item.amount.negated(), + senderPublicKey, + ), + ) + } catch (e) { + if (e instanceof NotEnoughGradidoBalanceError) { + this.logLastBalanceChangingTransactions(senderPublicKey, senderBlockchain) + } + throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) + } + transactionBuilder.setParentLedgerAnchor(ledgerAnchor) + const inboundTransaction = transactionBuilder.buildInbound() + try { + addToBlockchain( + inboundTransaction, + recipientBlockchain, + ledgerAnchor, + this.calculateBalances( + item, + recipientCommunityContext, + senderCommunityContext.communityId, + item.amount, + recipientPublicKey, + ), + ) + } catch (e) { + if (e instanceof NotEnoughGradidoBalanceError) { + this.logLastBalanceChangingTransactions(recipientPublicKey, recipientBlockchain) + } + throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) + } + // this.logLastBalanceChangingTransactions(senderPublicKey, senderCommunityContext.blockchain, 1) + // this.logLastBalanceChangingTransactions(recipientPublicKey, recipientCommunityContext.blockchain, 1) + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts new file mode 100644 index 000000000..5364a2bdf --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts @@ -0,0 +1,229 @@ +import Decimal from 'decimal.js-light' +import { and, asc, eq, gt, or } from 'drizzle-orm' +import { + AccountBalance, + AccountBalances, + AuthenticatedEncryption, + DurationSeconds, + EncryptedMemo, + Filter, + GradidoTransactionBuilder, + GradidoTransfer, + GradidoUnit, + KeyPairEd25519, + LedgerAnchor, + MemoryBlockPtr, + SearchDirection_DESC, + TransferAmount, + transactionTypeToString, +} from 'gradido-blockchain-js' +import * as v from 'valibot' +import { deriveFromCode } from '../../../../data/deriveKeyPair' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' +import { addToBlockchain } from '../../blockchain' +import { Context } from '../../Context' +import { transactionLinksTable, usersTable } from '../../drizzle.schema' +import { BlockchainError, DatabaseError, NegativeBalanceError } from '../../errors' +import { reverseLegacyDecay, toMysqlDateTime } from '../../utils' +import { CommunityContext, TransactionLinkDb, transactionLinkDbSchema } from '../../valibot.schema' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' + +export class TransactionLinkFundingsSyncRole extends AbstractSyncRole { + constructor(context: Context) { + super(context) + this.accountBalances.reserve(2) + } + getDate(): Date { + return this.peek().createdAt + } + + getCommunityUuids(): Uuidv4[] { + return [this.peek().user.communityUuid] + } + + getLastIndex(): IndexType { + const lastItem = this.peekLast() + return { date: lastItem.createdAt, id: lastItem.id } + } + + itemTypeName(): string { + return 'transactionLinkFundings' + } + + async loadFromDb(lastIndex: IndexType, count: number): Promise { + const result = await this.context.db + .select() + .from(transactionLinksTable) + .innerJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id)) + .where( + or( + gt(transactionLinksTable.createdAt, toMysqlDateTime(lastIndex.date)), + and( + eq(transactionLinksTable.createdAt, toMysqlDateTime(lastIndex.date)), + gt(transactionLinksTable.id, lastIndex.id), + ), + ), + ) + .orderBy(asc(transactionLinksTable.createdAt), asc(transactionLinksTable.id)) + .limit(count) + + return result.map((row) => { + const item = { + ...row.transaction_links, + user: row.users, + } + try { + return v.parse(transactionLinkDbSchema, item) + } catch (e) { + throw new DatabaseError('loadTransactionLinkFundings', item, e as Error) + } + }) + } + + buildTransaction( + communityContext: CommunityContext, + item: TransactionLinkDb, + blockedAmount: GradidoUnit, + duration: DurationSeconds, + senderKeyPair: KeyPairEd25519, + recipientKeyPair: KeyPairEd25519, + ): GradidoTransactionBuilder { + return this.transactionBuilder + .setCreatedAt(item.createdAt) + .addMemo( + new EncryptedMemo( + item.memo, + new AuthenticatedEncryption(senderKeyPair), + new AuthenticatedEncryption(recipientKeyPair), + ), + ) + .setSenderCommunity(communityContext.communityId) + .setDeferredTransfer( + new GradidoTransfer( + new TransferAmount( + senderKeyPair.getPublicKey(), + blockedAmount, + communityContext.communityId, + ), + recipientKeyPair.getPublicKey(), + ), + duration, + ) + .sign(senderKeyPair) + } + + calculateBalances( + item: TransactionLinkDb, + blockedAmount: GradidoUnit, + communityContext: CommunityContext, + senderPublicKey: MemoryBlockPtr, + recipientPublicKey: MemoryBlockPtr, + ): AccountBalances { + this.accountBalances.clear() + const senderLastBalance = this.getLastBalanceForUser( + senderPublicKey, + communityContext.blockchain, + communityContext.communityId, + ) + try { + senderLastBalance.updateLegacyDecay(blockedAmount.negated(), item.createdAt) + } catch (e) { + if (e instanceof NegativeBalanceError) { + this.logLastBalanceChangingTransactions(senderPublicKey, communityContext.blockchain) + this.context.logger.debug(`sender public key: ${senderPublicKey.convertToHex()}`) + throw e + } + } + + this.accountBalances.add(senderLastBalance.getAccountBalance()) + this.accountBalances.add( + new AccountBalance(recipientPublicKey, blockedAmount, communityContext.communityId), + ) + return this.accountBalances + } + + pushToBlockchain(item: TransactionLinkDb): void { + const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid) + const blockchain = communityContext.blockchain + + const senderKeyPair = this.getAccountKeyPair(communityContext, item.user.gradidoId) + const senderPublicKey = senderKeyPair.getPublicKey() + const recipientKeyPair = deriveFromCode(item.code) + const recipientPublicKey = recipientKeyPair.getPublicKey() + + if (!senderKeyPair || !senderPublicKey || !recipientKeyPair || !recipientPublicKey) { + throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`) + } + const duration = new DurationSeconds( + (item.validUntil.getTime() - item.createdAt.getTime()) / 1000, + ) + let blockedAmount = GradidoUnit.fromString( + reverseLegacyDecay(new Decimal(item.amount.toString()), duration.getSeconds()).toString(), + ) + let accountBalances: AccountBalances + try { + accountBalances = this.calculateBalances( + item, + blockedAmount, + communityContext, + senderPublicKey, + recipientPublicKey, + ) + } catch (e) { + if (item.deletedAt && e instanceof NegativeBalanceError) { + const senderLastBalance = this.getLastBalanceForUser( + senderPublicKey, + communityContext.blockchain, + communityContext.communityId, + ) + senderLastBalance.updateLegacyDecay(GradidoUnit.zero(), item.createdAt) + const oldBlockedAmountString = blockedAmount.toString() + blockedAmount = senderLastBalance.getBalance() + accountBalances = this.calculateBalances( + item, + blockedAmount, + communityContext, + senderPublicKey, + recipientPublicKey, + ) + this.context.logger.warn( + `workaround: fix founding for deleted link, reduce funding to actual sender balance: ${senderPublicKey.convertToHex()}: from ${oldBlockedAmountString} GDD to ${blockedAmount.toString()} GDD`, + ) + } else { + this.context.logger.error( + `error calculate account balances for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`, + ) + throw e + } + } + try { + addToBlockchain( + this.buildTransaction( + communityContext, + item, + blockedAmount, + duration, + senderKeyPair, + recipientKeyPair, + ).build(), + blockchain, + new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_LINK_ID), + accountBalances, + ) + } catch (e) { + if (e instanceof NegativeBalanceError) { + if ( + !item.deletedAt && + !item.redeemedAt && + item.validUntil.getTime() < new Date().getTime() + ) { + this.context.logger.warn( + `TransactionLinks: ${item.id} skipped, because else it lead to negative balance error, but it wasn't used.`, + ) + return + } + } + throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) + } + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/UsersSync.role.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/UsersSync.role.ts new file mode 100644 index 000000000..0bbec7c61 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/UsersSync.role.ts @@ -0,0 +1,132 @@ +import { and, asc, eq, gt, isNotNull, or } from 'drizzle-orm' +import { + AccountBalance, + AccountBalances, + AddressType_COMMUNITY_HUMAN, + GradidoTransactionBuilder, + GradidoUnit, + KeyPairEd25519, + LedgerAnchor, + MemoryBlockPtr, +} from 'gradido-blockchain-js' +import * as v from 'valibot' +import { deriveFromKeyPairAndUuid } from '../../../../data/deriveKeyPair' +import { Uuidv4Hash } from '../../../../data/Uuidv4Hash' +import { Uuidv4 } from '../../../../schemas/typeGuard.schema' +import { addToBlockchain } from '../../blockchain' +import { Context } from '../../Context' +import { usersTable } from '../../drizzle.schema' +import { BlockchainError, DatabaseError } from '../../errors' +import { toMysqlDateTime } from '../../utils' +import { CommunityContext, UserDb, userDbSchema } from '../../valibot.schema' +import { AbstractSyncRole, IndexType } from './AbstractSync.role' + +export class UsersSyncRole extends AbstractSyncRole { + constructor(context: Context) { + super(context) + this.accountBalances.reserve(1) + } + getDate(): Date { + return this.peek().createdAt + } + + getCommunityUuids(): Uuidv4[] { + return [this.peek().communityUuid] + } + + getLastIndex(): IndexType { + const lastItem = this.peekLast() + return { date: lastItem.createdAt, id: lastItem.id } + } + + itemTypeName(): string { + return 'users' + } + + async loadFromDb(lastIndex: IndexType, count: number): Promise { + const result = await this.context.db + .select() + .from(usersTable) + .where( + and( + or( + gt(usersTable.createdAt, toMysqlDateTime(lastIndex.date)), + and( + eq(usersTable.createdAt, toMysqlDateTime(lastIndex.date)), + gt(usersTable.id, lastIndex.id), + ), + ), + isNotNull(usersTable.communityUuid), + ), + ) + .orderBy(asc(usersTable.createdAt), asc(usersTable.id)) + .limit(count) + + return result.map((row) => { + try { + return v.parse(userDbSchema, row) + } catch (e) { + throw new DatabaseError('loadUsers', row, e as Error) + } + }) + } + + buildTransaction( + communityContext: CommunityContext, + item: UserDb, + communityKeyPair: KeyPairEd25519, + accountKeyPair: KeyPairEd25519, + userKeyPair: KeyPairEd25519, + ): GradidoTransactionBuilder { + return this.transactionBuilder + .setCreatedAt(item.createdAt) + .setSenderCommunity(communityContext.communityId) + .setRegisterAddress( + userKeyPair.getPublicKey(), + AddressType_COMMUNITY_HUMAN, + new Uuidv4Hash(item.gradidoId).getAsMemoryBlock(), + accountKeyPair.getPublicKey(), + ) + .sign(communityKeyPair) + .sign(accountKeyPair) + .sign(userKeyPair) + } + + calculateAccountBalances( + accountPublicKey: MemoryBlockPtr, + communityContext: CommunityContext, + ): AccountBalances { + this.accountBalances.clear() + this.accountBalances.add( + new AccountBalance(accountPublicKey, GradidoUnit.zero(), communityContext.communityId), + ) + return this.accountBalances + } + + pushToBlockchain(item: UserDb): void { + const communityContext = this.context.getCommunityContextByUuid(item.communityUuid) + const userKeyPair = deriveFromKeyPairAndUuid(communityContext.keyPair, item.gradidoId) + const accountKeyPair = this.getAccountKeyPair(communityContext, item.gradidoId) + const accountPublicKey = accountKeyPair.getPublicKey() + if (!userKeyPair || !accountKeyPair || !accountPublicKey) { + throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`) + } + + try { + addToBlockchain( + this.buildTransaction( + communityContext, + item, + communityContext.keyPair, + accountKeyPair, + userKeyPair, + ).build(), + communityContext.blockchain, + new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_USER_ID), + this.calculateAccountBalances(accountPublicKey, communityContext), + ) + } catch (e) { + throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error) + } + } +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts new file mode 100644 index 000000000..3824e9b8d --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts @@ -0,0 +1,161 @@ +import { + Abstract, + Filter, + InteractionCreateTransactionByEvent, + LedgerAnchor, + Profiler, + Timestamp, +} from 'gradido-blockchain-js' +import { Logger } from 'log4js' +import { callTime } from '../../blockchain' +import { Context } from '../../Context' +import { CommunityContext } from '../../valibot.schema' +import { nanosBalanceForUser } from './AbstractSync.role' +import { ContributionLinkTransactionSyncRole } from './ContributionLinkTransactionSync.role' +import { CreationsSyncRole } from './CreationsSync.role' +import { DeletedTransactionLinksSyncRole } from './DeletedTransactionLinksSync.role' +import { LocalTransactionsSyncRole } from './LocalTransactionsSync.role' +import { RedeemTransactionLinksSyncRole } from './RedeemTransactionLinksSync.role' +import { RemoteTransactionsSyncRole } from './RemoteTransactionsSync.role' +import { TransactionLinkFundingsSyncRole } from './TransactionLinkFundingsSync.role' +import { UsersSyncRole } from './UsersSync.role' + +function processTransactionTrigger(context: CommunityContext, endDate: Date, logger: Logger) { + while (true) { + const lastTx = context.blockchain.findOne(Filter.LAST_TRANSACTION) + let confirmedAt: Timestamp | undefined + if (!lastTx) { + // no transaction, no triggers + return + } else { + const confirmedTx = lastTx.getConfirmedTransaction() + if (!confirmedTx) { + throw new Error('missing confirmed tx in transaction entry') + } + confirmedAt = confirmedTx.getConfirmedAt() + } + const triggerEvent = context.blockchain.findNextTransactionTriggerEventInRange( + confirmedAt, + new Timestamp(endDate), + ) + if (!triggerEvent) { + // no trigger, we can exit here + return + } + context.blockchain.removeTransactionTriggerEvent(triggerEvent) + try { + // InMemoryBlockchain extend Abstract, but between C++ -> Swig -> TypeScript it seems the info is gone, so I need to cheat a bit here + const createTransactionByEvent = new InteractionCreateTransactionByEvent( + context.blockchain as unknown as Abstract, + ) + if ( + !context.blockchain.createAndAddConfirmedTransaction( + createTransactionByEvent.run(triggerEvent), + new LedgerAnchor( + triggerEvent.getLinkedTransactionId(), + LedgerAnchor.Type_NODE_TRIGGER_TRANSACTION_ID, + ), + triggerEvent.getTargetDate(), + ) + ) { + throw new Error('Adding trigger created Transaction Failed') + } + } catch (e) { + context.blockchain.addTransactionTriggerEvent(triggerEvent) + logger.error( + `Error processing transaction trigger event for transaction: ${triggerEvent.getLinkedTransactionId()}`, + ) + throw e + } + } +} + +export async function syncDbWithBlockchainContext(context: Context, batchSize: number) { + const timeUsedDB = new Profiler() + const timeUsedBlockchain = new Profiler() + const timeUsedAll = new Profiler() + const timeBetweenPrints = new Profiler() + const containers = [ + new UsersSyncRole(context), + new CreationsSyncRole(context), + new LocalTransactionsSyncRole(context), + new TransactionLinkFundingsSyncRole(context), + new RedeemTransactionLinksSyncRole(context), + new ContributionLinkTransactionSyncRole(context), + new DeletedTransactionLinksSyncRole(context), + new RemoteTransactionsSyncRole(context), + ] + let transactionsCount = 0 + let transactionsCountSinceLastLog = 0 + let transactionsCountSinceLastPrint = 0 + let available = containers + const isDebug = context.logger.isDebugEnabled() + let lastPrintedCallTime = 0 + while (true) { + timeUsedDB.reset() + const results = await Promise.all(available.map((c) => c.ensureFilled(batchSize))) + const loadedItemsCount = results.reduce((acc, c) => acc + c, 0) + // log only, if at least one new item was loaded + if (loadedItemsCount && isDebug) { + context.logger.debug(`${loadedItemsCount} new items loaded from db in ${timeUsedDB.string()}`) + } + + // remove empty containers + available = available.filter((c) => !c.isEmpty()) + if (available.length === 0) { + break + } + + // sort by date, to ensure container on index 0 is the one with the smallest date + if (available.length > 1) { + // const sortTime = new Profiler() + available.sort((a, b) => a.getDate().getTime() - b.getDate().getTime()) + // context.logger.debug(`sorted ${available.length} containers in ${sortTime.string()}`) + } + const communityUuids = available[0].getCommunityUuids() + for (let i = 0; i < communityUuids.length; ++i) { + processTransactionTrigger( + context.getCommunityContextByUuid(communityUuids[i]), + available[0].getDate(), + context.logger, + ) + } + + available[0].toBlockchain() + transactionsCount++ + if (isDebug) { + if (timeBetweenPrints.millis() > 100) { + process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`) + timeBetweenPrints.reset() + } + transactionsCountSinceLastLog++ + if (transactionsCountSinceLastLog >= batchSize) { + context.logger.debug( + `${transactionsCountSinceLastLog} transactions added to blockchain in ${timeUsedBlockchain.string()}`, + ) + context.logger.info( + `Time for createAndConfirm: ${((callTime - lastPrintedCallTime) / 1000 / 1000).toFixed(2)} milliseconds`, + ) + lastPrintedCallTime = callTime + timeUsedBlockchain.reset() + transactionsCountSinceLastLog = 0 + } + } else { + transactionsCountSinceLastPrint++ + if (transactionsCountSinceLastPrint >= 100) { + process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`) + transactionsCountSinceLastPrint = 0 + } + } + } + process.stdout.write(`successfully added to blockchain: ${transactionsCount}\n`) + context.logger.info( + `Synced ${transactionsCount} transactions to blockchain in ${timeUsedAll.string()}`, + ) + context.logger.info( + `Time for createAndConfirm: ${(callTime / 1000 / 1000 / 1000).toFixed(2)} seconds`, + ) + context.logger.info( + `Time for call lastBalance of user: ${(nanosBalanceForUser / 1000 / 1000 / 1000).toFixed(2)} seconds`, + ) +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/utils.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/utils.ts new file mode 100644 index 000000000..86604b2c6 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/utils.ts @@ -0,0 +1,70 @@ +import Decimal from 'decimal.js-light' +import { crypto_generichash_batch, crypto_generichash_KEYBYTES } from 'sodium-native' + +export function bytesToMbyte(bytes: number): string { + return (bytes / 1024 / 1024).toFixed(4) +} + +export function bytesToKbyte(bytes: number): string { + return (bytes / 1024).toFixed(0) +} + +export function bytesString(bytes: number): string { + if (bytes > 1024 * 1024) { + return `${bytesToMbyte(bytes)} MB` + } else if (bytes > 1024) { + return `${bytesToKbyte(bytes)} KB` + } + return `${bytes.toFixed(0)} Bytes` +} + +export function toMysqlDateTime(date: Date): string { + return date.toISOString().slice(0, 23).replace('T', ' ') +} + +export function calculateOneHashStep(hash: Buffer, data: Buffer): Buffer { + const outputHash = Buffer.alloc(crypto_generichash_KEYBYTES, 0) + crypto_generichash_batch(outputHash, [hash, data]) + return outputHash +} + +export const DECAY_START_TIME = new Date('2021-05-13T17:46:31Z') +export const SECONDS_PER_YEAR_GREGORIAN_CALENDER = 31556952.0 +const FACTOR = new Decimal('0.99999997803504048973201202316767079413460520837376') + +export function legacyDecayFormula(value: Decimal, seconds: number): Decimal { + // TODO why do we need to convert this here to a string to work properly? + // chatgpt: We convert to string here to avoid precision loss: + // .pow(seconds) can internally round the result, especially for large values of `seconds`. + // Using .toString() ensures full precision is preserved in the multiplication. + return value.mul(FACTOR.pow(seconds).toString()) +} + +export function reverseLegacyDecay(result: Decimal, seconds: number): Decimal { + return result.div(FACTOR.pow(seconds).toString()) +} + +export function legacyCalculateDecay(amount: Decimal, from: Date, to: Date): Decimal { + const fromMs = from.getTime() + const toMs = to.getTime() + const startBlockMs = DECAY_START_TIME.getTime() + + if (toMs < fromMs) { + throw new Error( + `calculateDecay: to (${to.toISOString()}) < from (${from.toISOString()}), reverse decay calculation is invalid`, + ) + } + + // decay started after end date; no decay + if (startBlockMs > toMs) { + return amount + } + // decay started before start date; decay for full duration + let duration = (toMs - fromMs) / 1000 + + // decay started between start and end date; decay from decay start till end date + if (startBlockMs >= fromMs) { + duration = (toMs - startBlockMs) / 1000 + } + return legacyDecayFormula(amount, duration) +} diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/valibot.schema.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/valibot.schema.ts new file mode 100644 index 000000000..28cab0be7 --- /dev/null +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/valibot.schema.ts @@ -0,0 +1,174 @@ +import Decimal from 'decimal.js-light' +import { GradidoUnit, InMemoryBlockchain, KeyPairEd25519 } from 'gradido-blockchain-js' +import * as v from 'valibot' +import { booleanSchema, dateSchema } from '../../schemas/typeConverter.schema' +import { + gradidoAmountSchema, + identifierSeedSchema, + memoSchema, + uuidv4Schema, +} from '../../schemas/typeGuard.schema' +import { Balance } from './data/Balance' +import { TransactionTypeId } from './data/TransactionTypeId' + +const positiveNumberSchema = v.pipe(v.number(), v.minValue(1)) + +export const userDbSchema = v.object({ + id: positiveNumberSchema, + gradidoId: uuidv4Schema, + communityUuid: uuidv4Schema, + createdAt: dateSchema, +}) +/* +declare const validLegacyAmount: unique symbol +export type LegacyAmount = string & { [validLegacyAmount]: true } + +export const legacyAmountSchema = v.pipe( + v.string(), + v.regex(/^-?[0-9]+(\.[0-9]+)?$/), + v.transform((input: string) => input as LegacyAmount), +) + +declare const validGradidoAmount: unique symbol +export type GradidoAmount = GradidoUnit & { [validGradidoAmount]: true } + +export const gradidoAmountSchema = v.pipe( + v.union([legacyAmountSchema, v.instance(GradidoUnit, 'expect GradidoUnit type')]), + v.transform((input: LegacyAmount | GradidoUnit) => { + if (input instanceof GradidoUnit) { + return input as GradidoAmount + } + // round floor with decimal js beforehand + const rounded = new Decimal(input).toDecimalPlaces(4, Decimal.ROUND_FLOOR).toString() + return GradidoUnit.fromString(rounded) as GradidoAmount + }), +) +*/ +export const transactionBaseSchema = v.object({ + id: positiveNumberSchema, + amount: gradidoAmountSchema, + memo: memoSchema, + user: userDbSchema, +}) + +export const transactionDbSchema = v.pipe( + v.object({ + ...transactionBaseSchema.entries, + typeId: v.enum(TransactionTypeId), + balanceDate: dateSchema, + linkedUser: userDbSchema, + }), + v.custom((value: any) => { + if ( + value.user && + value.linkedUser && + !value.transactionLinkCode && + value.user.gradidoId === value.linkedUser.gradidoId + ) { + throw new Error( + `expect user to be different from linkedUser: ${JSON.stringify(value, null, 2)}`, + ) + } + // check that user and linked user exist before transaction balance date + const balanceDate = new Date(value.balanceDate) + if ( + value.user.createdAt.getTime() >= balanceDate.getTime() || + value.linkedUser?.createdAt.getTime() >= balanceDate.getTime() + ) { + throw new Error( + `at least one user was created after transaction balance date, logic error! ${JSON.stringify(value, null, 2)}`, + ) + } + + return value + }), +) + +export const creationTransactionDbSchema = v.pipe( + v.object({ + ...transactionBaseSchema.entries, + contributionDate: dateSchema, + confirmedAt: dateSchema, + confirmedByUser: userDbSchema, + transactionId: positiveNumberSchema, + }), + v.custom((value: any) => { + if ( + value.user && + value.confirmedByUser && + value.user.gradidoId === value.confirmedByUser.gradidoId + ) { + throw new Error( + `expect user to be different from confirmedByUser: ${JSON.stringify(value, null, 2)}`, + ) + } + // check that user and confirmedByUser exist before transaction balance date + const confirmedAt = new Date(value.confirmedAt) + if ( + value.user.createdAt.getTime() >= confirmedAt.getTime() || + value.confirmedByUser?.createdAt.getTime() >= confirmedAt.getTime() + ) { + throw new Error( + `at least one user was created after transaction confirmedAt date, logic error! ${JSON.stringify(value, null, 2)}`, + ) + } + + return value + }), +) + +export const transactionLinkDbSchema = v.object({ + ...transactionBaseSchema.entries, + code: identifierSeedSchema, + createdAt: dateSchema, + validUntil: dateSchema, + holdAvailableAmount: gradidoAmountSchema, + redeemedAt: v.nullish(dateSchema), + deletedAt: v.nullish(dateSchema), +}) + +export const redeemedTransactionLinkDbSchema = v.object({ + ...transactionLinkDbSchema.entries, + redeemedAt: dateSchema, + redeemedBy: userDbSchema, +}) + +export const deletedTransactionLinKDbSchema = v.object({ + id: positiveNumberSchema, + user: userDbSchema, + code: identifierSeedSchema, + deletedAt: dateSchema, +}) + +export const communityDbSchema = v.object({ + id: positiveNumberSchema, + foreign: booleanSchema, + communityUuid: uuidv4Schema, + name: v.string(), + creationDate: dateSchema, + userMinCreatedAt: v.nullish(dateSchema), +}) + +export const communityContextSchema = v.object({ + communityId: v.string(), + foreign: booleanSchema, + blockchain: v.instance(InMemoryBlockchain, 'expect InMemoryBlockchain type'), + keyPair: v.instance(KeyPairEd25519), + folder: v.pipe( + v.string(), + v.minLength(1, 'expect string length >= 1'), + v.maxLength(512, 'expect string length <= 512'), + v.regex(/^[a-zA-Z0-9-_]+$/, 'expect string to be a valid (alphanumeric, _, -) folder name'), + ), + gmwBalance: v.instance(Balance), + aufBalance: v.instance(Balance), +}) + +export type TransactionDb = v.InferOutput +export type CreationTransactionDb = v.InferOutput +export type UserDb = v.InferOutput +export type TransactionLinkDb = v.InferOutput +export type RedeemedTransactionLinkDb = v.InferOutput +export type DeletedTransactionLinkDb = v.InferOutput +export type CommunityDb = v.InferOutput +export type CommunityContext = v.InferOutput diff --git a/dlt-connector/src/schemas/account.schema.ts b/dlt-connector/src/schemas/account.schema.ts index e328a2f90..c36655bcb 100644 --- a/dlt-connector/src/schemas/account.schema.ts +++ b/dlt-connector/src/schemas/account.schema.ts @@ -11,6 +11,7 @@ export type IdentifierCommunityAccount = v.InferOutput { } let topic: HieroId const topicString = '0.0.261' +let communityUuid: Uuidv4 +const communityUuidString = 'fcd48487-6d31-4f4c-be9b-b3c8ca853912' + beforeAll(() => { topic = v.parse(hieroIdSchema, topicString) + communityUuid = v.parse(uuidv4Schema, communityUuidString) }) describe('transaction schemas', () => { @@ -55,6 +59,7 @@ describe('transaction schemas', () => { registerAddress = { user: { communityTopicId: topicString, + communityId: communityUuidString, account: { userUuid: userUuidString }, }, type: InputTransactionType.REGISTER_ADDRESS, @@ -66,6 +71,7 @@ describe('transaction schemas', () => { expect(v.parse(transactionSchema, registerAddress)).toEqual({ user: { communityTopicId: topic, + communityId: communityUuid, account: { userUuid, accountNr: 0, @@ -80,6 +86,7 @@ describe('transaction schemas', () => { expect(v.parse(registerAddressTransactionSchema, registerAddress)).toEqual({ user: { communityTopicId: topic, + communityId: communityUuid, account: { userUuid, accountNr: 0, @@ -101,10 +108,12 @@ describe('transaction schemas', () => { const gradidoTransfer: TransactionInput = { user: { communityTopicId: topicString, + communityId: communityUuidString, account: { userUuid: userUuidString }, }, linkedUser: { communityTopicId: topicString, + communityId: communityUuidString, account: { userUuid: userUuidString }, }, amount: '100', @@ -115,6 +124,7 @@ describe('transaction schemas', () => { expect(v.parse(transactionSchema, gradidoTransfer)).toEqual({ user: { communityTopicId: topic, + communityId: communityUuid, account: { userUuid, accountNr: 0, @@ -122,6 +132,7 @@ describe('transaction schemas', () => { }, linkedUser: { communityTopicId: topic, + communityId: communityUuid, account: { userUuid, accountNr: 0, @@ -138,10 +149,12 @@ describe('transaction schemas', () => { const gradidoCreation: TransactionInput = { user: { communityTopicId: topicString, + communityId: communityUuidString, account: { userUuid: userUuidString }, }, linkedUser: { communityTopicId: topicString, + communityId: communityUuidString, account: { userUuid: userUuidString }, }, amount: '1000', @@ -153,10 +166,12 @@ describe('transaction schemas', () => { expect(v.parse(transactionSchema, gradidoCreation)).toEqual({ user: { communityTopicId: topic, + communityId: communityUuid, account: { userUuid, accountNr: 0 }, }, linkedUser: { communityTopicId: topic, + communityId: communityUuid, account: { userUuid, accountNr: 0 }, }, amount: v.parse(gradidoAmountSchema, gradidoCreation.amount!), @@ -172,12 +187,14 @@ describe('transaction schemas', () => { const gradidoTransactionLink: TransactionInput = { user: { communityTopicId: topicString, + communityId: communityUuidString, account: { userUuid: userUuidString, }, }, linkedUser: { communityTopicId: topicString, + communityId: communityUuidString, seed, }, amount: '100', @@ -189,6 +206,7 @@ describe('transaction schemas', () => { expect(v.parse(transactionSchema, gradidoTransactionLink)).toEqual({ user: { communityTopicId: topic, + communityId: communityUuid, account: { userUuid, accountNr: 0, @@ -196,6 +214,7 @@ describe('transaction schemas', () => { }, linkedUser: { communityTopicId: topic, + communityId: communityUuid, seed: seedParsed, }, amount: v.parse(gradidoAmountSchema, gradidoTransactionLink.amount!), diff --git a/dlt-connector/src/schemas/transaction.schema.ts b/dlt-connector/src/schemas/transaction.schema.ts index b018a8e86..c1e16c050 100644 --- a/dlt-connector/src/schemas/transaction.schema.ts +++ b/dlt-connector/src/schemas/transaction.schema.ts @@ -43,12 +43,14 @@ export type Transaction = v.InferOutput // if the account is identified by seed export const seedAccountSchema = v.object({ communityTopicId: hieroIdSchema, + communityId: uuidv4Schema, seed: identifierSeedSchema, }) // if the account is identified by userUuid and accountNr export const userAccountSchema = v.object({ communityTopicId: hieroIdSchema, + communityId: uuidv4Schema, account: identifierCommunityAccountSchema, }) diff --git a/dlt-connector/src/schemas/typeConverter.schema.test.ts b/dlt-connector/src/schemas/typeConverter.schema.test.ts index 5420de106..27259ee1e 100644 --- a/dlt-connector/src/schemas/typeConverter.schema.test.ts +++ b/dlt-connector/src/schemas/typeConverter.schema.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'bun:test' import { TypeCompiler } from '@sinclair/typebox/compiler' import { Static, TypeBoxFromValibot } from '@sinclair/typemap' -import { AddressType_COMMUNITY_AUF } from 'gradido-blockchain-js' +import { AddressType_COMMUNITY_AUF, InMemoryBlockchainProvider } from 'gradido-blockchain-js' import * as v from 'valibot' import { AccountType } from '../data/AccountType.enum' import { @@ -96,12 +96,15 @@ describe('basic.schema', () => { }) it('confirmedTransactionSchema', () => { - const confirmedTransaction = v.parse( - confirmedTransactionSchema, - 'CAcS5AEKZgpkCiCBZwMplGmI7fRR9MQkaR2Dz1qQQ5BCiC1btyJD71Ue9BJABODQ9sS70th9yHn8X3K+SNv2gsiIdX/V09baCvQCb+yEj2Dd/fzShIYqf3pooIMwJ01BkDJdNGBZs5MDzEAkChJ6ChkIAhIVRGFua2UgZnVlciBkZWluIFNlaW4hEggIgMy5/wUQABoDMy41IAAyTAooCiDbDtYSWhTwMKvtG/yDHgohjPn6v87n7NWBwMDniPAXxxCUmD0aABIgJE0o18xb6P6PsNjh0bkN52AzhggteTzoh09jV+blMq0aCAjC8rn/BRAAIgMzLjUqICiljeEjGHifWe4VNzoe+DN9oOLNZvJmv3VlkP+1RH7MMiAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADomCiDbDtYSWhTwMKvtG/yDHgohjPn6v87n7NWBwMDniPAXxxDAhD06JwogJE0o18xb6P6PsNjh0bkN52AzhggteTzoh09jV+blMq0Q65SlBA==', - ) + // create blockchain in native module + const communityId = 'fcd48487-6d31-4f4c-be9b-b3c8ca853912' + InMemoryBlockchainProvider.getInstance().getBlockchain(communityId) + const confirmedTransaction = v.parse(confirmedTransactionSchema, { + base64: + 'CAcS4AEKZgpkCiCBZwMplGmI7fRR9MQkaR2Dz1qQQ5BCiC1btyJD71Ue9BJABODQ9sS70th9yHn8X3K+SNv2gsiIdX/V09baCvQCb+zo7nEQgCUXOEe/tN7YaRppwt6TDcXBPxkwnw4gfpCODhJ0ChkIAhIVRGFua2UgZnVlciBkZWluIFNlaW4hEgYIgMy5/wUaAzMuNTJKCiYKINsO1hJaFPAwq+0b/IMeCiGM+fq/zufs1YHAwOeI8BfHEJSYPRIgJE0o18xb6P6PsNjh0bkN52AzhggteTzoh09jV+blMq0aABoGCMLyuf8FIgMzLjcqIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMhUIAhoRCgkIqemnUhD+4wESBBj8sgc6Jgog2w7WEloU8DCr7Rv8gx4KIYz5+r/O5+zVgcDA54jwF8cQwIQ9OicKICRNKNfMW+j+j7DY4dG5DedgM4YILXk86IdPY1fm5TKtEOuUpQRAAg==', + communityId, + }) expect(confirmedTransaction.getId()).toBe(7) expect(confirmedTransaction.getConfirmedAt().getSeconds()).toBe(1609464130) - expect(confirmedTransaction.getVersionNumber()).toBe('3.5') }) }) diff --git a/dlt-connector/src/schemas/typeConverter.schema.ts b/dlt-connector/src/schemas/typeConverter.schema.ts index 8774bd6c2..167712660 100644 --- a/dlt-connector/src/schemas/typeConverter.schema.ts +++ b/dlt-connector/src/schemas/typeConverter.schema.ts @@ -8,6 +8,7 @@ import { toAccountType, toAddressType, } from '../utils/typeConverter' +import { Uuidv4, uuidv4Schema } from './typeGuard.schema' /** * dateSchema for creating a date from string or Date object @@ -72,17 +73,23 @@ export const accountTypeSchema = v.pipe( export const confirmedTransactionSchema = v.pipe( v.union([ v.instance(ConfirmedTransaction, 'expect ConfirmedTransaction'), - v.pipe( - v.string('expect confirmed Transaction base64 as string type'), - v.base64('expect to be valid base64'), - ), + v.object({ + base64: v.pipe( + v.string('expect confirmed Transaction base64 as string type'), + v.base64('expect to be valid base64'), + ), + communityId: uuidv4Schema, + }), ]), - v.transform( - (data: string | ConfirmedTransaction) => { + v.transform( + (data: string | ConfirmedTransaction | { base64: string; communityId: Uuidv4 }) => { if (data instanceof ConfirmedTransaction) { return data } - return confirmedTransactionFromBase64(data) + if (typeof data === 'object' && 'base64' in data && 'communityId' in data) { + return confirmedTransactionFromBase64(data.base64, data.communityId) + } + throw new Error("invalid data, community id missing, couldn't deserialize") }, ), ) diff --git a/dlt-connector/src/schemas/typeGuard.schema.test.ts b/dlt-connector/src/schemas/typeGuard.schema.test.ts index ecd9eca7a..445d83c9c 100644 --- a/dlt-connector/src/schemas/typeGuard.schema.test.ts +++ b/dlt-connector/src/schemas/typeGuard.schema.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'bun:test' import { v4 as uuidv4 } from 'uuid' import * as v from 'valibot' -import { memoSchema, uuidv4Schema } from './typeGuard.schema' +import { MEMO_MAX_CHARS, MEMO_MIN_CHARS, memoSchema, uuidv4Schema } from './typeGuard.schema' describe('typeGuard.schema', () => { describe('Uuidv4', () => { @@ -20,18 +20,20 @@ describe('typeGuard.schema', () => { expect(memoValueParsed.toString()).toBe(memoValue) }) it('max length', () => { - const memoValue = 's'.repeat(255) + const memoValue = 's'.repeat(MEMO_MAX_CHARS) const memoValueParsed = v.parse(memoSchema, memoValue) expect(memoValueParsed.toString()).toBe(memoValue) }) it('to short', () => { const memoValue = 'memo' - expect(() => v.parse(memoSchema, memoValue)).toThrow(new Error('expect string length >= 5')) + expect(() => v.parse(memoSchema, memoValue)).toThrow( + new Error(`expect string length >= ${MEMO_MIN_CHARS}`), + ) }) it('to long', () => { - const memoValue = 's'.repeat(256) + const memoValue = 's'.repeat(MEMO_MAX_CHARS + 1) expect(() => v.parse(memoSchema, memoValue)).toThrow( - new Error('expect string length <= 255'), + new Error(`expect string length <= ${MEMO_MAX_CHARS}`), ) }) }) diff --git a/dlt-connector/src/schemas/typeGuard.schema.ts b/dlt-connector/src/schemas/typeGuard.schema.ts index 3a93ac62f..1f9f12eaf 100644 --- a/dlt-connector/src/schemas/typeGuard.schema.ts +++ b/dlt-connector/src/schemas/typeGuard.schema.ts @@ -163,7 +163,7 @@ export type HieroTransactionIdInput = v.InferInput { describe('Server', () => { it('send register address transaction', async () => { + // create blockchain in native module + const communityId = '1e88a0f4-d4fc-4cae-a7e8-a88e613ce324' + InMemoryBlockchainProvider.getInstance().getBlockchain(communityId) const transaction = { user: { communityTopicId: '0.0.21732', + communityId, account: { userUuid, accountNr: 0, diff --git a/dlt-connector/src/server/index.ts b/dlt-connector/src/server/index.ts index 1692bacfb..8afae814d 100644 --- a/dlt-connector/src/server/index.ts +++ b/dlt-connector/src/server/index.ts @@ -69,9 +69,10 @@ export const appRoutes = new Elysia() // check if account exists by user, call example: // GET /isAccountExist/by-user/0.0.21732/408780b2-59b3-402a-94be-56a4f4f4e8ec/0 .get( - '/isAccountExist/by-user/:communityTopicId/:userUuid/:accountNr', - async ({ params: { communityTopicId, userUuid, accountNr } }) => ({ + '/isAccountExist/by-user/:communityId/:communityTopicId/:userUuid/:accountNr', + async ({ params: { communityId, communityTopicId, userUuid, accountNr } }) => ({ exists: await isAccountExist({ + communityId, communityTopicId, account: { userUuid, accountNr }, }), @@ -84,9 +85,10 @@ export const appRoutes = new Elysia() // check if account exists by seed, call example: // GET /isAccountExist/by-seed/0.0.21732/0c4676adfd96519a0551596c .get( - '/isAccountExist/by-seed/:communityTopicId/:seed', - async ({ params: { communityTopicId, seed } }) => ({ + '/isAccountExist/by-seed/:communityId/:communityTopicId/:seed', + async ({ params: { communityId, communityTopicId, seed } }) => ({ exists: await isAccountExist({ + communityId, communityTopicId, seed, }), @@ -145,7 +147,7 @@ async function isAccountExist(identifierAccount: IdentifierAccountInput): Promis // ask gradido node server for account type, if type !== NONE account exist const addressType = await GradidoNodeClient.getInstance().getAddressType( publicKey.convertToHex(), - identifierAccountParsed.communityTopicId, + identifierAccountParsed.communityId, ) const exists = addressType !== AddressType_NONE const endTime = Date.now() diff --git a/dlt-connector/src/server/input.schema.ts b/dlt-connector/src/server/input.schema.ts index df93f1d9b..501fdc829 100644 --- a/dlt-connector/src/server/input.schema.ts +++ b/dlt-connector/src/server/input.schema.ts @@ -3,6 +3,7 @@ import { t } from 'elysia' import { hieroIdSchema, uuidv4Schema } from '../schemas/typeGuard.schema' export const accountIdentifierUserTypeBoxSchema = t.Object({ + communityId: TypeBoxFromValibot(uuidv4Schema), communityTopicId: TypeBoxFromValibot(hieroIdSchema), userUuid: TypeBoxFromValibot(uuidv4Schema), accountNr: t.Number({ min: 0 }), @@ -10,6 +11,7 @@ export const accountIdentifierUserTypeBoxSchema = t.Object({ // identifier for a gradido account created by transaction link / deferred transfer export const accountIdentifierSeedTypeBoxSchema = t.Object({ + communityId: TypeBoxFromValibot(uuidv4Schema), communityTopicId: TypeBoxFromValibot(hieroIdSchema), seed: TypeBoxFromValibot(uuidv4Schema), }) diff --git a/dlt-connector/src/utils/filesystem.ts b/dlt-connector/src/utils/filesystem.ts index ea2d64e39..338d536d8 100644 --- a/dlt-connector/src/utils/filesystem.ts +++ b/dlt-connector/src/utils/filesystem.ts @@ -9,7 +9,7 @@ export function checkFileExist(filePath: string): boolean { fs.accessSync(filePath, fs.constants.R_OK | fs.constants.W_OK) return true } catch (_err) { - // logger.debug(`file ${filePath} does not exist: ${_err}`) + logger.debug(`file ${filePath} does not exist: ${_err}`) return false } } @@ -28,3 +28,7 @@ export function checkPathExist(path: string, createIfMissing: boolean = false): } return false } + +export function toFolderName(name: string): string { + return name.toLowerCase().replace(/[^a-z0-9]/g, '_') +} diff --git a/dlt-connector/src/utils/network.ts b/dlt-connector/src/utils/network.ts index 5f348c640..954cbfd76 100644 --- a/dlt-connector/src/utils/network.ts +++ b/dlt-connector/src/utils/network.ts @@ -29,7 +29,7 @@ export async function isPortOpen( socket.destroy() const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.network.isPortOpen`) logger.addContext('url', url) - logger.error(`${err.message}: ${err.code}`) + logger.debug(`${err.message}: ${err.code}`) resolve(false) }) }) diff --git a/dlt-connector/src/utils/typeConverter.ts b/dlt-connector/src/utils/typeConverter.ts index 148efebd6..1e0ac464a 100644 --- a/dlt-connector/src/utils/typeConverter.ts +++ b/dlt-connector/src/utils/typeConverter.ts @@ -6,14 +6,18 @@ import { } from 'gradido-blockchain-js' import { AccountType } from '../data/AccountType.enum' import { AddressType } from '../data/AddressType.enum' +import { Uuidv4 } from '../schemas/typeGuard.schema' -export const confirmedTransactionFromBase64 = (base64: string): ConfirmedTransaction => { +export const confirmedTransactionFromBase64 = ( + base64: string, + communityId: Uuidv4, +): ConfirmedTransaction => { const confirmedTransactionBinaryPtr = MemoryBlock.createPtr(MemoryBlock.fromBase64(base64)) const deserializer = new InteractionDeserialize( confirmedTransactionBinaryPtr, DeserializeType_CONFIRMED_TRANSACTION, ) - deserializer.run() + deserializer.run(communityId) const confirmedTransaction = deserializer.getConfirmedTransaction() if (!confirmedTransaction) { throw new Error("invalid data, couldn't deserialize") diff --git a/inspector b/inspector index 0c14b7eea..e1d13e333 160000 --- a/inspector +++ b/inspector @@ -1 +1 @@ -Subproject commit 0c14b7eea29b8911cbe3cb303f5b0b61ce9bf6f4 +Subproject commit e1d13e3336199eae615557d11d6671c034860326 diff --git a/package.json b/package.json index 92696ba63..0ae1230fa 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "auto-changelog": "^2.4.0", "cross-env": "^7.0.3", "jose": "^4.14.4", - "turbo": "^2.5.0", + "turbo": "^2.8.12", "uuid": "^8.3.2" }, "devDependencies": { diff --git a/shared/src/const/index.ts b/shared/src/const/index.ts index 76990ffc8..e208103ac 100644 --- a/shared/src/const/index.ts +++ b/shared/src/const/index.ts @@ -1,4 +1,7 @@ +import Decimal from 'decimal.js-light' + export const DECAY_START_TIME = new Date('2021-05-13T17:46:31Z') +export const DECAY_FACTOR = new Decimal('0.99999997803504048973201202316767079413460520837376') export const SECONDS_PER_YEAR_GREGORIAN_CALENDER = 31556952.0 export const LOG4JS_BASE_CATEGORY_NAME = 'shared' export const REDEEM_JWT_TOKEN_EXPIRATION = '10m' diff --git a/shared/src/logic/decay.ts b/shared/src/logic/decay.ts index eec9f0cff..2346c2635 100644 --- a/shared/src/logic/decay.ts +++ b/shared/src/logic/decay.ts @@ -1,6 +1,6 @@ import { Decimal } from 'decimal.js-light' -import { DECAY_START_TIME, SECONDS_PER_YEAR_GREGORIAN_CALENDER } from '../const' +import { DECAY_FACTOR, DECAY_START_TIME, SECONDS_PER_YEAR_GREGORIAN_CALENDER } from '../const' Decimal.set({ precision: 25, @@ -16,21 +16,28 @@ export interface Decay { duration: number | null } +// legacy decay formula export function decayFormula(value: Decimal, seconds: number): Decimal { // TODO why do we need to convert this here to a string to work properly? // chatgpt: We convert to string here to avoid precision loss: // .pow(seconds) can internally round the result, especially for large values of `seconds`. // Using .toString() ensures full precision is preserved in the multiplication. - return value.mul( - new Decimal('0.99999997803504048973201202316767079413460520837376').pow(seconds).toString(), - ) + return value.mul(DECAY_FACTOR.pow(seconds).toString()) } +// legacy reverse decay formula +export function reverseLegacyDecay(result: Decimal, seconds: number): Decimal { + return result.div(DECAY_FACTOR.pow(seconds).toString()) +} + +// fast and more correct decay formula export function decayFormulaFast(value: Decimal, seconds: number): Decimal { return value.mul( new Decimal(2).pow(new Decimal(-seconds).div(new Decimal(SECONDS_PER_YEAR_GREGORIAN_CALENDER))), ) } + +// compound interest formula, the reverse decay formula for decayFormulaFast export function compoundInterest(value: Decimal, seconds: number): Decimal { return value.mul( new Decimal(2).pow(new Decimal(seconds).div(new Decimal(SECONDS_PER_YEAR_GREGORIAN_CALENDER))),