IT4Change/gradido
3
0
Fork 0
mirror of https://github.com/IT4Change/gradido.git synced 2026-04-06 01:25:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3607 from gradido/migrate_with_zig

Browse Source 
feat(dlt): migrate production db to blockchain
This commit is contained in:
einhornimmond 2026-03-10 12:03:47 +01:00 committed by GitHub
commit 98c1b6ed10
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
97 changed files with 3483 additions and 1091 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/test_submodules.yml vendored
View File

@ -45,5 +45,7 @@ jobs:
bun install --global turbo@^2
- name: typecheck, locales && unit test
run: turbo core#test core#typecheck core#locales database#test database#typecheck shared#test shared#typecheck config-schema#test config-schema#typecheck
run: turbo core#typecheck core#locales database#test database#typecheck shared#test shared#typecheck config-schema#test config-schema#typecheck
- name: core test extra
run: turbo core#test

8
backend/src/apis/dltConnector/model/AccountIdentifier.ts
View File

@ -1,15 +1,21 @@
import { CommunityAccountIdentifier } from './CommunityAccountIdentifier'
export class AccountIdentifier {
communityTopicId: string
communityId: string
account?: CommunityAccountIdentifier
seed?: string // used for deferred transfers
constructor(communityTopicId: string, input: CommunityAccountIdentifier | string) {
constructor(
communityTopicId: string,
communityUuid: string,
input: CommunityAccountIdentifier | string,
) {
if (input instanceof CommunityAccountIdentifier) {
this.account = input
} else {
this.seed = input
}
this.communityTopicId = communityTopicId
this.communityId = communityUuid
}
}

4
backend/src/apis/dltConnector/model/CommunityAccountIdentifier.ts
View File

@ -1,9 +1,9 @@
export class CommunityAccountIdentifier {
// for community user, uuid and communityUuid used
userUuid: string
accountNr?: number
accountNr: number
constructor(userUuid: string, accountNr?: number) {
constructor(userUuid: string, accountNr: number = 1) {
this.userUuid = userUuid
this.accountNr = accountNr
}

19
backend/src/apis/dltConnector/model/TransactionDraft.ts
View File

@ -36,6 +36,7 @@ export class TransactionDraft {
const draft = new TransactionDraft()
draft.user = new AccountIdentifier(
community.hieroTopicId,
community.communityUuid!,
new CommunityAccountIdentifier(user.gradidoID),
)
draft.type = TransactionType.REGISTER_ADDRESS
@ -58,10 +59,12 @@ export class TransactionDraft {
const draft = new TransactionDraft()
draft.user = new AccountIdentifier(
community.hieroTopicId,
community.communityUuid!,
new CommunityAccountIdentifier(contribution.user.gradidoID),
)
draft.linkedUser = new AccountIdentifier(
community.hieroTopicId,
community.communityUuid!,
new CommunityAccountIdentifier(signingUser.gradidoID),
)
draft.type = TransactionType.GRADIDO_CREATION
@ -96,10 +99,12 @@ export class TransactionDraft {
const draft = new TransactionDraft()
draft.user = new AccountIdentifier(
senderUserTopic,
sendingUser.community.communityUuid!,
new CommunityAccountIdentifier(sendingUser.gradidoID),
)
draft.linkedUser = new AccountIdentifier(
receiverUserTopic,
receivingUser.community.communityUuid!,
new CommunityAccountIdentifier(receivingUser.gradidoID),
)
draft.type = TransactionType.GRADIDO_TRANSFER
@ -125,9 +130,14 @@ export class TransactionDraft {
const draft = new TransactionDraft()
draft.user = new AccountIdentifier(
senderUserTopic,
sendingUser.community.communityUuid!,
new CommunityAccountIdentifier(sendingUser.gradidoID),
)
draft.linkedUser = new AccountIdentifier(senderUserTopic, transactionLink.code)
draft.linkedUser = new AccountIdentifier(
senderUserTopic,
sendingUser.community.communityUuid!,
transactionLink.code,
)
draft.type = TransactionType.GRADIDO_DEFERRED_TRANSFER
draft.createdAt = createdAtOnlySeconds.toISOString()
draft.amount = transactionLink.amount.toString()
@ -159,9 +169,14 @@ export class TransactionDraft {
const createdAtOnlySeconds = createdAt
createdAtOnlySeconds.setMilliseconds(0)
const draft = new TransactionDraft()
draft.user = new AccountIdentifier(senderUserTopic, transactionLink.code)
draft.user = new AccountIdentifier(
senderUserTopic,
transactionLink.user.community.communityUuid!,
transactionLink.code,
)
draft.linkedUser = new AccountIdentifier(
recipientUserTopic,
recipientUser.community.communityUuid!,
new CommunityAccountIdentifier(recipientUser.gradidoID),
)
draft.type = TransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER

1
backend/src/federation/client/1_0/query/getPublicCommunityInfo.ts
View File

@ -8,6 +8,7 @@ export const getPublicCommunityInfo = gql`
creationDate
publicKey
publicJwtKey
hieroTopicId
}
}
`

18
backend/src/graphql/resolver/CommunityResolver.ts
View File

@ -2,7 +2,12 @@ import { Paginated } from '@arg/Paginated'
import { EditCommunityInput } from '@input/EditCommunityInput'
import { AdminCommunityView } from '@model/AdminCommunityView'
import { Community } from '@model/Community'
import { Community as DbCommunity, getHomeCommunity, getReachableCommunities } from 'database'
import {
Community as DbCommunity,
getAuthorizedCommunities,
getHomeCommunity,
getReachableCommunities,
} from 'database'
import { updateAllDefinedAndChanged } from 'shared'
import { Arg, Args, Authorized, Mutation, Query, Resolver } from 'type-graphql'
import { RIGHTS } from '@/auth/RIGHTS'
@ -34,6 +39,17 @@ export class CommunityResolver {
return dbCommunities.map((dbCom: DbCommunity) => new Community(dbCom))
}
@Authorized([RIGHTS.COMMUNITIES])
@Query(() => [Community])
async authorizedCommunities(): Promise<Community[]> {
const dbCommunities: DbCommunity[] = await getAuthorizedCommunities({
// order by
foreign: 'ASC', // home community first
name: 'ASC',
})
return dbCommunities.map((dbCom: DbCommunity) => new Community(dbCom))
}
@Authorized([RIGHTS.COMMUNITIES])
@Query(() => Community)
async communityByIdentifier(

1
backend/src/seeds/index.ts
View File

@ -86,6 +86,7 @@ async function clearDatabase(db: AppDatabase) {
await trx.query(`SET FOREIGN_KEY_CHECKS = 0`)
await trx.query(`TRUNCATE TABLE contributions`)
await trx.query(`TRUNCATE TABLE contribution_links`)
await trx.query(`TRUNCATE TABLE events`)
await trx.query(`TRUNCATE TABLE users`)
await trx.query(`TRUNCATE TABLE user_contacts`)
await trx.query(`TRUNCATE TABLE user_roles`)

34
bun.lock
View File

@ -7,7 +7,7 @@
"auto-changelog": "^2.4.0",
"cross-env": "^7.0.3",
"jose": "^4.14.4",
"turbo": "^2.5.0",
"turbo": "^2.8.12",
"uuid": "^8.3.2",
},
"devDependencies": {
@ -18,7 +18,7 @@
},
"admin": {
"name": "admin",
"version": "2.7.3",
"version": "2.7.4",
"dependencies": {
"@iconify/json": "^2.2.228",
"@popperjs/core": "^2.11.8",
@ -88,7 +88,7 @@
},
"backend": {
"name": "backend",
"version": "2.7.3",
"version": "2.7.4",
"dependencies": {
"cross-env": "^7.0.3",
"email-templates": "^10.0.1",
@ -165,7 +165,7 @@
},
"config-schema": {
"name": "config-schema",
"version": "2.7.3",
"version": "2.7.4",
"dependencies": {
"esbuild": "^0.25.2",
"joi": "17.13.3",
@ -183,7 +183,7 @@
},
"core": {
"name": "core",
"version": "2.7.3",
"version": "2.7.4",
"dependencies": {
"database": "*",
"email-templates": "^10.0.1",
@ -220,7 +220,7 @@
},
"database": {
"name": "database",
"version": "2.7.3",
"version": "2.7.4",
"dependencies": {
"@types/uuid": "^8.3.4",
"cross-env": "^7.0.3",
@ -256,7 +256,7 @@
},
"dht-node": {
"name": "dht-node",
"version": "2.7.3",
"version": "2.7.4",
"dependencies": {
"cross-env": "^7.0.3",
"dht-rpc": "6.18.1",
@ -294,7 +294,7 @@
},
"federation": {
"name": "federation",
"version": "2.7.3",
"version": "2.7.4",
"dependencies": {
"cross-env": "^7.0.3",
"email-templates": "^10.0.1",
@ -355,7 +355,7 @@
},
"frontend": {
"name": "frontend",
"version": "2.7.3",
"version": "2.7.4",
"dependencies": {
"@morev/vue-transitions": "^3.0.2",
"@types/leaflet": "^1.9.12",
@ -451,7 +451,7 @@
},
"shared": {
"name": "shared",
"version": "2.7.3",
"version": "2.7.4",
"dependencies": {
"decimal.js-light": "^2.5.1",
"esbuild": "^0.25.2",
@ -3357,19 +3357,19 @@
"tunnel": ["tunnel@0.0.6", "", {}, "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg=="],
"turbo": ["turbo@2.6.1", "", { "optionalDependencies": { "turbo-darwin-64": "2.6.1", "turbo-darwin-arm64": "2.6.1", "turbo-linux-64": "2.6.1", "turbo-linux-arm64": "2.6.1", "turbo-windows-64": "2.6.1", "turbo-windows-arm64": "2.6.1" }, "bin": { "turbo": "bin/turbo" } }, "sha512-qBwXXuDT3rA53kbNafGbT5r++BrhRgx3sAo0cHoDAeG9g1ItTmUMgltz3Hy7Hazy1ODqNpR+C7QwqL6DYB52yA=="],
"turbo": ["turbo@2.8.12", "", { "optionalDependencies": { "turbo-darwin-64": "2.8.12", "turbo-darwin-arm64": "2.8.12", "turbo-linux-64": "2.8.12", "turbo-linux-arm64": "2.8.12", "turbo-windows-64": "2.8.12", "turbo-windows-arm64": "2.8.12" }, "bin": { "turbo": "bin/turbo" } }, "sha512-auUAMLmi0eJhxDhQrxzvuhfEbICnVt0CTiYQYY8WyRJ5nwCDZxD0JG8bCSxT4nusI2CwJzmZAay5BfF6LmK7Hw=="],
"turbo-darwin-64": ["turbo-darwin-64@2.6.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-Dm0HwhyZF4J0uLqkhUyCVJvKM9Rw7M03v3J9A7drHDQW0qAbIGBrUijQ8g4Q9Cciw/BXRRd8Uzkc3oue+qn+ZQ=="],
"turbo-darwin-64": ["turbo-darwin-64@2.8.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-EiHJmW2MeQQx+21x8hjMHw/uPhXt9PIxvDrxzOtyVwrXzL0tQmsxtO4qHf2l7uA+K6PUJ4+TjY1MHZDuCvWXrw=="],
"turbo-darwin-arm64": ["turbo-darwin-arm64@2.6.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-U0PIPTPyxdLsrC3jN7jaJUwgzX5sVUBsKLO7+6AL+OASaa1NbT1pPdiZoTkblBAALLP76FM0LlnsVQOnmjYhyw=="],
"turbo-darwin-arm64": ["turbo-darwin-arm64@2.8.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-cbqqGN0vd7ly2TeuaM8k9AK9u1CABO4kBA5KPSqovTiLL3sORccn/mZzJSbvQf0EsYRfU34MgW5FotfwW3kx8Q=="],
"turbo-linux-64": ["turbo-linux-64@2.6.1", "", { "os": "linux", "cpu": "x64" }, "sha512-eM1uLWgzv89bxlK29qwQEr9xYWBhmO/EGiH22UGfq+uXr+QW1OvNKKMogSN65Ry8lElMH4LZh0aX2DEc7eC0Mw=="],
"turbo-linux-64": ["turbo-linux-64@2.8.12", "", { "os": "linux", "cpu": "x64" }, "sha512-jXKw9j4r4q6s0goSXuKI3aKbQK2qiNeP25lGGEnq018TM6SWRW1CCpPMxyG91aCKrub7wDm/K45sGNT4ZFBcFQ=="],
"turbo-linux-arm64": ["turbo-linux-arm64@2.6.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-MFFh7AxAQAycXKuZDrbeutfWM5Ep0CEZ9u7zs4Hn2FvOViTCzIfEhmuJou3/a5+q5VX1zTxQrKGy+4Lf5cdpsA=="],
"turbo-linux-arm64": ["turbo-linux-arm64@2.8.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-BRJCMdyXjyBoL0GYpvj9d2WNfMHwc3tKmJG5ATn2Efvil9LsiOsd/93/NxDqW0jACtHFNVOPnd/CBwXRPiRbwA=="],
"turbo-windows-64": ["turbo-windows-64@2.6.1", "", { "os": "win32", "cpu": "x64" }, "sha512-buq7/VAN7KOjMYi4tSZT5m+jpqyhbRU2EUTTvp6V0Ii8dAkY2tAAjQN1q5q2ByflYWKecbQNTqxmVploE0LVwQ=="],
"turbo-windows-64": ["turbo-windows-64@2.8.12", "", { "os": "win32", "cpu": "x64" }, "sha512-vyFOlpFFzQFkikvSVhVkESEfzIopgs2J7J1rYvtSwSHQ4zmHxkC95Q8Kjkus8gg+8X2mZyP1GS5jirmaypGiPw=="],
"turbo-windows-arm64": ["turbo-windows-arm64@2.6.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-7w+AD5vJp3R+FB0YOj1YJcNcOOvBior7bcHTodqp90S3x3bLgpr7tE6xOea1e8JkP7GK6ciKVUpQvV7psiwU5Q=="],
"turbo-windows-arm64": ["turbo-windows-arm64@2.8.12", "", { "os": "win32", "cpu": "arm64" }, "sha512-9nRnlw5DF0LkJClkIws1evaIF36dmmMEO84J5Uj4oQ8C0QTHwlH7DNe5Kq2Jdmu8GXESCNDNuUYG8Cx6W/vm3g=="],
"type-check": ["type-check@0.4.0", "", { "dependencies": { "prelude-ls": "^1.2.1" } }, "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew=="],

345
database/migration/migrations/0098-fix_production_data_for_blockchain2.ts Normal file
View File

@ -0,0 +1,345 @@
import Decimal from 'decimal.js-light'
import { DECAY_FACTOR, reverseLegacyDecay } from 'shared'
function calculateEffectiveSeconds(holdOriginal: Decimal, holdCorrected: Decimal): Decimal {
return holdOriginal.div(holdCorrected).ln().div(DECAY_FACTOR.ln())
}
export async function upgrade(queryFn: (query: string, values?: any[]) => Promise<Array<any>>) {
/**
* Migration: Correct historical inconsistencies in transactions, users, and contribution_links.
*
* Background:
* Early Gradido production data contains several inconsistencies that violate
* stricter blockchain validation rules. These inconsistencies include:
* - Contribution transactions confirmed by users who did not exist at the
* time of the transaction.
* - Users confirming their own contributions (self-signed transactions).
* - Users whose `created_at` timestamp is after or equal to their first
* transaction.
* - Transaction memos shorter than the required minimum length (5 characters).
* - Existing contribution_links without an associated 'ADMIN_CONTRIBUTION_LINK_CREATE' event,
* which is used to find someone who confirmed the contribution.
*
* Purpose:
* This migration performs the following corrections to ensure historical
* consistency and full compatibility with blockchain validation rules:
* 1. Fix self-signed contributions by assigning the actual moderator.
* 2. Replace invalid moderators with the earliest ADMIN or MODERATOR where
* the linked user was created after the transaction.
* 3. Update user creation dates to be before their first transaction.
* 4. Ensure all transaction memos meet the minimum length requirement.
* 5. Insert missing 'ADMIN_CONTRIBUTION_LINK_CREATE' events for contribution_links
* that do not yet have such events, using the first Admin as acting_user.
*
* Outcome:
* After this migration:
* - All contribution transactions reference a valid moderator existing at the time of the transaction.
* - User creation dates are logically consistent with their transactions.
* - Transaction memos meet the minimum formatting rules.
* - Every contribution_link has a corresponding 'ADMIN_CONTRIBUTION_LINK_CREATE' event,
* ensuring blockchain consistency for contributions.
*/
/**
* Fix 0: Update transaction links to match holdAvailableAmount with validUntil, because the old formula lead to incorrect values
*/
let count = 0
let lastProcessedId = 0
const LIMIT = 200
do {
const rows = await queryFn(
`
SELECT id, amount, hold_available_amount, validUntil, createdAt, redeemedAt, deletedAt
FROM transaction_links
WHERE id > ?
ORDER BY id ASC
LIMIT ?
`,
[lastProcessedId, LIMIT],
)
if (!rows.length) {
break
}
const updates: Array<{ id: number; newValidUntil: string }> = []
for (const row of rows) {
const validUntil = new Date(row.validUntil)
const redeemedAt = row.redeemedAt ? new Date(row.redeemedAt) : null
const deletedAt = row.deletedAt ? new Date(row.deletedAt) : null
const createdAt = new Date(row.createdAt)
const amount = new Decimal(row.amount)
const duration = (validUntil.getTime() - createdAt.getTime()) / 1000
const blockedAmountCorrected = reverseLegacyDecay(amount, duration)
// fix only if the difference is big enough to have an impact
if (blockedAmountCorrected.sub(amount).abs().lt(new Decimal('0.001'))) {
continue
}
const holdAvailableAmount = new Decimal(row.hold_available_amount)
const secondsDiff = calculateEffectiveSeconds(
new Decimal(holdAvailableAmount.toString()),
new Decimal(blockedAmountCorrected.toString()),
)
const newValidUntil = new Date(validUntil.getTime() - secondsDiff.mul(1000).toNumber())
if (
(redeemedAt && redeemedAt.getTime() < validUntil.getTime()) ||
(deletedAt && deletedAt.getTime() < validUntil.getTime())
) {
continue
}
updates.push({
id: row.id,
newValidUntil: newValidUntil.toISOString().replace('T', ' ').replace('Z', ''),
})
}
if (updates.length > 0) {
const caseStatements = updates.map((u) => `WHEN ${u.id} THEN '${u.newValidUntil}'`).join('\n')
await queryFn(
`
UPDATE transaction_links
SET validUntil = CASE id
${caseStatements}
END
WHERE id IN (?)
`,
[updates.map((u) => u.id)],
)
}
count = rows.length
lastProcessedId = rows[rows.length - 1].id
} while (count === LIMIT)
///*/
/**
* Fix 1: Remove self-signed contributions.
*
* Background:
* A core rule in the system states that *no user may confirm their own
* contribution* — a moderator must always be someone else.
*
* However, early production data contains transactions where the `linked_user_id`
* matches the `user_id`, meaning the contributor confirmed their own contribution.
*
* This query corrects those records by replacing the `linked_user` with the
* moderator stored in `contributions.moderator_id`.
*
* Only transactions where:
* - the type is a contribution (type_id = 1),
* - the linked user equals the contributor (`t.user_id = t.linked_user_id`),
* - the moderator existed before the time of the transaction,
* - and the moderator is not the same person,
* are updated.
*/
await queryFn(`
UPDATE transactions t
JOIN contributions c ON(t.id = c.transaction_id)
JOIN users u ON(c.moderator_id = u.id)
SET t.linked_user_id = u.id,
t.linked_user_community_uuid = u.community_uuid,
t.linked_user_gradido_id = u.gradido_id,
t.linked_user_name = CONCAT(u.first_name, ' ', u.last_name)
WHERE t.type_id = 1
AND t.user_id = t.linked_user_id
AND u.created_at < t.balance_date
AND t.user_id <> u.id
;`)
await queryFn(`
UPDATE contributions c
JOIN users u ON(c.moderator_id = u.id)
SET c.confirmed_by = u.id
WHERE c.contribution_status = 'CONFIRMED'
AND c.user_id = c.confirmed_by
AND u.created_at < c.confirmed_at
AND c.user_id <> u.id
;`)
/**
* Fix 2: Replace invalid moderators with the earliest ADMIN.
*
* Background:
* Early production records contain contribution transactions where the assigned
* moderator ("linked_user" or "contribution.moderator_id") was created *after* the contribution itself. This
* is invalid in the blockchain verification process, which requires that the
* moderator account must have existed *before* the time of the transaction.
*
* This migration:
* 1. Identifies the earliest ADMIN or MODERATOR user in the system.
* 2. Reassigns them as moderator for all affected transactions where:
* - the type is a contribution (type_id = 1),
* - the linked user was created after or at the transaction date,
* - the transaction occurred after the ADMIN’s or MODERATOR's creation,
* - and the contributor is not the ADMIN or MODERATOR.
*
* Using the earliest ADMIN or MODERATOR ensures:
* - historical consistency,
* - minimal intrusion,
* - and compatibility with blockchain validation rules.
*/
await queryFn(`
UPDATE transactions t
JOIN (
SELECT t_sub.id as sub_t_id, u_sub.created_at, u_sub.id, u_sub.community_uuid, u_sub.gradido_id, CONCAT(u_sub.first_name, ' ', u_sub.last_name) AS linked_user_name
FROM transactions t_sub
JOIN users u_sub on(t_sub.user_id <> u_sub.id)
JOIN user_roles r_sub ON u_sub.id = r_sub.user_id
WHERE r_sub.role IN ('ADMIN', 'MODERATOR')
GROUP BY t_sub.id
ORDER BY r_sub.created_at ASC
) moderator ON (t.id = moderator.sub_t_id)
LEFT JOIN users u on(t.linked_user_id = u.id)
SET t.linked_user_id = moderator.id,
t.linked_user_community_uuid = moderator.community_uuid,
t.linked_user_gradido_id = moderator.gradido_id,
t.linked_user_name = moderator.linked_user_name
WHERE t.type_id = 1
AND t.balance_date <= u.created_at
AND t.balance_date > moderator.created_at
AND t.user_id <> moderator.id
;`)
// similar but with confirmed by user
await queryFn(`
UPDATE contributions c
JOIN (
SELECT c_sub.id as sub_c_id, u_sub.created_at, u_sub.id
FROM contributions c_sub
JOIN users u_sub ON (c_sub.confirmed_by <> u_sub.id AND c_sub.user_id <> u_sub.id)
JOIN user_roles r_sub ON (u_sub.id = r_sub.user_id)
WHERE r_sub.role IN ('ADMIN', 'MODERATOR')
GROUP BY c_sub.id
ORDER BY r_sub.created_at ASC
) confirmingUser ON (c.id = confirmingUser.sub_c_id)
LEFT JOIN users u on(c.confirmed_by = u.id)
SET c.confirmed_by = confirmingUser.id
WHERE c.confirmed_at <= u.created_at
AND c.confirmed_at > confirmingUser.created_at
AND c.user_id <> confirmingUser.id
;`)
/**
* Fix 3: Update user creation dates to ensure historical consistency.
*
* Background:
* In early production data, some users have a `created_at` timestamp that is
* **after or equal** to their first recorded transaction (`balance_date`).
* This violates logical consistency, because a user cannot exist *after* their
* own transaction.
*
* What this query does:
* - For each user, it finds the earliest transaction date (`first_date`) from
* the `transactions` table.
* - It updates the user's `created_at` timestamp to **1 second before** their
* first transaction.
*
* Notes:
* - Only users where `created_at >= first transaction date` are affected.
* - This is a historical data fix to ensure all transactions reference a user
* that already exists at the time of the transaction, which is required for
* blockchain validation and logical consistency in the system.
*/
await queryFn(`
UPDATE users u
LEFT JOIN (
SELECT user_id, MIN(balance_date) AS first_date
FROM transactions
GROUP BY user_id
) t ON t.user_id = u.id
SET u.created_at = DATE_SUB(t.first_date, INTERVAL 1 SECOND)
WHERE u.created_at >= t.first_date;
;`)
// linked user also, but we need to use gradido_id as index, because on cross group transactions linked_user_id is empty
await queryFn(`
UPDATE users u
LEFT JOIN (
SELECT linked_user_gradido_id , MIN(balance_date) AS first_date
FROM transactions
GROUP BY linked_user_gradido_id
) t ON t.linked_user_gradido_id = u.gradido_id
SET u.created_at = DATE_SUB(t.first_date, INTERVAL 1 SECOND)
WHERE u.created_at >= t.first_date;
;`)
/**
* Fix 4: Ensure all transaction memos meet the minimum length requirement.
*
* Background:
* In early Gradido production data, some transactions have a `memo` field
* shorter than the current rule of 5 characters. This can cause issues in
* reporting, display, or blockchain validation processes that expect
* a minimum memo length.
*
* What this query does:
* - For memos with 0 characters, sets the value to 'empty empty'.
* - For memos with 1-4 characters, pads the memo on the left with spaces
* until it reaches 5 characters.
* - Memos that are already 5 characters or longer are left unchanged.
*
* Notes:
* - This ensures all memos are at least 5 characters long.
* - The padding uses spaces.
* - Only memos shorter than 5 characters are affected.
*/
await queryFn(`
UPDATE transactions t
SET t.memo = CASE
WHEN CHAR_LENGTH(t.memo) = 0 THEN 'empty empty'
WHEN CHAR_LENGTH(t.memo) < 5 THEN LPAD(t.memo, 5, ' ')
ELSE t.memo
END
WHERE CHAR_LENGTH(t.memo) < 5
;`)
await queryFn(`
UPDATE contributions t
SET t.memo = CASE
WHEN CHAR_LENGTH(t.memo) = 0 THEN 'empty empty'
WHEN CHAR_LENGTH(t.memo) < 5 THEN LPAD(t.memo, 5, ' ')
ELSE t.memo
END
WHERE CHAR_LENGTH(t.memo) < 5
;`)
await queryFn(`
UPDATE transaction_links t
SET t.memo = CASE
WHEN CHAR_LENGTH(t.memo) = 0 THEN 'empty empty'
WHEN CHAR_LENGTH(t.memo) < 5 THEN LPAD(t.memo, 5, ' ')
ELSE t.memo
END
WHERE CHAR_LENGTH(t.memo) < 5
;`)
/**
* Fix 5: Insert missing 'ADMIN_CONTRIBUTION_LINK_CREATE' events for contribution_links.
*
* Background:
* Each contribution in the blockchain requires a confirmation by a user.
* In the current DB version, there is no information about who confirmed contributions based on contribution_links.
* Recently, functionality was added to create an 'ADMIN_CONTRIBUTION_LINK_CREATE' event
* for newly created contribution_links, but existing contribution_links were not updated.
*
* This query inserts an 'ADMIN_CONTRIBUTION_LINK_CREATE' event for every contribution_link
* that does not already have such an event.
* The acting_user_id is set to the first Admin, and affected_user_id is set to 0.
*/
await queryFn(`
INSERT INTO \`events\`(acting_user_id, affected_user_id, \`type\`, involved_contribution_link_id)
SELECT (
SELECT u.id
FROM users u
JOIN user_roles r ON r.user_id = u.id
WHERE r.role = 'ADMIN'
ORDER BY r.id ASC
LIMIT 1
) AS acting_user_id, 0 as affected_user_id, 'ADMIN_CONTRIBUTION_LINK_CREATE' AS \`type\`, c.id AS involved_contribution_link_id
FROM contribution_links c
LEFT JOIN \`events\` e ON e.involved_contribution_link_id = c.id AND e.type = 'ADMIN_CONTRIBUTION_LINK_CREATE'
WHERE e.id IS NULL
;`)
}
export async function downgrade(queryFn: (query: string, values?: any[]) => Promise<Array<any>>) {
// downgrade not possible
}

15
database/src/queries/communities.ts
View File

@ -84,7 +84,7 @@ export async function getReachableCommunities(
federatedCommunities: {
verifiedAt: MoreThanOrEqual(new Date(Date.now() - authenticationTimeoutMs)),
},
},
}, // or
{ foreign: false },
],
order,
@ -99,3 +99,16 @@ export async function getNotReachableCommunities(
order,
})
}
// return the home community and all communities which had at least once make it through the first handshake
export async function getAuthorizedCommunities(
order?: FindOptionsOrder<DbCommunity>,
): Promise<DbCommunity[]> {
return await DbCommunity.find({
where: [
{ authenticatedAt: Not(IsNull()) }, // or
{ foreign: false },
],
order,
})
}

2
dlt-connector/.gitignore vendored
View File

@ -3,6 +3,8 @@
/.env.bak
/build/
/locales/
lib
.zigar-cache
package-json.lock
coverage
# emacs

21
dlt-connector/bun.lock
View File

@ -4,7 +4,9 @@
"": {
"name": "dlt-connector",
"dependencies": {
"gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#f265dbb1780a912cf8b0418dfe3eaf5cdc5b51cf",
"bun-zigar": "^0.15.2",
"cross-env": "^7.0.3",
"gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#785fd766289726d41ae01f1e80a274aed871a7fb",
},
"devDependencies": {
"@biomejs/biome": "2.0.0",
@ -18,6 +20,7 @@
"@types/uuid": "^8.3.4",
"adm-zip": "^0.5.16",
"async-mutex": "^0.5.0",
"decimal.js-light": "^2.5.1",
"dotenv": "^10.0.0",
"drizzle-orm": "^0.44.7",
"elysia": "1.3.8",
@ -389,6 +392,8 @@
"bun-types": ["bun-types@1.3.1", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-NMrcy7smratanWJ2mMXdpatalovtxVggkj11bScuWuiOoXTiKIu2eVS1/7qbyI/4yHedtsn175n4Sm4JcdHLXw=="],
"bun-zigar": ["bun-zigar@0.15.2", "", { "dependencies": { "node-zigar-addon": "0.15.2", "zigar-compiler": "^0.15.2" }, "bin": { "zigar": "bin/cli.js", "bun-zigar": "bin/cli.js" } }, "sha512-slEHTEapQEIqB86OeiToPuuFXe39DCIYISTPzbIMBTZL34vRzCIa5wFn5ATudauHFFwl5/y5JYv8tluk2QL9Eg=="],
"call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="],
"camelcase": ["camelcase@6.3.0", "", {}, "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA=="],
@ -431,6 +436,8 @@
"cookie": ["cookie@1.0.2", "", {}, "sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA=="],
"cross-env": ["cross-env@7.0.3", "", { "dependencies": { "cross-spawn": "^7.0.1" }, "bin": { "cross-env": "src/bin/cross-env.js", "cross-env-shell": "src/bin/cross-env-shell.js" } }, "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw=="],
"cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="],
"crypto-js": ["crypto-js@4.2.0", "", {}, "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q=="],
@ -443,6 +450,8 @@
"debug": ["debug@4.4.1", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ=="],
"decimal.js-light": ["decimal.js-light@2.5.1", "", {}, "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg=="],
"deep-extend": ["deep-extend@0.6.0", "", {}, "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="],
"delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="],
@ -575,7 +584,7 @@
"graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="],
"gradido-blockchain-js": ["gradido-blockchain-js@github:gradido/gradido-blockchain-js#f265dbb", { "dependencies": { "bindings": "^1.5.0", "nan": "^2.20.0", "node-addon-api": "^7.1.1", "node-gyp-build": "^4.8.1", "prebuildify": "git+https://github.com/einhornimmond/prebuildify#65d94455fab86b902c0d59bb9c06ac70470e56b2" } }, "gradido-gradido-blockchain-js-f265dbb"],
"gradido-blockchain-js": ["gradido-blockchain-js@github:gradido/gradido-blockchain-js#785fd76", { "dependencies": { "bindings": "^1.5.0", "nan": "^2.20.0", "node-addon-api": "^7.1.1", "node-gyp-build": "^4.8.1", "prebuildify": "git+https://github.com/einhornimmond/prebuildify#65d94455fab86b902c0d59bb9c06ac70470e56b2" } }, "gradido-gradido-blockchain-js-785fd76"],
"graphql": ["graphql@16.11.0", "", {}, "sha512-mS1lbMsxgQj6hge1XZ6p7GPhbrtFwUFYi3wRzXAC/FmYnyXMTvvI3td3rjmQ2u8ewXueaSvRPWaEcgVVOT9Jnw=="],
@ -777,7 +786,7 @@
"node-addon-api": ["node-addon-api@7.1.1", "", {}, "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ=="],
"node-api-headers": ["node-api-headers@1.6.0", "", {}, "sha512-81T99+mWLZnxX0LlZPYuafyFlxVVaWKQ0BDAbSrOqLO+v+gzCzu0GTAVNeVK8lucqjqo9L/1UcK9cpkem8Py4Q=="],
"node-api-headers": ["node-api-headers@1.8.0", "", {}, "sha512-jfnmiKWjRAGbdD1yQS28bknFM1tbHC1oucyuMPjmkEs+kpiu76aRs40WlTmBmyEgzDM76ge1DQ7XJ3R5deiVjQ=="],
"node-gyp-build": ["node-gyp-build@4.8.4", "", { "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", "node-gyp-build-test": "build-test.js" } }, "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ=="],
@ -785,6 +794,8 @@
"node-releases": ["node-releases@2.0.26", "", {}, "sha512-S2M9YimhSjBSvYnlr5/+umAnPHE++ODwt5e2Ij6FoX45HA/s4vHdkDx1eax2pAPeAOqu4s9b7ppahsyEFdVqQA=="],
"node-zigar-addon": ["node-zigar-addon@0.15.2", "", { "dependencies": { "node-api-headers": "^1.7.0" } }, "sha512-QjJcPRtUZLkULaFXapAvTzLKKRddgaupr7wQqgDUQo541FMCXAhgWdZJtNcIgCNykJG0bG0Fza5VTKBdSvyavQ=="],
"normalize-path": ["normalize-path@3.0.0", "", {}, "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="],
"npm-path": ["npm-path@2.0.4", "", { "dependencies": { "which": "^1.2.10" }, "bin": { "npm-path": "bin/npm-path" } }, "sha512-IFsj0R9C7ZdR5cP+ET342q77uSRdtWOlWpih5eC+lu29tIDbNEgDbzgVJ5UFvYHWhxDZ5TFkJafFioO0pPQjCw=="],
@ -1053,6 +1064,8 @@
"yoctocolors-cjs": ["yoctocolors-cjs@2.1.3", "", {}, "sha512-U/PBtDf35ff0D8X8D0jfdzHYEPFxAI7jJlxZXwCSez5M3190m+QobIfh+sWDWSHMCWWJN2AWamkegn6vr6YBTw=="],
"zigar-compiler": ["zigar-compiler@0.15.2", "", {}, "sha512-zlJ8kUwndwrLl4iRlIWEcidC2rcSsfeWM0jvbSoxUVf+SEKd4bVik3z4YDivuyX3SUiUjpCMNyp65etD6BKRmQ=="],
"zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="],
"@babel/core/debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
@ -1097,6 +1110,8 @@
"cmake-js/fs-extra": ["fs-extra@11.3.2", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" } }, "sha512-Xr9F6z6up6Ws+NjzMCZc6WXg2YFRlrLP9NQDO3VQrWrfiojdhS56TzueT88ze0uBdCTwEIhQ3ptnmKeWGFAe0A=="],
"cmake-js/node-api-headers": ["node-api-headers@1.6.0", "", {}, "sha512-81T99+mWLZnxX0LlZPYuafyFlxVVaWKQ0BDAbSrOqLO+v+gzCzu0GTAVNeVK8lucqjqo9L/1UcK9cpkem8Py4Q=="],
"connect/debug": ["debug@2.6.9", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="],
"elliptic/bn.js": ["bn.js@4.12.2", "", {}, "sha512-n4DSx829VRTRByMRGdjQ9iqsN0Bh4OolPsFnaZBLcbi8iXcB+kJ9s7EnRt4wILZNV3kPLHkRVfOc/HvhC3ovDw=="],

1
dlt-connector/bunfig.toml Normal file
View File

@ -0,0 +1 @@
preload = ["bun-zigar"]

15
dlt-connector/package.json
View File

@ -7,18 +7,20 @@
"license": "Apache-2.0",
"private": true,
"scripts": {
"start": "bun run src/index.ts",
"start": "cross-env TZ=UTC bun run src/index.ts",
"build": "bun build src/index.ts --outdir=build --target=bun --external=gradido-blockchain-js --minify",
"dev": "bun run --watch src/index.ts",
"migrate": "bun src/migrations/db-v2.7.0_to_blockchain-v3.5",
"test": "bun test",
"test:debug": "bun test --inspect-brk",
"dev": "cross-env TZ=UTC bun run --watch src/index.ts",
"migrate": "cross-env TZ=UTC bun src/migrations/db-v2.7.0_to_blockchain-v3.7",
"test": "cross-env TZ=UTC bun test",
"test:debug": "cross-env TZ=UTC bun test --inspect-brk",
"typecheck": "tsc --noEmit",
"lint": "biome check --error-on-warnings .",
"lint:fix": "biome check --error-on-warnings . --write"
},
"dependencies": {
"gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#f265dbb1780a912cf8b0418dfe3eaf5cdc5b51cf"
"bun-zigar": "^0.15.2",
"cross-env": "^7.0.3",
"gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#785fd766289726d41ae01f1e80a274aed871a7fb"
},
"devDependencies": {
"@biomejs/biome": "2.0.0",
@ -32,6 +34,7 @@
"@types/uuid": "^8.3.4",
"adm-zip": "^0.5.16",
"async-mutex": "^0.5.0",
"decimal.js-light": "^2.5.1",
"dotenv": "^10.0.0",
"drizzle-orm": "^0.44.7",
"elysia": "1.3.8",

33
dlt-connector/src/bootstrap/init.ts
View File

@ -1,9 +1,11 @@
import { readFileSync } from 'node:fs'
import { loadCryptoKeys, MemoryBlock } from 'gradido-blockchain-js'
import { InMemoryBlockchainProvider, loadCryptoKeys, MemoryBlock } from 'gradido-blockchain-js'
import { configure, getLogger, Logger } from 'log4js'
import * as v from 'valibot'
import { CONFIG } from '../config'
import { MIN_TOPIC_EXPIRE_MILLISECONDS_FOR_UPDATE } from '../config/const'
import { KeyPairIdentifierLogic } from '../data/KeyPairIdentifier.logic'
import { ResolveKeyPair } from '../interactions/resolveKeyPair/ResolveKeyPair.context'
import { SendToHieroContext } from '../interactions/sendToHiero/SendToHiero.context'
import { Community, communitySchema } from '../schemas/transaction.schema'
import { isPortOpenRetry } from '../utils/network'
@ -33,11 +35,18 @@ export async function checkHieroAccount(logger: Logger, clients: AppContextClien
export async function checkHomeCommunity(
appContext: AppContext,
logger: Logger,
): Promise<Community> {
): Promise<Community | undefined> {
const { backend, hiero } = appContext.clients
// wait for backend server
await isPortOpenRetry(backend.url)
try {
logger.info(`Waiting for backend server to become available at ${backend.url}`)
await isPortOpenRetry(backend.url)
} catch (e) {
logger.error(`Backend server at ${backend.url} is not reachable (${e})`)
return
}
// ask backend for home community
let homeCommunity = await backend.getHomeCommunityDraft()
// on missing topicId, create one
@ -56,7 +65,7 @@ export async function checkHomeCommunity(
await hiero.updateTopic(homeCommunity.hieroTopicId)
topicInfo = await hiero.getTopicInfo(homeCommunity.hieroTopicId)
logger.info(
`updated topic info, new expiration time: ${topicInfo.expirationTime.toLocaleDateString()}`,
`Topic expiration extended. New expiration time: ${topicInfo.expirationTime.toLocaleDateString()}`,
)
}
}
@ -64,9 +73,16 @@ export async function checkHomeCommunity(
throw new Error('still no topic id, after creating topic and update community in backend.')
}
appContext.cache.setHomeCommunityTopicId(homeCommunity.hieroTopicId)
logger.info(`home community topic: ${homeCommunity.hieroTopicId}`)
logger.info(`gradido node server: ${appContext.clients.gradidoNode.url}`)
logger.info(`gradido backend server: ${appContext.clients.backend.url}`)
logger.info(`Home community topic id: ${homeCommunity.hieroTopicId}`)
logger.info(`Gradido node server: ${appContext.clients.gradidoNode.url}`)
logger.info(`Gradido backend server: ${appContext.clients.backend.url}`)
await ResolveKeyPair(
new KeyPairIdentifierLogic({
communityTopicId: homeCommunity.hieroTopicId,
communityId: homeCommunity.uuid,
}),
)
return v.parse(communitySchema, homeCommunity)
}
@ -80,10 +96,11 @@ export async function checkGradidoNode(
// ask gradido node if community blockchain was created
try {
InMemoryBlockchainProvider.getInstance().getBlockchain(homeCommunity.uuid)
if (
!(await clients.gradidoNode.getTransaction({
transactionId: 1,
topic: homeCommunity.hieroTopicId,
communityId: homeCommunity.uuid,
}))
) {
// if not exist, create community root transaction

23
dlt-connector/src/bootstrap/initGradidoNode.ts
View File

@ -7,11 +7,7 @@ import { exportCommunities } from '../client/GradidoNode/communities'
import { GradidoNodeProcess } from '../client/GradidoNode/GradidoNodeProcess'
import { HieroClient } from '../client/hiero/HieroClient'
import { CONFIG } from '../config'
import {
GRADIDO_NODE_HOME_FOLDER_NAME,
GRADIDO_NODE_RUNTIME_PATH,
LOG4JS_BASE_CATEGORY,
} from '../config/const'
import { GRADIDO_NODE_HOME_FOLDER_NAME, LOG4JS_BASE_CATEGORY } from '../config/const'
import { checkFileExist, checkPathExist } from '../utils/filesystem'
import { isPortOpen } from '../utils/network'
import { AppContextClients } from './appContext'
@ -37,7 +33,7 @@ export async function initGradidoNode(clients: AppContextClients): Promise<void>
// write Hedera Address Book
exportHederaAddressbooks(gradidoNodeHomeFolder, clients.hiero),
// check GradidoNode Runtime, download when missing
ensureGradidoNodeRuntimeAvailable(GRADIDO_NODE_RUNTIME_PATH),
ensureGradidoNodeRuntimeAvailable(GradidoNodeProcess.getRuntimePathFileName()),
// export communities to GradidoNode Folder
exportCommunities(gradidoNodeHomeFolder, clients.backend),
])
@ -57,11 +53,22 @@ async function exportHederaAddressbooks(
async function ensureGradidoNodeRuntimeAvailable(runtimeFileName: string): Promise<void> {
const runtimeFolder = path.dirname(runtimeFileName)
const wantedVersion = `v${CONFIG.DLT_GRADIDO_NODE_SERVER_VERSION}`
checkPathExist(runtimeFolder, true)
if (!checkFileExist(runtimeFileName)) {
let versionMatch = false
const isFileExist = checkFileExist(runtimeFileName)
if (isFileExist) {
const foundVersion = await GradidoNodeProcess.checkRuntimeVersion()
if (wantedVersion !== foundVersion) {
logger.info(`GradidoNode version detected: ${foundVersion}, required: ${wantedVersion}`)
} else {
versionMatch = true
}
}
if (!isFileExist || !versionMatch) {
const runtimeArchiveFilename = createGradidoNodeRuntimeArchiveFilename()
const downloadUrl = new URL(
`https://github.com/gradido/gradido_node/releases/download/v${CONFIG.DLT_GRADIDO_NODE_SERVER_VERSION}/${runtimeArchiveFilename}`,
`https://github.com/gradido/gradido_node/releases/download/${wantedVersion}/${runtimeArchiveFilename}`,
)
logger.debug(`download GradidoNode Runtime from ${downloadUrl}`)
const archive = await fetch(downloadUrl)

10
dlt-connector/src/cache/KeyPairCacheManager.ts vendored
View File

@ -76,4 +76,14 @@ export class KeyPairCacheManager {
}
return keyPair
}
public getKeyPairSync(input: string, createKeyPair: () => KeyPairEd25519): KeyPairEd25519 {
const keyPair = this.cache.get(input)
if (!keyPair) {
const keyPair = createKeyPair()
this.cache.set(input, keyPair)
return keyPair
}
return keyPair
}
}

44
dlt-connector/src/client/GradidoNode/GradidoNodeClient.ts
View File

@ -8,7 +8,7 @@ import { LOG4JS_BASE_CATEGORY } from '../../config/const'
import { AddressType } from '../../data/AddressType.enum'
import { Uuidv4Hash } from '../../data/Uuidv4Hash'
import { addressTypeSchema, confirmedTransactionSchema } from '../../schemas/typeConverter.schema'
import { Hex32, Hex32Input, HieroId, hex32Schema } from '../../schemas/typeGuard.schema'
import { Hex32, Hex32Input, hex32Schema, Uuidv4 } from '../../schemas/typeGuard.schema'
import { isPortOpenRetry } from '../../utils/network'
import { GradidoNodeErrorCodes } from './GradidoNodeErrorCodes'
import {
@ -75,7 +75,10 @@ export class GradidoNodeClient {
const response = await this.rpcCall<{ transaction: string }>('getTransaction', parameter)
if (response.isSuccess()) {
// this.logger.debug('result: ', response.result.transaction)
return v.parse(confirmedTransactionSchema, response.result.transaction)
return v.parse(confirmedTransactionSchema, {
base64: response.result.transaction,
communityId: parameter.communityId,
})
}
if (response.isError()) {
if (response.error.code === GradidoNodeErrorCodes.TRANSACTION_NOT_FOUND) {
@ -88,19 +91,22 @@ export class GradidoNodeClient {
/**
* getLastTransaction
* get the last confirmed transaction from a specific community
* @param hieroTopic the community hiero topic id
* @param communityId the community id
* @returns the last confirmed transaction or undefined if blockchain for community is empty or not found
* @throws GradidoNodeRequestError
*/
public async getLastTransaction(hieroTopic: HieroId): Promise<ConfirmedTransaction | undefined> {
public async getLastTransaction(communityId: Uuidv4): Promise<ConfirmedTransaction | undefined> {
const parameter = {
format: 'base64',
topic: hieroTopic,
communityId,
}
const response = await this.rpcCall<{ transaction: string }>('getLastTransaction', parameter)
if (response.isSuccess()) {
return v.parse(confirmedTransactionSchema, response.result.transaction)
return v.parse(confirmedTransactionSchema, {
base64: response.result.transaction,
communityId: parameter.communityId,
})
}
if (response.isError()) {
if (response.error.code === GradidoNodeErrorCodes.GRADIDO_NODE_ERROR) {
@ -115,7 +121,7 @@ export class GradidoNodeClient {
* get list of confirmed transactions from a specific community
* @param input fromTransactionId is the id of the first transaction to return
* @param input maxResultCount is the max number of transactions to return
* @param input topic is the community hiero topic id
* @param input communityId is the community id
* @returns list of confirmed transactions
* @throws GradidoNodeRequestError
* @example
@ -123,7 +129,7 @@ export class GradidoNodeClient {
* const transactions = await getTransactions({
* fromTransactionId: 1,
* maxResultCount: 100,
* topic: communityUuid,
* communityId: communityUuid,
* })
* ```
*/
@ -137,7 +143,10 @@ export class GradidoNodeClient {
parameter,
)
return result.transactions.map((transactionBase64) =>
v.parse(confirmedTransactionSchema, transactionBase64),
v.parse(confirmedTransactionSchema, {
base64: transactionBase64,
communityId: parameter.communityId,
}),
)
}
@ -163,7 +172,10 @@ export class GradidoNodeClient {
parameter,
)
return response.transactions.map((transactionBase64) =>
v.parse(confirmedTransactionSchema, transactionBase64),
v.parse(confirmedTransactionSchema, {
base64: transactionBase64,
communityId: parameter.communityId,
}),
)
}
@ -173,15 +185,15 @@ export class GradidoNodeClient {
* can be used to check if user/account exists on blockchain
* look also for gmw, auf and deferred transfer accounts
* @param pubkey the public key of the user or account
* @param hieroTopic the community hiero topic id
* @param communityId the community id
* @returns the address type of the user/account, AddressType.NONE if not found
* @throws GradidoNodeRequestError
*/
public async getAddressType(pubkey: Hex32Input, hieroTopic: HieroId): Promise<AddressType> {
public async getAddressType(pubkey: Hex32Input, communityId: Uuidv4): Promise<AddressType> {
const parameter = {
pubkey: v.parse(hex32Schema, pubkey),
topic: hieroTopic,
communityId,
}
const response = await this.rpcCallResolved<{ addressType: string }>(
'getAddressType',
@ -194,17 +206,17 @@ export class GradidoNodeClient {
* findUserByNameHash
* find a user by name hash
* @param nameHash the name hash of the user
* @param hieroTopic the community hiero topic id
* @param communityId the community id
* @returns the public key of the user as hex32 string or undefined if user is not found
* @throws GradidoNodeRequestError
*/
public async findUserByNameHash(
nameHash: Uuidv4Hash,
hieroTopic: HieroId,
communityId: Uuidv4,
): Promise<Hex32 | undefined> {
const parameter = {
nameHash: nameHash.getAsHexString(),
topic: hieroTopic,
communityId,
}
const response = await this.rpcCall<{ pubkey: string; timeUsed: string }>(
'findUserByNameHash',

21
dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts
View File

@ -1,12 +1,12 @@
import path from 'node:path'
import { Mutex } from 'async-mutex'
import { Subprocess, spawn } from 'bun'
import { $, Subprocess, spawn } from 'bun'
import { getLogger, Logger } from 'log4js'
import { CONFIG } from '../../config'
import {
GRADIDO_NODE_KILL_TIMEOUT_MILLISECONDS,
GRADIDO_NODE_MIN_RUNTIME_BEFORE_EXIT_MILLISECONDS,
GRADIDO_NODE_MIN_RUNTIME_BEFORE_RESTART_MILLISECONDS,
GRADIDO_NODE_RUNTIME_PATH,
LOG4JS_BASE_CATEGORY,
} from '../../config/const'
import { delay } from '../../utils/time'
@ -43,20 +43,33 @@ export class GradidoNodeProcess {
return GradidoNodeProcess.instance
}
public static getRuntimePathFileName(): string {
const isWindows = process.platform === 'win32'
const binaryName = isWindows ? 'GradidoNode.exe' : 'GradidoNode'
return path.join(CONFIG.DLT_GRADIDO_NODE_SERVER_HOME_FOLDER, 'bin', binaryName)
}
public static async checkRuntimeVersion(): Promise<string> {
return (await $`${GradidoNodeProcess.getRuntimePathFileName()} --version`.text()).trim()
}
public start() {
if (this.proc) {
this.logger.warn('GradidoNodeProcess already running.')
return
}
this.logger.info(`starting GradidoNodeProcess with path: ${GRADIDO_NODE_RUNTIME_PATH}`)
const gradidoNodeRuntimePath = GradidoNodeProcess.getRuntimePathFileName()
this.logger.info(`starting GradidoNodeProcess with path: ${gradidoNodeRuntimePath}`)
this.lastStarted = new Date()
const logger = this.logger
this.proc = spawn([GRADIDO_NODE_RUNTIME_PATH], {
this.proc = spawn([gradidoNodeRuntimePath], {
env: {
CLIENTS_HIERO_NETWORKTYPE: CONFIG.HIERO_HEDERA_NETWORK,
SERVER_JSON_RPC_PORT: CONFIG.DLT_NODE_SERVER_PORT.toString(),
USERPROFILE: CONFIG.DLT_GRADIDO_NODE_SERVER_HOME_FOLDER,
HOME: CONFIG.DLT_GRADIDO_NODE_SERVER_HOME_FOLDER,
UNSECURE_ALLOW_CORS_ALL: CONFIG.DLT_GRADIDO_NODE_SERVER_ALLOW_CORS ? '1' : '0',
},
onExit(_proc, exitCode, signalCode, error) {
logger.warn(`GradidoNodeProcess exited with code ${exitCode} and signalCode ${signalCode}`)

20
dlt-connector/src/client/GradidoNode/communities.ts
View File

@ -5,7 +5,7 @@ import { getLogger } from 'log4js'
import { CONFIG } from '../../config'
import { GRADIDO_NODE_HOME_FOLDER_NAME, LOG4JS_BASE_CATEGORY } from '../../config/const'
import { HieroId } from '../../schemas/typeGuard.schema'
import { checkFileExist, checkPathExist } from '../../utils/filesystem'
import { checkFileExist, checkPathExist, toFolderName } from '../../utils/filesystem'
import { BackendClient } from '../backend/BackendClient'
import { GradidoNodeProcess } from './GradidoNodeProcess'
@ -15,7 +15,7 @@ const ensureCommunitiesAvailableMutex: Mutex = new Mutex()
// prototype, later add api call to gradido dlt node server for adding/updating communities
type CommunityForDltNodeServer = {
communityId: string
hieroTopicId: string
hieroTopicId?: string | null
alias: string
folder: string
}
@ -38,28 +38,20 @@ export async function ensureCommunitiesAvailable(communityTopicIds: HieroId[]):
}
export async function exportCommunities(homeFolder: string, client: BackendClient): Promise<void> {
const communities = await client.getReachableCommunities()
const communities = await client.getAuthorizedCommunities()
const communitiesPath = path.join(homeFolder, 'communities.json')
checkPathExist(path.dirname(communitiesPath), true)
// make sure communityName is unique
const communityName = new Set<string>()
const communitiesForDltNodeServer: CommunityForDltNodeServer[] = []
for (const com of communities) {
if (!com.uuid || !com.hieroTopicId) {
if (!com.uuid) {
continue
}
// use name as alias if not empty and unique, otherwise use uuid
let alias = com.name
if (!alias || communityName.has(alias)) {
alias = com.uuid
}
communityName.add(alias)
communitiesForDltNodeServer.push({
communityId: com.uuid,
hieroTopicId: com.hieroTopicId,
alias,
alias: com.name,
// use only alpha-numeric chars for folder name
folder: alias.replace(/[^a-zA-Z0-9]/g, '_'),
folder: toFolderName(com.uuid),
})
}
fs.writeFileSync(communitiesPath, JSON.stringify(communitiesForDltNodeServer, null, 2))

24
dlt-connector/src/client/GradidoNode/input.schema.test.ts
View File

@ -1,18 +1,20 @@
import { beforeAll, describe, expect, it } from 'bun:test'
import { v4 as uuidv4 } from 'uuid'
import * as v from 'valibot'
import {
HieroId,
HieroTransactionIdString,
hieroIdSchema,
hieroTransactionIdStringSchema,
Uuidv4,
uuidv4Schema,
} from '../../schemas/typeGuard.schema'
import { transactionIdentifierSchema } from './input.schema'
let topic: HieroId
const topicString = '0.0.261'
let communityId: Uuidv4
const uuidv4String = uuidv4()
let hieroTransactionId: HieroTransactionIdString
beforeAll(() => {
topic = v.parse(hieroIdSchema, topicString)
communityId = v.parse(uuidv4Schema, uuidv4String)
hieroTransactionId = v.parse(hieroTransactionIdStringSchema, '0.0.261-1755348116-1281621')
})
@ -21,39 +23,39 @@ describe('transactionIdentifierSchema ', () => {
expect(
v.parse(transactionIdentifierSchema, {
transactionId: 1,
topic: topicString,
communityId,
}),
).toEqual({
transactionId: 1,
hieroTransactionId: undefined,
topic,
communityId,
})
})
it('valid, transaction identified by hieroTransactionId and topic', () => {
expect(
v.parse(transactionIdentifierSchema, {
hieroTransactionId: '0.0.261-1755348116-1281621',
topic: topicString,
communityId,
}),
).toEqual({
hieroTransactionId,
topic,
communityId,
})
})
it('invalid, missing topic', () => {
it('invalid, missing communityId', () => {
expect(() =>
v.parse(transactionIdentifierSchema, {
transactionId: 1,
hieroTransactionId: '0.0.261-1755348116-1281621',
}),
).toThrowError(new Error('Invalid key: Expected "topic" but received undefined'))
).toThrowError(new Error('Invalid key: Expected "communityId" but received undefined'))
})
it('invalid, transactionNr and iotaMessageId set', () => {
expect(() =>
v.parse(transactionIdentifierSchema, {
transactionId: 1,
hieroTransactionId: '0.0.261-1755348116-1281621',
topic,
communityId,
}),
).toThrowError(new Error('expect transactionNr or hieroTransactionId not both'))
})

6
dlt-connector/src/client/GradidoNode/input.schema.ts
View File

@ -1,12 +1,12 @@
import * as v from 'valibot'
import { hieroIdSchema, hieroTransactionIdStringSchema } from '../../schemas/typeGuard.schema'
import { hieroTransactionIdStringSchema, uuidv4Schema } from '../../schemas/typeGuard.schema'
export const transactionsRangeSchema = v.object({
// default value is 1, from first transactions
fromTransactionId: v.nullish(v.pipe(v.number(), v.minValue(1, 'expect number >= 1')), 1),
// default value is 100, max 100 transactions
maxResultCount: v.nullish(v.pipe(v.number(), v.minValue(1, 'expect number >= 1')), 100),
topic: hieroIdSchema,
communityId: uuidv4Schema,
})
export type TransactionsRangeInput = v.InferInput<typeof transactionsRangeSchema>
@ -19,7 +19,7 @@ export const transactionIdentifierSchema = v.pipe(
undefined,
),
hieroTransactionId: v.nullish(hieroTransactionIdStringSchema, undefined),
topic: hieroIdSchema,
communityId: uuidv4Schema,
}),
v.custom((value: any) => {
const setFieldsCount =

14
dlt-connector/src/client/backend/BackendClient.ts
View File

@ -6,6 +6,7 @@ import { CONFIG } from '../../config'
import { LOG4JS_BASE_CATEGORY } from '../../config/const'
import { HieroId, Uuidv4 } from '../../schemas/typeGuard.schema'
import {
getAuthorizedCommunities,
getReachableCommunities,
homeCommunityGraphqlQuery,
setHomeCommunityTopicId,
@ -101,6 +102,19 @@ export class BackendClient {
return v.parse(v.array(communitySchema), data.reachableCommunities)
}
public async getAuthorizedCommunities(): Promise<Community[]> {
this.logger.info('get authorized communities on backend')
const { data, errors } = await this.client.rawRequest<{ authorizedCommunities: Community[] }>(
getAuthorizedCommunities,
{},
await this.getRequestHeader(),
)
if (errors) {
throw errors[0]
}
return v.parse(v.array(communitySchema), data.authorizedCommunities)
}
private async getRequestHeader(): Promise<{
authorization: string
}> {

9
dlt-connector/src/client/backend/graphql.ts
View File

@ -44,3 +44,12 @@ export const getReachableCommunities = gql`
}
${communityFragment}
`
export const getAuthorizedCommunities = gql`
query {
authorizedCommunities {
...Community_common
}
}
${communityFragment}
`

9
dlt-connector/src/client/hiero/HieroClient.ts
View File

@ -20,7 +20,7 @@ import { getLogger, Logger } from 'log4js'
import * as v from 'valibot'
import { CONFIG } from '../../config'
import { LOG4JS_BASE_CATEGORY } from '../../config/const'
import { HieroId, hieroIdSchema } from '../../schemas/typeGuard.schema'
import { HieroId, hieroIdSchema, Uuidv4 } from '../../schemas/typeGuard.schema'
import { durationInMinutesFromDates, printTimeDuration } from '../../utils/time'
import { GradidoNodeClient } from '../GradidoNode/GradidoNodeClient'
import { GradidoNodeProcess } from '../GradidoNode/GradidoNodeProcess'
@ -72,6 +72,7 @@ export class HieroClient {
public async sendMessage(
topicId: HieroId,
communityId: Uuidv4,
transaction: GradidoTransaction,
): Promise<TransactionId | null> {
const timeUsed = new Profiler()
@ -99,10 +100,10 @@ export class HieroClient {
)
// TODO: fix issue in GradidoNode
// hot fix, when gradido node is running some time, the hiero listener stop working, so we check if our new transaction is received
// after 10 seconds, else restart GradidoNode
// after 20 seconds, else restart GradidoNode
setTimeout(async () => {
const transaction = await GradidoNodeClient.getInstance().getTransaction({
topic: topicId,
communityId,
hieroTransactionId: sendResponse.transactionId.toString(),
})
if (!transaction) {
@ -121,7 +122,7 @@ export class HieroClient {
GradidoNodeProcess.getInstance().start()
}
}
}, 10000)
}, 20000)
if (logger.isInfoEnabled()) {
// only for logging
sendResponse.getReceiptWithSigner(this.wallet).then((receipt) => {

2
dlt-connector/src/config/const.ts
View File

@ -17,6 +17,6 @@ export const GRADIDO_NODE_RUNTIME_PATH = path.join(
// if last start was less than this time, do not restart
export const GRADIDO_NODE_MIN_RUNTIME_BEFORE_RESTART_MILLISECONDS = 1000 * 30
export const GRADIDO_NODE_MIN_RUNTIME_BEFORE_EXIT_MILLISECONDS = 1000 * 2
export const GRADIDO_NODE_KILL_TIMEOUT_MILLISECONDS = 10000
export const GRADIDO_NODE_KILL_TIMEOUT_MILLISECONDS = 60 * 1000
// currently hard coded in gradido node, update in future
export const GRADIDO_NODE_HOME_FOLDER_NAME = '.gradido'

11
dlt-connector/src/config/schema.ts
View File

@ -82,14 +82,21 @@ export const configSchema = v.object({
DLT_GRADIDO_NODE_SERVER_VERSION: v.optional(
v.pipe(
v.string('The version of the DLT node server, for example: 0.9.0'),
v.regex(/^\d+\.\d+\.\d+$/),
v.regex(/^\d+\.\d+\.\d+(.\d+)?$/),
),
'0.9.2',
'0.9.6.10',
),
DLT_GRADIDO_NODE_SERVER_HOME_FOLDER: v.optional(
v.string('The home folder for the gradido dlt node server'),
path.join(__dirname, '..', '..', 'gradido_node'),
),
DLT_GRADIDO_NODE_SERVER_ALLOW_CORS: v.optional(
v.pipe(
v.string('Whether to allow CORS for the gradido dlt node server'),
v.transform<string, boolean>((input: string) => input === 'true'),
),
'false',
),
BACKEND_PORT: v.optional(
v.pipe(
v.string('A valid port on which the backend server is running'),

7
dlt-connector/src/data/KeyPairIdentifier.logic.ts
View File

@ -54,6 +54,9 @@ export class KeyPairIdentifierLogic {
return this.identifier.seed
}
getCommunityId(): Uuidv4 {
return this.identifier.communityId
}
getCommunityTopicId(): HieroId {
return this.identifier.communityTopicId
}
@ -76,7 +79,7 @@ export class KeyPairIdentifierLogic {
return this.getSeed()
}
getCommunityKey(): string {
return this.getCommunityTopicId()
return this.getCommunityId()
}
getCommunityUserKey(): string {
return this.deriveCommunityUserHash(0)
@ -107,7 +110,7 @@ export class KeyPairIdentifierLogic {
)
}
const resultString =
this.identifier.communityTopicId +
this.identifier.communityId +
this.identifier.account.userUuid.replace(/-/g, '') +
accountNr.toString()
return new MemoryBlock(resultString).calculateHash().convertToHex()

46
dlt-connector/src/data/deriveKeyPair.ts Normal file
View File

@ -0,0 +1,46 @@
import { KeyPairEd25519, MemoryBlock } from 'gradido-blockchain-js'
import { GradidoBlockchainCryptoError, ParameterError } from '../errors'
import { Hex32, Uuidv4 } from '../schemas/typeGuard.schema'
import { hardenDerivationIndex } from '../utils/derivationHelper'
export function deriveFromSeed(seed: Hex32): KeyPairEd25519 {
const keyPair = KeyPairEd25519.create(MemoryBlock.fromHex(seed))
if (!keyPair) {
throw new Error(`couldn't create keyPair from seed: ${seed}`)
}
return keyPair
}
export function deriveFromCode(code: string): KeyPairEd25519 {
// code is expected to be 24 bytes long, but we need 32
// so hash the seed with blake2 and we have 32 Bytes
const hash = new MemoryBlock(code).calculateHash()
const keyPair = KeyPairEd25519.create(hash)
if (!keyPair) {
throw new ParameterError(`error creating Ed25519 KeyPair from seed: ${code.substring(0, 5)}...`)
}
return keyPair
}
export function deriveFromKeyPairAndUuid(keyPair: KeyPairEd25519, uuid: Uuidv4): KeyPairEd25519 {
const wholeHex = Buffer.from(uuid.replace(/-/g, ''), 'hex')
const parts = []
for (let i = 0; i < 4; i++) {
parts[i] = hardenDerivationIndex(wholeHex.subarray(i * 4, (i + 1) * 4).readUInt32BE())
}
// parts: [2206563009, 2629978174, 2324817329, 2405141782]
return parts.reduce(
(keyPair: KeyPairEd25519, node: number) => deriveFromKeyPairAndIndex(keyPair, node),
keyPair,
)
}
export function deriveFromKeyPairAndIndex(keyPair: KeyPairEd25519, index: number): KeyPairEd25519 {
const localKeyPair = keyPair.deriveChild(index)
if (!localKeyPair) {
throw new GradidoBlockchainCryptoError(
`KeyPairEd25519 child derivation failed, has private key: ${keyPair.hasPrivateKey()}, index: ${index}`,
)
}
return localKeyPair
}

3
dlt-connector/src/index.ts
View File

@ -21,6 +21,9 @@ async function main() {
// get home community, create topic if not exist, or check topic expiration and update it if needed
const homeCommunity = await checkHomeCommunity(appContext, logger)
if (!homeCommunity) {
process.exit(1)
}
// ask gradido node if community blockchain was created
// if not exist, create community root transaction

8
dlt-connector/src/interactions/resolveKeyPair/AbstractRemoteKeyPair.role.ts
View File

@ -1,10 +1,10 @@
import { KeyPairEd25519 } from 'gradido-blockchain-js'
import { HieroId } from '../../schemas/typeGuard.schema'
import { Uuidv4 } from '../../schemas/typeGuard.schema'
export abstract class AbstractRemoteKeyPairRole {
protected topic: HieroId
public constructor(communityTopicId: HieroId) {
this.topic = communityTopicId
protected communityId: Uuidv4
public constructor(communityId: Uuidv4) {
this.communityId = communityId
}
public abstract retrieveKeyPair(): Promise<KeyPairEd25519>
}

2
dlt-connector/src/interactions/resolveKeyPair/ForeignCommunityKeyPair.role.ts
View File

@ -10,7 +10,7 @@ export class ForeignCommunityKeyPairRole extends AbstractRemoteKeyPairRole {
public async retrieveKeyPair(): Promise<KeyPairEd25519> {
const transactionIdentifier = {
transactionId: 1,
topic: this.topic,
communityId: this.communityId,
}
const firstTransaction =
await GradidoNodeClient.getInstance().getTransaction(transactionIdentifier)

4
dlt-connector/src/interactions/resolveKeyPair/RemoteAccountKeyPair.role.ts
View File

@ -7,7 +7,7 @@ import { AbstractRemoteKeyPairRole } from './AbstractRemoteKeyPair.role'
export class RemoteAccountKeyPairRole extends AbstractRemoteKeyPairRole {
public constructor(private identifier: IdentifierAccount) {
super(identifier.communityTopicId)
super(identifier.communityId)
}
public async retrieveKeyPair(): Promise<KeyPairEd25519> {
@ -17,7 +17,7 @@ export class RemoteAccountKeyPairRole extends AbstractRemoteKeyPairRole {
const accountPublicKey = await GradidoNodeClient.getInstance().findUserByNameHash(
new Uuidv4Hash(this.identifier.account.userUuid),
this.topic,
this.communityId,
)
if (accountPublicKey) {
return new KeyPairEd25519(MemoryBlock.createPtr(MemoryBlock.fromHex(accountPublicKey)))

5
dlt-connector/src/interactions/resolveKeyPair/ResolveKeyPair.context.test.ts
View File

@ -33,6 +33,7 @@ mock.module('../../config', () => ({
}))
const topicId = '0.0.21732'
const communityId = '1e88a0f4-d4fc-4cae-a7e8-a88e613ce324'
const userUuid = 'aa25cf6f-2879-4745-b2ea-6d3c37fb44b0'
afterAll(() => {
@ -45,7 +46,7 @@ describe('KeyPairCalculation', () => {
})
it('community key pair', async () => {
const identifier = new KeyPairIdentifierLogic(
v.parse(identifierKeyPairSchema, { communityTopicId: topicId }),
v.parse(identifierKeyPairSchema, { communityId, communityTopicId: topicId }),
)
const keyPair = await ResolveKeyPair(identifier)
expect(keyPair.getPublicKey()?.convertToHex()).toBe(
@ -55,6 +56,7 @@ describe('KeyPairCalculation', () => {
it('user key pair', async () => {
const identifier = new KeyPairIdentifierLogic(
v.parse(identifierKeyPairSchema, {
communityId,
communityTopicId: topicId,
account: { userUuid },
}),
@ -70,6 +72,7 @@ describe('KeyPairCalculation', () => {
it('account key pair', async () => {
const identifier = new KeyPairIdentifierLogic(
v.parse(identifierKeyPairSchema, {
communityId,
communityTopicId: topicId,
account: { userUuid, accountNr: 1 },
}),

2
dlt-connector/src/interactions/resolveKeyPair/ResolveKeyPair.context.ts
View File

@ -45,7 +45,7 @@ export async function ResolveKeyPair(input: KeyPairIdentifierLogic): Promise<Key
if (cache.getHomeCommunityTopicId() !== input.getCommunityTopicId()) {
const role = input.isAccountKeyPair()
? new RemoteAccountKeyPairRole(input.identifier)
: new ForeignCommunityKeyPairRole(input.getCommunityTopicId())
: new ForeignCommunityKeyPairRole(input.getCommunityId())
return await role.retrieveKeyPair()
}
// Community

6
dlt-connector/src/interactions/sendToHiero/CommunityRootTransaction.role.ts
View File

@ -27,7 +27,10 @@ export class CommunityRootTransactionRole extends AbstractTransactionRole {
public async getGradidoTransactionBuilder(): Promise<GradidoTransactionBuilder> {
const builder = new GradidoTransactionBuilder()
const communityKeyPair = await ResolveKeyPair(
new KeyPairIdentifierLogic({ communityTopicId: this.community.hieroTopicId }),
new KeyPairIdentifierLogic({
communityTopicId: this.community.hieroTopicId,
communityId: this.community.uuid,
}),
)
const gmwKeyPair = communityKeyPair.deriveChild(
hardenDerivationIndex(GMW_ACCOUNT_DERIVATION_INDEX),
@ -47,6 +50,7 @@ export class CommunityRootTransactionRole extends AbstractTransactionRole {
}
builder
.setCreatedAt(this.community.creationDate)
.setSenderCommunity(this.community.uuid)
.setCommunityRoot(
communityKeyPair.getPublicKey(),
gmwKeyPair.getPublicKey(),

10
dlt-connector/src/interactions/sendToHiero/CreationTransaction.role.ts
View File

@ -36,7 +36,7 @@ export class CreationTransactionRole extends AbstractTransactionRole {
}
getRecipientCommunityTopicId(): HieroId {
throw new Error('creation: cannot be used as cross group transaction')
return this.creationTransaction.user.communityTopicId
}
public async getGradidoTransactionBuilder(): Promise<GradidoTransactionBuilder> {
@ -52,6 +52,7 @@ export class CreationTransactionRole extends AbstractTransactionRole {
const homeCommunityKeyPair = await ResolveKeyPair(
new KeyPairIdentifierLogic({
communityTopicId: this.homeCommunityTopicId,
communityId: this.creationTransaction.user.communityId,
}),
)
// Memo: encrypted, home community and recipient can decrypt it
@ -64,8 +65,13 @@ export class CreationTransactionRole extends AbstractTransactionRole {
new AuthenticatedEncryption(recipientKeyPair),
),
)
.setRecipientCommunity(this.creationTransaction.user.communityId)
.setTransactionCreation(
new TransferAmount(recipientKeyPair.getPublicKey(), this.creationTransaction.amount),
new TransferAmount(
recipientKeyPair.getPublicKey(),
this.creationTransaction.amount,
this.creationTransaction.user.communityId,
),
this.creationTransaction.targetDate,
)
.sign(signerKeyPair)

3
dlt-connector/src/interactions/sendToHiero/DeferredTransferTransaction.role.ts
View File

@ -41,6 +41,7 @@ export class DeferredTransferTransactionRole extends AbstractTransactionRole {
const recipientKeyPair = await ResolveKeyPair(
new KeyPairIdentifierLogic({
communityTopicId: this.deferredTransferTransaction.linkedUser.communityTopicId,
communityId: this.deferredTransferTransaction.linkedUser.communityId,
seed: this.seed,
}),
)
@ -54,6 +55,7 @@ export class DeferredTransferTransactionRole extends AbstractTransactionRole {
new AuthenticatedEncryption(recipientKeyPair),
),
)
.setSenderCommunity(this.deferredTransferTransaction.user.communityId)
.setDeferredTransfer(
new GradidoTransfer(
new TransferAmount(
@ -61,6 +63,7 @@ export class DeferredTransferTransactionRole extends AbstractTransactionRole {
this.deferredTransferTransaction.amount.calculateCompoundInterest(
this.deferredTransferTransaction.timeoutDuration.getSeconds(),
),
this.deferredTransferTransaction.user.communityId,
),
recipientKeyPair.getPublicKey(),
),

9
dlt-connector/src/interactions/sendToHiero/RedeemDeferredTransferTransaction.role.ts
View File

@ -59,12 +59,15 @@ export class RedeemDeferredTransferTransactionRole extends AbstractTransactionRo
builder
.setCreatedAt(this.redeemDeferredTransferTransaction.createdAt)
.setSenderCommunity(this.redeemDeferredTransferTransaction.user.communityId)
.setRecipientCommunity(this.linkedUser.communityId)
.setRedeemDeferredTransfer(
this.parentDeferredTransaction.getId(),
new GradidoTransfer(
new TransferAmount(
senderKeyPair.getPublicKey(),
this.redeemDeferredTransferTransaction.amount,
this.redeemDeferredTransferTransaction.user.communityId,
),
recipientKeyPair.getPublicKey(),
),
@ -73,12 +76,6 @@ export class RedeemDeferredTransferTransactionRole extends AbstractTransactionRo
for (let i = 0; i < memos.size(); i++) {
builder.addMemo(memos.get(i))
}
const senderCommunity = this.redeemDeferredTransferTransaction.user.communityTopicId
const recipientCommunity = this.linkedUser.communityTopicId
if (senderCommunity !== recipientCommunity) {
// we have a cross group transaction
builder.setSenderCommunity(senderCommunity).setRecipientCommunity(recipientCommunity)
}
builder.sign(senderKeyPair)
return builder
}

10
dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.test.ts
View File

@ -1,14 +1,20 @@
import { describe, expect, it } from 'bun:test'
import { InteractionValidate, ValidateType_SINGLE } from 'gradido-blockchain-js'
import {
InMemoryBlockchainProvider,
InteractionValidate,
ValidateType_SINGLE,
} from 'gradido-blockchain-js'
import * as v from 'valibot'
import { transactionSchema } from '../../schemas/transaction.schema'
import { hieroIdSchema } from '../../schemas/typeGuard.schema'
import { RegisterAddressTransactionRole } from './RegisterAddressTransaction.role'
const userUuid = '408780b2-59b3-402a-94be-56a4f4f4e8ec'
const communityId = '1e88a0f4-d4fc-4cae-a7e8-a88e613ce324'
const transaction = {
user: {
communityTopicId: '0.0.21732',
communityId,
account: {
userUuid,
accountNr: 0,
@ -18,6 +24,8 @@ const transaction = {
accountType: 'COMMUNITY_HUMAN',
createdAt: '2022-01-01T00:00:00.000Z',
}
// create blockchain in native module
InMemoryBlockchainProvider.getInstance().getBlockchain(communityId)
describe('RegisterAddressTransaction.role', () => {
it('get correct prepared builder', async () => {

8
dlt-connector/src/interactions/sendToHiero/RegisterAddressTransaction.role.ts
View File

@ -35,7 +35,12 @@ export class RegisterAddressTransactionRole extends AbstractTransactionRole {
public async getGradidoTransactionBuilder(): Promise<GradidoTransactionBuilder> {
const builder = new GradidoTransactionBuilder()
const communityTopicId = this.registerAddressTransaction.user.communityTopicId
const communityKeyPair = await ResolveKeyPair(new KeyPairIdentifierLogic({ communityTopicId }))
const communityKeyPair = await ResolveKeyPair(
new KeyPairIdentifierLogic({
communityTopicId,
communityId: this.registerAddressTransaction.user.communityId,
}),
)
const keyPairIdentifier = this.registerAddressTransaction.user
// when accountNr is 0 it is the user account
keyPairIdentifier.account.accountNr = 0
@ -45,6 +50,7 @@ export class RegisterAddressTransactionRole extends AbstractTransactionRole {
builder
.setCreatedAt(this.registerAddressTransaction.createdAt)
.setSenderCommunity(this.registerAddressTransaction.user.communityId)
.setRegisterAddress(
userKeyPair.getPublicKey(),
this.registerAddressTransaction.accountType as AddressType,

24
dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts
View File

@ -1,8 +1,8 @@
import {
GradidoTransaction,
HieroTransactionId,
InteractionSerialize,
InteractionValidate,
LedgerAnchor,
ValidateType_SINGLE,
} from 'gradido-blockchain-js'
import { getLogger } from 'log4js'
@ -23,6 +23,8 @@ import {
HieroTransactionIdString,
hieroTransactionIdStringSchema,
identifierSeedSchema,
Uuidv4,
uuidv4Schema,
} from '../../schemas/typeGuard.schema'
import { isTopicStillOpen } from '../../utils/hiero'
import { LinkedTransactionKeyPairRole } from '../resolveKeyPair/LinkedTransactionKeyPair.role'
@ -59,20 +61,24 @@ export async function SendToHieroContext(
const outboundHieroTransactionIdString = await sendViaHiero(
outboundTransaction,
role.getSenderCommunityTopicId(),
v.parse(uuidv4Schema, outboundTransaction.getCommunityId()),
)
// serialize Hiero transaction ID and attach it to the builder for the inbound transaction
const transactionIdSerializer = new InteractionSerialize(
new HieroTransactionId(outboundHieroTransactionIdString),
// attach Hiero transaction ID to the builder for the inbound transaction
builder.setParentLedgerAnchor(
new LedgerAnchor(new HieroTransactionId(outboundHieroTransactionIdString)),
)
builder.setParentMessageId(transactionIdSerializer.run())
// build and validate inbound transaction
const inboundTransaction = builder.buildInbound()
validate(inboundTransaction)
// send inbound transaction to hiero
await sendViaHiero(inboundTransaction, role.getRecipientCommunityTopicId())
await sendViaHiero(
inboundTransaction,
role.getRecipientCommunityTopicId(),
v.parse(uuidv4Schema, inboundTransaction.getCommunityId()),
)
return outboundHieroTransactionIdString
} else {
// build and validate local transaction
@ -83,6 +89,7 @@ export async function SendToHieroContext(
const hieroTransactionIdString = await sendViaHiero(
transaction,
role.getSenderCommunityTopicId(),
v.parse(uuidv4Schema, transaction.getCommunityId()),
)
return hieroTransactionIdString
}
@ -98,9 +105,10 @@ function validate(transaction: GradidoTransaction): void {
async function sendViaHiero(
gradidoTransaction: GradidoTransaction,
topic: HieroId,
communityId: Uuidv4,
): Promise<HieroTransactionIdString> {
const client = HieroClient.getInstance()
const transactionId = await client.sendMessage(topic, gradidoTransaction)
const transactionId = await client.sendMessage(topic, communityId, gradidoTransaction)
if (!transactionId) {
throw new Error('missing transaction id from hiero')
}
@ -156,7 +164,7 @@ async function chooseCorrectRole(
throw new Error("redeem deferred transfer: couldn't generate seed public key")
}
const transactions = await GradidoNodeClient.getInstance().getTransactionsForAccount(
{ maxResultCount: 2, topic: transaction.user.communityTopicId },
{ maxResultCount: 2, communityId: transaction.user.communityId },
seedPublicKey.convertToHex(),
)
if (!transactions || transactions.length !== 1) {

15
dlt-connector/src/interactions/sendToHiero/TransferTransaction.role.ts
View File

@ -40,7 +40,6 @@ export class TransferTransactionRole extends AbstractTransactionRole {
const recipientKeyPair = await ResolveKeyPair(
new KeyPairIdentifierLogic(this.transferTransaction.linkedUser),
)
builder
.setCreatedAt(this.transferTransaction.createdAt)
.addMemo(
@ -50,16 +49,16 @@ export class TransferTransactionRole extends AbstractTransactionRole {
new AuthenticatedEncryption(recipientKeyPair),
),
)
.setSenderCommunity(this.transferTransaction.user.communityId)
.setRecipientCommunity(this.transferTransaction.linkedUser.communityId)
.setTransactionTransfer(
new TransferAmount(senderKeyPair.getPublicKey(), this.transferTransaction.amount),
new TransferAmount(
senderKeyPair.getPublicKey(),
this.transferTransaction.amount,
this.transferTransaction.user.communityId,
),
recipientKeyPair.getPublicKey(),
)
const senderCommunity = this.transferTransaction.user.communityTopicId
const recipientCommunity = this.transferTransaction.linkedUser.communityTopicId
if (senderCommunity !== recipientCommunity) {
// we have a cross group transaction
builder.setSenderCommunity(senderCommunity).setRecipientCommunity(recipientCommunity)
}
builder.sign(senderKeyPair)
return builder
}

181
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts
View File

@ -1,181 +0,0 @@
import {
Filter,
GradidoTransactionBuilder,
HieroAccountId,
HieroTransactionId,
InMemoryBlockchain,
InteractionSerialize,
Timestamp,
} from 'gradido-blockchain-js'
import { getLogger } from 'log4js'
import * as v from 'valibot'
import { LOG4JS_BASE_CATEGORY } from '../../config/const'
import { InputTransactionType } from '../../data/InputTransactionType.enum'
import { LinkedTransactionKeyPairRole } from '../../interactions/resolveKeyPair/LinkedTransactionKeyPair.role'
import { CommunityRootTransactionRole } from '../../interactions/sendToHiero/CommunityRootTransaction.role'
import { CreationTransactionRole } from '../../interactions/sendToHiero/CreationTransaction.role'
import { DeferredTransferTransactionRole } from '../../interactions/sendToHiero/DeferredTransferTransaction.role'
import { RedeemDeferredTransferTransactionRole } from '../../interactions/sendToHiero/RedeemDeferredTransferTransaction.role'
import { RegisterAddressTransactionRole } from '../../interactions/sendToHiero/RegisterAddressTransaction.role'
import { TransferTransactionRole } from '../../interactions/sendToHiero/TransferTransaction.role'
import { Community, Transaction } from '../../schemas/transaction.schema'
import { identifierSeedSchema } from '../../schemas/typeGuard.schema'
const logger = getLogger(
`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`,
)
export const defaultHieroAccount = new HieroAccountId(0, 0, 2)
function addToBlockchain(
builder: GradidoTransactionBuilder,
blockchain: InMemoryBlockchain,
createdAtTimestamp: Timestamp,
): boolean {
const transaction = builder.build()
// TOD: use actual transaction id if exist in dlt_transactions table
const transactionId = new HieroTransactionId(createdAtTimestamp, defaultHieroAccount)
const interactionSerialize = new InteractionSerialize(transactionId)
try {
const result = blockchain.createAndAddConfirmedTransaction(
transaction,
interactionSerialize.run(),
createdAtTimestamp,
)
return result
} catch (error) {
logger.error(`Transaction ${transaction.toJson(true)} not added: ${error}`)
return false
}
}
export async function addCommunityRootTransaction(
blockchain: InMemoryBlockchain,
community: Community,
): Promise<void> {
const communityRootTransactionRole = new CommunityRootTransactionRole(community)
if (
addToBlockchain(
await communityRootTransactionRole.getGradidoTransactionBuilder(),
blockchain,
new Timestamp(community.creationDate),
)
) {
logger.info(`Community Root Transaction added`)
} else {
throw new Error(`Community Root Transaction not added`)
}
}
export async function addRegisterAddressTransaction(
blockchain: InMemoryBlockchain,
transaction: Transaction,
): Promise<void> {
const registerAddressRole = new RegisterAddressTransactionRole(transaction)
if (
addToBlockchain(
await registerAddressRole.getGradidoTransactionBuilder(),
blockchain,
new Timestamp(transaction.createdAt),
)
) {
logger.debug(
`Register Address Transaction added for user ${transaction.user.account!.userUuid}`,
)
} else {
throw new Error(
`Register Address Transaction not added for user ${transaction.user.account!.userUuid}`,
)
}
}
export async function addTransaction(
senderBlockchain: InMemoryBlockchain,
_recipientBlockchain: InMemoryBlockchain,
transaction: Transaction,
): Promise<void> {
const createdAtTimestamp = new Timestamp(transaction.createdAt)
if (transaction.type === InputTransactionType.GRADIDO_CREATION) {
const creationTransactionRole = new CreationTransactionRole(transaction)
if (
addToBlockchain(
await creationTransactionRole.getGradidoTransactionBuilder(),
senderBlockchain,
createdAtTimestamp,
)
) {
logger.debug(`Creation Transaction added for user ${transaction.user.account!.userUuid}`)
} else {
throw new Error(
`Creation Transaction not added for user ${transaction.user.account!.userUuid}`,
)
}
} else if (transaction.type === InputTransactionType.GRADIDO_TRANSFER) {
const transferTransactionRole = new TransferTransactionRole(transaction)
// will crash with cross group transaction
if (
addToBlockchain(
await transferTransactionRole.getGradidoTransactionBuilder(),
senderBlockchain,
createdAtTimestamp,
)
) {
logger.debug(`Transfer Transaction added for user ${transaction.user.account!.userUuid}`)
} else {
throw new Error(
`Transfer Transaction not added for user ${transaction.user.account!.userUuid}`,
)
}
} else if (transaction.type === InputTransactionType.GRADIDO_DEFERRED_TRANSFER) {
const transferTransactionRole = new DeferredTransferTransactionRole(transaction)
if (
addToBlockchain(
await transferTransactionRole.getGradidoTransactionBuilder(),
senderBlockchain,
createdAtTimestamp,
)
) {
logger.debug(
`Deferred Transfer Transaction added for user ${transaction.user.account!.userUuid}`,
)
} else {
throw new Error(
`Deferred Transfer Transaction not added for user ${transaction.user.account!.userUuid}`,
)
}
} else if (transaction.type === InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER) {
const seedKeyPairRole = new LinkedTransactionKeyPairRole(
v.parse(identifierSeedSchema, transaction.user.seed),
)
const f = new Filter()
f.involvedPublicKey = seedKeyPairRole.generateKeyPair().getPublicKey()
const deferredTransaction = senderBlockchain.findOne(f)
if (!deferredTransaction) {
throw new Error(
`redeem deferred transfer: couldn't find parent deferred transfer on Gradido Node for ${JSON.stringify(transaction, null, 2)} and public key from seed: ${f.involvedPublicKey?.convertToHex()}`,
)
}
const confirmedDeferredTransaction = deferredTransaction.getConfirmedTransaction()
if (!confirmedDeferredTransaction) {
throw new Error('redeem deferred transfer: invalid TransactionEntry')
}
const redeemTransactionRole = new RedeemDeferredTransferTransactionRole(
transaction,
confirmedDeferredTransaction,
)
const involvedUser = transaction.user.account
? transaction.user.account.userUuid
: transaction.linkedUser?.account?.userUuid
if (
addToBlockchain(
await redeemTransactionRole.getGradidoTransactionBuilder(),
senderBlockchain,
createdAtTimestamp,
)
) {
logger.debug(`Redeem Deferred Transfer Transaction added for user ${involvedUser}`)
} else {
throw new Error(`Redeem Deferred Transfer Transaction not added for user ${involvedUser}`)
}
}
}

55
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/bootstrap.ts
View File

@ -1,55 +0,0 @@
import { InMemoryBlockchainProvider } from 'gradido-blockchain-js'
import * as v from 'valibot'
import { HieroId, hieroIdSchema } from '../../schemas/typeGuard.schema'
import { addCommunityRootTransaction } from './blockchain'
import { Context } from './Context'
import { communityDbToCommunity } from './convert'
import { loadCommunities } from './database'
import { generateKeyPairCommunity } from './keyPair'
import { CommunityContext } from './valibot.schema'
export async function bootstrap(): Promise<Context> {
const context = await Context.create()
context.communities = await bootstrapCommunities(context)
return context
}
async function bootstrapCommunities(context: Context): Promise<Map<string, CommunityContext>> {
const communities = new Map<string, CommunityContext>()
const communitiesDb = await loadCommunities(context.db)
const topicIds = new Set<HieroId>()
for (const communityDb of communitiesDb) {
const blockchain = InMemoryBlockchainProvider.getInstance().findBlockchain(
communityDb.uniqueAlias,
)
if (!blockchain) {
throw new Error(`Couldn't create Blockchain for community ${communityDb.communityUuid}`)
}
context.logger.info(`Blockchain for community '${communityDb.uniqueAlias}' created`)
// make sure topic id is unique
let topicId: HieroId
do {
topicId = v.parse(hieroIdSchema, '0.0.' + Math.floor(Math.random() * 10000))
} while (topicIds.has(topicId))
topicIds.add(topicId)
communities.set(communityDb.communityUuid, {
communityId: communityDb.uniqueAlias,
blockchain,
topicId,
folder: communityDb.uniqueAlias.replace(/[^a-zA-Z0-9]/g, '_'),
})
generateKeyPairCommunity(communityDb, context.cache, topicId)
let creationDate = communityDb.creationDate
if (communityDb.userMinCreatedAt && communityDb.userMinCreatedAt < communityDb.creationDate) {
// create community root transaction 1 minute before first user
creationDate = new Date(new Date(communityDb.userMinCreatedAt).getTime() - 1000 * 60)
}
// community from db to community format the dlt connector normally uses
const community = communityDbToCommunity(topicId, communityDb, creationDate)
await addCommunityRootTransaction(blockchain, community)
}
return communities
}

136
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/convert.ts
View File

@ -1,136 +0,0 @@
import * as v from 'valibot'
import { AccountType } from '../../data/AccountType.enum'
import { InputTransactionType } from '../../data/InputTransactionType.enum'
import {
Community,
communitySchema,
Transaction,
TransactionInput,
transactionSchema,
} from '../../schemas/transaction.schema'
import {
gradidoAmountSchema,
HieroId,
memoSchema,
timeoutDurationSchema,
} from '../../schemas/typeGuard.schema'
import { TransactionTypeId } from './TransactionTypeId'
import { CommunityDb, CreatedUserDb, TransactionDb, TransactionLinkDb } from './valibot.schema'
export function getInputTransactionTypeFromTypeId(typeId: TransactionTypeId): InputTransactionType {
switch (typeId) {
case TransactionTypeId.CREATION:
return InputTransactionType.GRADIDO_CREATION
case TransactionTypeId.SEND:
return InputTransactionType.GRADIDO_TRANSFER
case TransactionTypeId.RECEIVE:
throw new Error('not used')
default:
throw new Error('not implemented')
}
}
export function communityDbToCommunity(
topicId: HieroId,
communityDb: CommunityDb,
creationDate: Date,
): Community {
return v.parse(communitySchema, {
hieroTopicId: topicId,
uuid: communityDb.communityUuid,
foreign: communityDb.foreign,
creationDate,
})
}
export function userDbToTransaction(userDb: CreatedUserDb, communityTopicId: HieroId): Transaction {
return v.parse(transactionSchema, {
user: {
communityTopicId: communityTopicId,
account: { userUuid: userDb.gradidoId },
},
type: InputTransactionType.REGISTER_ADDRESS,
accountType: AccountType.COMMUNITY_HUMAN,
createdAt: userDb.createdAt,
})
}
export function transactionDbToTransaction(
transactionDb: TransactionDb,
communityTopicId: HieroId,
recipientCommunityTopicId: HieroId,
): Transaction {
if (
transactionDb.typeId !== TransactionTypeId.CREATION &&
transactionDb.typeId !== TransactionTypeId.SEND &&
transactionDb.typeId !== TransactionTypeId.RECEIVE
) {
throw new Error('not implemented')
}
const user = {
communityTopicId: communityTopicId,
account: { userUuid: transactionDb.user.gradidoId },
}
const linkedUser = {
communityTopicId: recipientCommunityTopicId,
account: { userUuid: transactionDb.linkedUser.gradidoId },
}
const transaction: TransactionInput = {
user,
linkedUser,
amount: v.parse(gradidoAmountSchema, transactionDb.amount),
memo: v.parse(memoSchema, transactionDb.memo),
type: InputTransactionType.GRADIDO_TRANSFER,
createdAt: transactionDb.balanceDate,
}
if (transactionDb.typeId === TransactionTypeId.CREATION) {
if (!transactionDb.creationDate) {
throw new Error('contribution transaction without creation date')
}
transaction.targetDate = transactionDb.creationDate
transaction.type = InputTransactionType.GRADIDO_CREATION
} else if (transactionDb.typeId === TransactionTypeId.RECEIVE) {
transaction.user = linkedUser
transaction.linkedUser = user
}
if (transactionDb.transactionLinkCode) {
if (transactionDb.typeId !== TransactionTypeId.RECEIVE) {
throw new Error(
"linked transaction which isn't receive, send will taken care of on link creation",
)
}
transaction.user = {
communityTopicId: recipientCommunityTopicId,
seed: transactionDb.transactionLinkCode,
}
transaction.type = InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER
}
return v.parse(transactionSchema, transaction)
}
export function transactionLinkDbToTransaction(
transactionLinkDb: TransactionLinkDb,
communityTopicId: HieroId,
): Transaction {
return v.parse(transactionSchema, {
user: {
communityTopicId: communityTopicId,
account: { userUuid: transactionLinkDb.user.gradidoId },
},
linkedUser: {
communityTopicId: communityTopicId,
seed: transactionLinkDb.code,
},
type: InputTransactionType.GRADIDO_DEFERRED_TRANSFER,
amount: v.parse(gradidoAmountSchema, transactionLinkDb.amount),
memo: v.parse(memoSchema, transactionLinkDb.memo),
createdAt: transactionLinkDb.createdAt,
timeoutDuration: v.parse(
timeoutDurationSchema,
Math.round(
(transactionLinkDb.validUntil.getTime() - transactionLinkDb.createdAt.getTime()) / 1000,
),
),
})
}

189
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts
View File

@ -1,189 +0,0 @@
import { asc, eq, inArray, isNotNull, sql } from 'drizzle-orm'
import { alias } from 'drizzle-orm/mysql-core'
import { MySql2Database } from 'drizzle-orm/mysql2'
import { GradidoUnit } from 'gradido-blockchain-js'
import { getLogger } from 'log4js'
import * as v from 'valibot'
import { LOG4JS_BASE_CATEGORY } from '../../config/const'
import {
communitiesTable,
transactionLinksTable,
transactionsTable,
usersTable,
} from './drizzle.schema'
import { TransactionTypeId } from './TransactionTypeId'
import {
CommunityDb,
CreatedUserDb,
communityDbSchema,
createdUserDbSchema,
TransactionDb,
TransactionLinkDb,
transactionDbSchema,
transactionLinkDbSchema,
} from './valibot.schema'
const logger = getLogger(
`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`,
)
// queries
export async function loadCommunities(db: MySql2Database): Promise<CommunityDb[]> {
const result = await db
.select({
foreign: communitiesTable.foreign,
communityUuid: communitiesTable.communityUuid,
name: communitiesTable.name,
creationDate: communitiesTable.creationDate,
userMinCreatedAt: sql`MIN(${usersTable.createdAt})`,
})
.from(communitiesTable)
.leftJoin(usersTable, eq(communitiesTable.communityUuid, usersTable.communityUuid))
.where(isNotNull(communitiesTable.communityUuid))
.orderBy(asc(communitiesTable.id))
.groupBy(communitiesTable.communityUuid)
const communityNames = new Set<string>()
return result.map((row: any) => {
let alias = row.name
if (communityNames.has(row.name)) {
alias = row.community_uuid
} else {
communityNames.add(row.name)
}
return v.parse(communityDbSchema, {
...row,
uniqueAlias: alias,
})
})
}
export async function loadUsers(
db: MySql2Database,
offset: number,
count: number,
): Promise<CreatedUserDb[]> {
const result = await db
.select()
.from(usersTable)
.orderBy(asc(usersTable.createdAt), asc(usersTable.id))
.limit(count)
.offset(offset)
return result.map((row: any) => {
return v.parse(createdUserDbSchema, row)
})
}
export async function loadTransactions(
db: MySql2Database,
offset: number,
count: number,
): Promise<TransactionDb[]> {
const linkedUsers = alias(usersTable, 'linkedUser')
const result = await db
.select({
transaction: transactionsTable,
user: usersTable,
linkedUser: linkedUsers,
transactionLink: transactionLinksTable,
})
.from(transactionsTable)
.where(
inArray(transactionsTable.typeId, [TransactionTypeId.CREATION, TransactionTypeId.RECEIVE]),
)
.leftJoin(usersTable, eq(transactionsTable.userId, usersTable.id))
.leftJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id))
.leftJoin(
transactionLinksTable,
eq(transactionsTable.transactionLinkId, transactionLinksTable.id),
)
.orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id))
.limit(count)
.offset(offset)
return result.map((row: any) => {
// console.log(row)
try {
// check for consistent data beforehand
const userCreatedAt = new Date(row.user.createdAt)
const linkedUserCreatedAd = new Date(row.linkedUser.createdAt)
const balanceDate = new Date(row.transaction.balanceDate)
if (
userCreatedAt.getTime() > balanceDate.getTime() ||
linkedUserCreatedAd.getTime() > balanceDate.getTime()
) {
logger.error(`table row: `, row)
throw new Error(
'at least one user was created after transaction balance date, logic error!',
)
}
let amount = GradidoUnit.fromString(row.transaction.amount)
if (row.transaction.typeId === TransactionTypeId.SEND) {
amount = amount.mul(new GradidoUnit(-1))
}
return v.parse(transactionDbSchema, {
...row.transaction,
transactionLinkCode: row.transactionLink ? row.transactionLink.code : null,
user: row.user,
linkedUser: row.linkedUser,
})
} catch (e) {
logger.error(`table row: ${JSON.stringify(row, null, 2)}`)
if (e instanceof v.ValiError) {
logger.error(v.flatten(e.issues))
}
throw e
}
})
}
export async function loadTransactionLinks(
db: MySql2Database,
offset: number,
count: number,
): Promise<TransactionLinkDb[]> {
const result = await db
.select()
.from(transactionLinksTable)
.leftJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id))
.orderBy(asc(transactionLinksTable.createdAt), asc(transactionLinksTable.id))
.limit(count)
.offset(offset)
return result.map((row: any) => {
return v.parse(transactionLinkDbSchema, {
...row.transaction_links,
user: row.users,
})
})
}
export async function loadDeletedTransactionLinks(
db: MySql2Database,
offset: number,
count: number,
): Promise<TransactionDb[]> {
const result = await db
.select()
.from(transactionLinksTable)
.leftJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id))
.where(isNotNull(transactionLinksTable.deletedAt))
.orderBy(asc(transactionLinksTable.deletedAt), asc(transactionLinksTable.id))
.limit(count)
.offset(offset)
return result.map((row: any) => {
return v.parse(transactionDbSchema, {
typeId: TransactionTypeId.RECEIVE,
amount: row.transaction_links.amount,
balanceDate: new Date(row.transaction_links.deletedAt),
memo: row.transaction_links.memo,
transactionLinkCode: row.transaction_links.code,
user: row.users,
linkedUser: row.users,
})
})
}

36
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts
View File

@ -1,36 +0,0 @@
import { Filter } from 'gradido-blockchain-js'
import { onShutdown } from '../../../../shared/src/helper/onShutdown'
import { exportAllCommunities } from './binaryExport'
import { bootstrap } from './bootstrap'
import { syncDbWithBlockchainContext } from './interaction/syncDbWithBlockchain/syncDbWithBlockchain.context'
const BATCH_SIZE = 100
async function main() {
// prepare in memory blockchains
const context = await bootstrap()
onShutdown(async (reason, error) => {
context.logger.info(`shutdown reason: ${reason}`)
if (error) {
context.logger.error(error)
}
})
// synchronize to in memory blockchain
await syncDbWithBlockchainContext(context, BATCH_SIZE)
// write as binary file for GradidoNode
exportAllCommunities(context, BATCH_SIZE)
// log runtime statistics
context.logRuntimeStatistics()
// needed because of shutdown handler (TODO: fix shutdown handler)
process.exit(0)
}
main().catch((e) => {
// biome-ignore lint/suspicious/noConsole: maybe logger isn't initialized here
console.error(e)
process.exit(1)
})

70
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts
View File

@ -1,70 +0,0 @@
import { Profiler } from 'gradido-blockchain-js'
import { getLogger, Logger } from 'log4js'
import { LOG4JS_BASE_CATEGORY } from '../../../../config/const'
import { Context } from '../../Context'
export abstract class AbstractSyncRole<T> {
private items: T[] = []
private offset = 0
protected logger: Logger
constructor(protected readonly context: Context) {
this.logger = getLogger(
`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5.interaction.syncDbWithBlockchain`,
)
}
abstract getDate(): Date
abstract loadFromDb(offset: number, count: number): Promise<T[]>
abstract pushToBlockchain(item: T): Promise<void>
abstract itemTypeName(): string
// return count of new loaded items
async ensureFilled(batchSize: number): Promise<number> {
if (this.items.length === 0) {
let timeUsed: Profiler | undefined
if (this.logger.isDebugEnabled()) {
timeUsed = new Profiler()
}
this.items = await this.loadFromDb(this.offset, batchSize)
this.offset += this.items.length
if (timeUsed && this.items.length) {
this.logger.debug(
`${timeUsed.string()} for loading ${this.items.length} ${this.itemTypeName()} from db`,
)
}
return this.items.length
}
return 0
}
async toBlockchain(): Promise<void> {
if (this.isEmpty()) {
throw new Error(`[toBlockchain] No items, please call this only if isEmpty returns false`)
}
await this.pushToBlockchain(this.shift())
}
peek(): T {
if (this.isEmpty()) {
throw new Error(`[peek] No items, please call this only if isEmpty returns false`)
}
return this.items[0]
}
shift(): T {
const item = this.items.shift()
if (!item) {
throw new Error(`[shift] No items, shift return undefined`)
}
return item
}
get length(): number {
return this.items.length
}
isEmpty(): boolean {
return this.items.length === 0
}
}

13
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts
View File

@ -1,13 +0,0 @@
import { loadDeletedTransactionLinks } from '../../database'
import { TransactionDb } from '../../valibot.schema'
import { TransactionsSyncRole } from './TransactionsSync.role'
export class DeletedTransactionLinksSyncRole extends TransactionsSyncRole {
itemTypeName(): string {
return 'deletedTransactionLinks'
}
async loadFromDb(offset: number, count: number): Promise<TransactionDb[]> {
return await loadDeletedTransactionLinks(this.context.db, offset, count)
}
}

25
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinksSync.role.ts
View File

@ -1,25 +0,0 @@
import { addTransaction } from '../../blockchain'
import { transactionLinkDbToTransaction } from '../../convert'
import { loadTransactionLinks } from '../../database'
import { TransactionLinkDb } from '../../valibot.schema'
import { AbstractSyncRole } from './AbstractSync.role'
export class TransactionLinksSyncRole extends AbstractSyncRole<TransactionLinkDb> {
getDate(): Date {
return this.peek().createdAt
}
itemTypeName(): string {
return 'transactionLinks'
}
async loadFromDb(offset: number, count: number): Promise<TransactionLinkDb[]> {
return await loadTransactionLinks(this.context.db, offset, count)
}
async pushToBlockchain(item: TransactionLinkDb): Promise<void> {
const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid)
const transaction = transactionLinkDbToTransaction(item, communityContext.topicId)
await addTransaction(communityContext.blockchain, communityContext.blockchain, transaction)
}
}

37
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionsSync.role.ts
View File

@ -1,37 +0,0 @@
import { addTransaction } from '../../blockchain'
import { transactionDbToTransaction } from '../../convert'
import { loadTransactions } from '../../database'
import { TransactionDb } from '../../valibot.schema'
import { AbstractSyncRole } from './AbstractSync.role'
export class TransactionsSyncRole extends AbstractSyncRole<TransactionDb> {
getDate(): Date {
return this.peek().balanceDate
}
itemTypeName(): string {
return 'transactions'
}
async loadFromDb(offset: number, count: number): Promise<TransactionDb[]> {
return await loadTransactions(this.context.db, offset, count)
}
async pushToBlockchain(item: TransactionDb): Promise<void> {
const senderCommunityContext = this.context.getCommunityContextByUuid(item.user.communityUuid)
const recipientCommunityContext = this.context.getCommunityContextByUuid(
item.linkedUser.communityUuid,
)
this.context.cache.setHomeCommunityTopicId(senderCommunityContext.topicId)
const transaction = transactionDbToTransaction(
item,
senderCommunityContext.topicId,
recipientCommunityContext.topicId,
)
await addTransaction(
senderCommunityContext.blockchain,
recipientCommunityContext.blockchain,
transaction,
)
}
}

31
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts
View File

@ -1,31 +0,0 @@
import { addRegisterAddressTransaction } from '../../blockchain'
import { userDbToTransaction } from '../../convert'
import { loadUsers } from '../../database'
import { generateKeyPairUserAccount } from '../../keyPair'
import { CreatedUserDb } from '../../valibot.schema'
import { AbstractSyncRole } from './AbstractSync.role'
export class UsersSyncRole extends AbstractSyncRole<CreatedUserDb> {
getDate(): Date {
return this.peek().createdAt
}
itemTypeName(): string {
return 'users'
}
async loadFromDb(offset: number, count: number): Promise<CreatedUserDb[]> {
const users = await loadUsers(this.context.db, offset, count)
for (const user of users) {
const communityContext = this.context.getCommunityContextByUuid(user.communityUuid)
await generateKeyPairUserAccount(user, this.context.cache, communityContext.topicId)
}
return users
}
async pushToBlockchain(item: CreatedUserDb): Promise<void> {
const communityContext = this.context.getCommunityContextByUuid(item.communityUuid)
const transaction = userDbToTransaction(item, communityContext.topicId)
return await addRegisterAddressTransaction(communityContext.blockchain, transaction)
}
}

38
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts
View File

@ -1,38 +0,0 @@
import { Profiler } from 'gradido-blockchain-js'
import { Context } from '../../Context'
import { DeletedTransactionLinksSyncRole } from './DeletedTransactionLinksSync.role'
import { TransactionLinksSyncRole } from './TransactionLinksSync.role'
import { TransactionsSyncRole } from './TransactionsSync.role'
import { UsersSyncRole } from './UsersSync.role'
export async function syncDbWithBlockchainContext(context: Context, batchSize: number) {
const timeUsed = new Profiler()
const containers = [
new UsersSyncRole(context),
new TransactionsSyncRole(context),
new DeletedTransactionLinksSyncRole(context),
new TransactionLinksSyncRole(context),
]
while (true) {
timeUsed.reset()
const results = await Promise.all(containers.map((c) => c.ensureFilled(batchSize)))
const loadedItemsCount = results.reduce((acc, c) => acc + c, 0)
// log only, if at least one new item was loaded
if (loadedItemsCount && context.logger.isInfoEnabled()) {
context.logger.info(`${loadedItemsCount} new items loaded from db in ${timeUsed.string()}`)
}
// remove empty containers
const available = containers.filter((c) => !c.isEmpty())
if (available.length === 0) {
break
}
// sort by date, to ensure container on index 0 is the one with the smallest date
if (available.length > 0) {
available.sort((a, b) => a.getDate().getTime() - b.getDate().getTime())
}
await available[0].toBlockchain()
}
}

15
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts
View File

@ -1,15 +0,0 @@
import { crypto_generichash_batch, crypto_generichash_KEYBYTES } from 'sodium-native'
export function bytesToMbyte(bytes: number): string {
return (bytes / 1024 / 1024).toFixed(4)
}
export function bytesToKbyte(bytes: number): string {
return (bytes / 1024).toFixed(0)
}
export function calculateOneHashStep(hash: Buffer, data: Buffer): Buffer<ArrayBuffer> {
const outputHash = Buffer.alloc(crypto_generichash_KEYBYTES, 0)
crypto_generichash_batch(outputHash, [hash, data])
return outputHash
}

70
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts
View File

@ -1,70 +0,0 @@
import { InMemoryBlockchain } from 'gradido-blockchain-js'
import * as v from 'valibot'
import { booleanSchema, dateSchema } from '../../schemas/typeConverter.schema'
import {
gradidoAmountSchema,
hieroIdSchema,
identifierSeedSchema,
memoSchema,
uuidv4Schema,
} from '../../schemas/typeGuard.schema'
import { TransactionTypeId } from './TransactionTypeId'
export const createdUserDbSchema = v.object({
gradidoId: uuidv4Schema,
communityUuid: uuidv4Schema,
createdAt: dateSchema,
})
export const userDbSchema = v.object({
gradidoId: uuidv4Schema,
communityUuid: uuidv4Schema,
})
export const transactionDbSchema = v.object({
typeId: v.enum(TransactionTypeId),
amount: gradidoAmountSchema,
balanceDate: dateSchema,
memo: memoSchema,
creationDate: v.nullish(dateSchema),
user: userDbSchema,
linkedUser: userDbSchema,
transactionLinkCode: v.nullish(identifierSeedSchema),
})
export const transactionLinkDbSchema = v.object({
user: userDbSchema,
code: identifierSeedSchema,
amount: gradidoAmountSchema,
memo: memoSchema,
createdAt: dateSchema,
validUntil: dateSchema,
})
export const communityDbSchema = v.object({
foreign: booleanSchema,
communityUuid: uuidv4Schema,
name: v.string(),
creationDate: dateSchema,
userMinCreatedAt: v.nullish(dateSchema),
uniqueAlias: v.string(),
})
export const communityContextSchema = v.object({
communityId: v.string(),
blockchain: v.instance(InMemoryBlockchain, 'expect InMemoryBlockchain type'),
topicId: hieroIdSchema,
folder: v.pipe(
v.string(),
v.minLength(1, 'expect string length >= 1'),
v.maxLength(255, 'expect string length <= 255'),
v.regex(/^[a-zA-Z0-9-_]+$/, 'expect string to be a valid (alphanumeric, _, -) folder name'),
),
})
export type TransactionDb = v.InferOutput<typeof transactionDbSchema>
export type UserDb = v.InferOutput<typeof userDbSchema>
export type CreatedUserDb = v.InferOutput<typeof createdUserDbSchema>
export type TransactionLinkDb = v.InferOutput<typeof transactionLinkDbSchema>
export type CommunityDb = v.InferOutput<typeof communityDbSchema>
export type CommunityContext = v.InferOutput<typeof communityContextSchema>

11
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/Context.ts → dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts
View File

@ -1,4 +1,5 @@
import { heapStats } from 'bun:jsc'
import dotenv from 'dotenv'
import { drizzle, MySql2Database } from 'drizzle-orm/mysql2'
import { Filter, Profiler, SearchDirection_ASC } from 'gradido-blockchain-js'
import { getLogger, Logger } from 'log4js'
@ -11,6 +12,8 @@ import { Uuidv4 } from '../../schemas/typeGuard.schema'
import { bytesToMbyte } from './utils'
import { CommunityContext } from './valibot.schema'
dotenv.config()
export class Context {
public logger: Logger
public db: MySql2Database
@ -36,11 +39,9 @@ export class Context {
database: CONFIG.MYSQL_DATABASE,
port: CONFIG.MYSQL_PORT,
})
return new Context(
getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5`),
drizzle({ client: connection }),
KeyPairCacheManager.getInstance(),
)
const db = drizzle({ client: connection })
const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5`)
return new Context(logger, db, KeyPairCacheManager.getInstance())
}
getCommunityContextByUuid(communityUuid: Uuidv4): CommunityContext {

48
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/binaryExport.ts → dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/binaryExport.ts
View File

@ -9,12 +9,13 @@ import {
} from 'gradido-blockchain-js'
import { CONFIG } from '../../config'
import { Context } from './Context'
import { bytesToKbyte, calculateOneHashStep } from './utils'
import { bytesString, calculateOneHashStep } from './utils'
import { CommunityContext } from './valibot.schema'
export function exportAllCommunities(context: Context, batchSize: number) {
const timeUsed = new Profiler()
for (const communityContext of context.communities.values()) {
context.logger.info(`exporting community ${communityContext.communityId} to binary file`)
exportCommunity(communityContext, context, batchSize)
}
context.logger.info(`time used for exporting communities to binary file: ${timeUsed.string()}`)
@ -25,36 +26,77 @@ export function exportCommunity(
context: Context,
batchSize: number,
) {
const timeUsed = new Profiler()
const timeSinceLastPrint = new Profiler()
// write as binary file for GradidoNode
const f = new Filter()
f.pagination.size = batchSize
f.pagination.page = 1
f.searchDirection = SearchDirection_ASC
const binFilePath = prepareFolder(communityContext)
let count = 0
let printCount = 0
let lastTransactionCount = 0
let triggeredTransactionsCount = 0
let hash = Buffer.alloc(32, 0)
const isDebug = context.logger.isDebugEnabled()
const printConsole = () => {
if (triggeredTransactionsCount > 0) {
process.stdout.write(
`exported ${count} transactions + ${triggeredTransactionsCount} triggered from timeouted transaction links\r`,
)
} else {
process.stdout.write(`exported ${count} transactions\r`)
}
}
do {
const transactions = communityContext.blockchain.findAll(f)
lastTransactionCount = transactions.size()
for (let i = 0; i < lastTransactionCount; i++) {
const confirmedTransaction = transactions.get(i)?.getConfirmedTransaction()
const transactionNr = f.pagination.page * batchSize + i
const transactionNr = (f.pagination.page - 2) * batchSize + i
if (!confirmedTransaction) {
throw new Error(`invalid TransactionEntry at index: ${transactionNr} `)
}
hash = exportTransaction(confirmedTransaction, hash, binFilePath)
if (
confirmedTransaction
?.getGradidoTransaction()
?.getTransactionBody()
?.isTimeoutDeferredTransfer()
) {
triggeredTransactionsCount++
} else {
count++
}
if (isDebug) {
if (timeSinceLastPrint.millis() > 100) {
printConsole()
timeSinceLastPrint.reset()
}
} else {
printCount++
if (printCount >= 100) {
printConsole()
printCount = 0
}
}
}
f.pagination.page++
} while (lastTransactionCount === batchSize)
printConsole()
process.stdout.write(`\n`)
fs.appendFileSync(binFilePath, hash!)
context.logger.info(
`binary file for community ${communityContext.communityId} written to ${binFilePath}`,
)
const sumTransactionsCount = (f.pagination.page - 2) * batchSize + lastTransactionCount
const fileSize = fs.statSync(binFilePath).size
context.logger.info(
`transactions count: ${(f.pagination.page - 1) * batchSize + lastTransactionCount}, size: ${bytesToKbyte(fs.statSync(binFilePath).size)} KByte`,
`exported ${sumTransactionsCount} transactions (${bytesString(fileSize)}) in ${timeUsed.string()}`,
)
}

50
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/blockchain.ts Normal file
View File

@ -0,0 +1,50 @@
import {
AccountBalances,
Filter,
GradidoTransaction,
HieroAccountId,
InMemoryBlockchain,
LedgerAnchor,
Profiler,
} from 'gradido-blockchain-js'
import { NotEnoughGradidoBalanceError } from './errors'
export const defaultHieroAccount = new HieroAccountId(0, 0, 2)
export let callTime: number = 0
const timeUsed = new Profiler()
export function addToBlockchain(
transaction: GradidoTransaction,
blockchain: InMemoryBlockchain,
ledgerAnchor: LedgerAnchor,
accountBalances: AccountBalances,
): boolean {
try {
timeUsed.reset()
const result = blockchain.createAndAddConfirmedTransactionExternFast(
transaction,
ledgerAnchor,
accountBalances,
)
callTime += timeUsed.nanos()
return result
} catch (error) {
if (error instanceof Error) {
const matches = error.message.match(
/not enough Gradido Balance for (send coins|operation), needed: -?(\d+\.\d+), exist: (\d+\.\d+)/,
)
if (matches) {
const needed = parseFloat(matches[2])
const exist = parseFloat(matches[3])
throw new NotEnoughGradidoBalanceError(needed, exist)
}
}
// const wekingheim = InMemoryBlockchainProvider.getInstance().getBlockchain('wekingheim')
// const lastTransactionw = wekingheim?.findOne(Filter.LAST_TRANSACTION)
const lastTransaction = blockchain.findOne(Filter.LAST_TRANSACTION)
throw new Error(
`Transaction ${transaction.toJson(true)} not added: ${error}, last transaction was: ${lastTransaction?.getConfirmedTransaction()?.toJson(true)}`,
)
}
}

105
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/bootstrap.ts Normal file
View File

@ -0,0 +1,105 @@
import { randomBytes } from 'node:crypto'
import {
AccountBalances,
GradidoTransactionBuilder,
InMemoryBlockchainProvider,
LedgerAnchor,
} from 'gradido-blockchain-js'
import * as v from 'valibot'
import { CONFIG } from '../../config'
import { deriveFromSeed } from '../../data/deriveKeyPair'
import { Hex32, hex32Schema } from '../../schemas/typeGuard.schema'
import {
AUF_ACCOUNT_DERIVATION_INDEX,
GMW_ACCOUNT_DERIVATION_INDEX,
hardenDerivationIndex,
} from '../../utils/derivationHelper'
import { toFolderName } from '../../utils/filesystem'
import { addToBlockchain } from './blockchain'
import { Context } from './Context'
import { Balance } from './data/Balance'
import {
loadAdminUsersCache,
loadCommunities,
loadContributionLinkModeratorCache,
} from './database'
import { CommunityContext } from './valibot.schema'
export async function bootstrap(): Promise<Context> {
const context = await Context.create()
context.communities = await bootstrapCommunities(context)
await Promise.all([
loadContributionLinkModeratorCache(context.db),
loadAdminUsersCache(context.db),
])
return context
}
async function bootstrapCommunities(context: Context): Promise<Map<string, CommunityContext>> {
const communities = new Map<string, CommunityContext>()
const communitiesDb = await loadCommunities(context.db)
for (const communityDb of communitiesDb) {
const communityId = communityDb.communityUuid
const blockchain = InMemoryBlockchainProvider.getInstance().getBlockchain(communityId)
if (!blockchain) {
throw new Error(`Couldn't create Blockchain for community ${communityId}`)
}
context.logger.info(`Blockchain for community '${communityId}' created`)
let seed: Hex32
if (!communityDb.foreign) {
seed = v.parse(hex32Schema, CONFIG.HOME_COMMUNITY_SEED.convertToHex())
} else {
seed = v.parse(hex32Schema, randomBytes(32).toString('hex'))
}
let creationDate = communityDb.creationDate
if (communityDb.userMinCreatedAt && communityDb.userMinCreatedAt < communityDb.creationDate) {
// create community root transaction 1 minute before first user
creationDate = new Date(new Date(communityDb.userMinCreatedAt).getTime() - 1000 * 60)
}
const communityKeyPair = deriveFromSeed(seed)
const gmwKeyPair = communityKeyPair.deriveChild(
hardenDerivationIndex(GMW_ACCOUNT_DERIVATION_INDEX),
)
const aufKeyPair = communityKeyPair.deriveChild(
hardenDerivationIndex(AUF_ACCOUNT_DERIVATION_INDEX),
)
if (!communityKeyPair || !gmwKeyPair || !aufKeyPair) {
throw new Error(
`Error on creating key pair for community ${JSON.stringify(communityDb, null, 2)}`,
)
}
const builder = new GradidoTransactionBuilder()
builder
.setCreatedAt(creationDate)
.setSenderCommunity(communityId)
.setCommunityRoot(
communityKeyPair.getPublicKey(),
gmwKeyPair.getPublicKey(),
aufKeyPair.getPublicKey(),
)
.sign(communityKeyPair)
const communityContext: CommunityContext = {
communityId,
foreign: communityDb.foreign,
blockchain,
keyPair: communityKeyPair,
folder: toFolderName(communityId),
gmwBalance: new Balance(gmwKeyPair.getPublicKey()!, communityId),
aufBalance: new Balance(aufKeyPair.getPublicKey()!, communityId),
}
communities.set(communityId, communityContext)
const accountBalances = new AccountBalances()
accountBalances.add(communityContext.aufBalance.getAccountBalance())
accountBalances.add(communityContext.gmwBalance.getAccountBalance())
addToBlockchain(
builder.build(),
blockchain,
new LedgerAnchor(communityDb.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_COMMUNITY_ID),
accountBalances,
)
}
return communities
}

117
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/Balance.ts Normal file
View File

@ -0,0 +1,117 @@
import Decimal from 'decimal.js-light'
import { AccountBalance, GradidoUnit, MemoryBlockPtr } from 'gradido-blockchain-js'
import { NegativeBalanceError } from '../errors'
import { legacyCalculateDecay } from '../utils'
export class Balance {
private balance: GradidoUnit
private date: Date
private publicKey: MemoryBlockPtr
private communityId: string
constructor(publicKey: MemoryBlockPtr, communityId: string) {
this.balance = new GradidoUnit(0)
this.date = new Date()
this.publicKey = publicKey
this.communityId = communityId
}
static fromAccountBalance(
accountBalance: AccountBalance,
confirmedAt: Date,
communityId: string,
): Balance {
const balance = new Balance(accountBalance.getPublicKey()!, communityId)
balance.update(accountBalance.getBalance(), confirmedAt)
return balance
}
getBalance(): GradidoUnit {
return this.balance
}
getDate(): Date {
return this.date
}
updateLegacyDecay(amount: GradidoUnit, date: Date) {
// make sure to copy instead of referencing
const previousBalanceString = this.balance.toString()
const previousDate = new Date(this.date.getTime())
if (this.balance.equal(GradidoUnit.zero())) {
this.balance = amount
this.date = date
} else {
const decayedBalance = legacyCalculateDecay(
new Decimal(this.balance.toString()),
this.date,
date,
).toDecimalPlaces(4, Decimal.ROUND_CEIL)
const newBalance = decayedBalance.add(new Decimal(amount.toString()))
this.balance = GradidoUnit.fromString(newBalance.toString())
this.date = date
}
if (this.balance.lt(GradidoUnit.zero())) {
if (this.balance.lt(GradidoUnit.fromGradidoCent(100).negated())) {
const previousDecayedBalance = legacyCalculateDecay(
new Decimal(previousBalanceString),
previousDate,
date,
)
throw new NegativeBalanceError(
`negative Gradido amount detected in Balance.updateLegacyDecay`,
previousBalanceString,
amount.toString(),
previousDecayedBalance.toString(),
)
} else {
this.balance = GradidoUnit.zero()
}
}
}
update(amount: GradidoUnit, date: Date) {
const previousBalance = new GradidoUnit(this.balance.toString())
const previousDate = new Date(this.date.getTime())
if (this.balance.equal(GradidoUnit.zero())) {
this.balance = amount
this.date = date
} else {
this.balance = this.balance.calculateDecay(this.date, date).add(amount)
this.date = date
}
if (this.balance.lt(GradidoUnit.zero())) {
// ignore diffs less than a gradido cent
if (this.balance.lt(GradidoUnit.fromGradidoCent(100).negated())) {
const previousDecayedBalance = this.balance.calculateDecay(previousDate, date)
throw new NegativeBalanceError(
`negative Gradido amount detected in Balance.update`,
previousBalance.toString(),
amount.toString(),
previousDecayedBalance.toString(),
)
} else {
this.balance = GradidoUnit.zero()
}
}
}
getAccountBalance(): AccountBalance {
return new AccountBalance(this.publicKey, this.balance, this.communityId)
}
toString(): string {
return JSON.stringify(
{
balance: this.balance.toString(),
date: this.date,
publicKey: this.publicKey.convertToHex(),
communityId: this.communityId,
},
null,
2,
)
}
}

7
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/ContributionStatus.ts Normal file
View File

@ -0,0 +1,7 @@
export enum ContributionStatus {
PENDING = 'PENDING',
DELETED = 'DELETED',
IN_PROGRESS = 'IN_PROGRESS',
DENIED = 'DENIED',
CONFIRMED = 'CONFIRMED',
}

0
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/TransactionTypeId.ts → dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/TransactionTypeId.ts
View File

23
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/keyPair.ts → dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/keyPair.ts
View File

@ -1,13 +1,13 @@
import { KeyPairEd25519, MemoryBlock, MemoryBlockPtr } from 'gradido-blockchain-js'
import { getLogger } from 'log4js'
import { KeyPairCacheManager } from '../../cache/KeyPairCacheManager'
import { CONFIG } from '../../config'
import { LOG4JS_BASE_CATEGORY } from '../../config/const'
import { KeyPairIdentifierLogic } from '../../data/KeyPairIdentifier.logic'
import { AccountKeyPairRole } from '../../interactions/resolveKeyPair/AccountKeyPair.role'
import { UserKeyPairRole } from '../../interactions/resolveKeyPair/UserKeyPair.role'
import { HieroId } from '../../schemas/typeGuard.schema'
import { CommunityDb, UserDb } from './valibot.schema'
import { KeyPairCacheManager } from '../../../cache/KeyPairCacheManager'
import { CONFIG } from '../../../config'
import { LOG4JS_BASE_CATEGORY } from '../../../config/const'
import { KeyPairIdentifierLogic } from '../../../data/KeyPairIdentifier.logic'
import { AccountKeyPairRole } from '../../../interactions/resolveKeyPair/AccountKeyPair.role'
import { UserKeyPairRole } from '../../../interactions/resolveKeyPair/UserKeyPair.role'
import { HieroId } from '../../../schemas/typeGuard.schema'
import { CommunityDb, UserDb } from '../valibot.schema'
const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.keyPair`)
@ -30,7 +30,10 @@ export function generateKeyPairCommunity(
if (!keyPair) {
throw new Error(`Couldn't create key pair for community ${community.communityUuid}`)
}
const communityKeyPairKey = new KeyPairIdentifierLogic({ communityTopicId: topicId }).getKey()
const communityKeyPairKey = new KeyPairIdentifierLogic({
communityTopicId: topicId,
communityId: community.communityUuid,
}).getKey()
cache.addKeyPair(communityKeyPairKey, keyPair)
logger.info(`Community Key Pair added with key: ${communityKeyPairKey}`)
}
@ -44,6 +47,7 @@ export async function generateKeyPairUserAccount(
const userKeyPairRole = new UserKeyPairRole(user.gradidoId, communityKeyPair)
const userKeyPairKey = new KeyPairIdentifierLogic({
communityTopicId: communityTopicId,
communityId: user.communityUuid,
account: {
userUuid: user.gradidoId,
accountNr: 0,
@ -56,6 +60,7 @@ export async function generateKeyPairUserAccount(
const accountKeyPairRole = new AccountKeyPairRole(1, userKeyPair)
const accountKeyPairKey = new KeyPairIdentifierLogic({
communityTopicId: communityTopicId,
communityId: user.communityUuid,
account: {
userUuid: user.gradidoId,
accountNr: 1,

65
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/database.ts Normal file
View File

@ -0,0 +1,65 @@
import { and, asc, eq, isNotNull, isNull, or, sql } from 'drizzle-orm'
import { MySql2Database } from 'drizzle-orm/mysql2'
import * as v from 'valibot'
import { communitiesTable, eventsTable, userRolesTable, usersTable } from './drizzle.schema'
import { CommunityDb, communityDbSchema, UserDb, userDbSchema } from './valibot.schema'
export const contributionLinkModerators = new Map<number, UserDb>()
export const adminUsers = new Map<string, UserDb>()
export async function loadContributionLinkModeratorCache(db: MySql2Database): Promise<void> {
const result = await db
.select({
event: eventsTable,
user: usersTable,
})
.from(eventsTable)
.innerJoin(usersTable, eq(eventsTable.actingUserId, usersTable.id))
.where(eq(eventsTable.type, 'ADMIN_CONTRIBUTION_LINK_CREATE'))
.orderBy(asc(eventsTable.id))
result.map((row: any) => {
contributionLinkModerators.set(
row.event.involvedContributionLinkId,
v.parse(userDbSchema, row.user),
)
})
}
export async function loadAdminUsersCache(db: MySql2Database): Promise<void> {
const result = await db
.select({
user: usersTable,
})
.from(userRolesTable)
.where(eq(userRolesTable.role, 'ADMIN'))
.innerJoin(usersTable, eq(userRolesTable.userId, usersTable.id))
result.map((row: any) => {
adminUsers.set(row.gradidoId, v.parse(userDbSchema, row.user))
})
}
// queries
export async function loadCommunities(db: MySql2Database): Promise<CommunityDb[]> {
const result = await db
.select({
id: communitiesTable.id,
foreign: communitiesTable.foreign,
communityUuid: communitiesTable.communityUuid,
name: communitiesTable.name,
creationDate: communitiesTable.creationDate,
userMinCreatedAt: sql`MIN(${usersTable.createdAt})`,
})
.from(communitiesTable)
.innerJoin(usersTable, eq(communitiesTable.communityUuid, usersTable.communityUuid))
.where(
and(isNotNull(communitiesTable.communityUuid), sql`${usersTable.createdAt} > '2000-01-01'`),
)
.orderBy(asc(communitiesTable.id))
.groupBy(communitiesTable.communityUuid)
return result.map((row: any) => {
return v.parse(communityDbSchema, row)
})
}

39
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts → dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/drizzle.schema.ts
View File

@ -24,12 +24,34 @@ export const communitiesTable = mysqlTable(
(table) => [unique('uuid_key').on(table.communityUuid)],
)
export const contributionsTable = mysqlTable('contributions', {
id: int().autoincrement().notNull(),
userId: int('user_id').default(sql`NULL`),
contributionDate: datetime('contribution_date', { mode: 'string' }).default(sql`NULL`),
memo: varchar({ length: 512 }).notNull(),
amount: decimal({ precision: 40, scale: 20 }).notNull(),
contributionLinkId: int('contribution_link_id').default(sql`NULL`),
confirmedBy: int('confirmed_by').default(sql`NULL`),
confirmedAt: datetime('confirmed_at', { mode: 'string' }).default(sql`NULL`),
contributionStatus: varchar('contribution_status', { length: 12 }).default("'PENDING'").notNull(),
transactionId: int('transaction_id').default(sql`NULL`),
})
export const eventsTable = mysqlTable('events', {
id: int().autoincrement().notNull(),
type: varchar({ length: 100 }).notNull(),
actingUserId: int('acting_user_id').notNull(),
involvedContributionLinkId: int('involved_contribution_link_id').default(sql`NULL`),
})
export const usersTable = mysqlTable(
'users',
{
id: int().autoincrement().notNull(),
foreign: tinyint().default(0).notNull(),
gradidoId: char('gradido_id', { length: 36 }).notNull(),
communityUuid: varchar('community_uuid', { length: 36 }).default(sql`NULL`),
deletedAt: datetime('deleted_at', { mode: 'string', fsp: 3 }).default(sql`NULL`),
createdAt: datetime('created_at', { mode: 'string', fsp: 3 })
.default(sql`current_timestamp(3)`)
.notNull(),
@ -37,6 +59,16 @@ export const usersTable = mysqlTable(
(table) => [unique('uuid_key').on(table.gradidoId, table.communityUuid)],
)
export const userRolesTable = mysqlTable(
'user_roles',
{
id: int().autoincrement().notNull(),
userId: int('user_id').notNull(),
role: varchar({ length: 40 }).notNull(),
},
(table) => [index('user_id').on(table.userId)],
)
export const transactionsTable = mysqlTable(
'transactions',
{
@ -44,6 +76,7 @@ export const transactionsTable = mysqlTable(
typeId: int('type_id').default(sql`NULL`),
transactionLinkId: int('transaction_link_id').default(sql`NULL`),
amount: decimal({ precision: 40, scale: 20 }).default(sql`NULL`),
balance: decimal({ precision: 40, scale: 20 }).default(sql`NULL`),
balanceDate: datetime('balance_date', { mode: 'string', fsp: 3 })
.default(sql`current_timestamp(3)`)
.notNull(),
@ -51,6 +84,9 @@ export const transactionsTable = mysqlTable(
creationDate: datetime('creation_date', { mode: 'string', fsp: 3 }).default(sql`NULL`),
userId: int('user_id').notNull(),
linkedUserId: int('linked_user_id').default(sql`NULL`),
linkedUserCommunityUuid: char('linked_user_community_uuid', { length: 36 }).default(sql`NULL`),
linkedUserGradidoId: char('linked_user_gradido_id', { length: 36 }).default(sql`NULL`),
linkedTransactionId: int('linked_transaction_id').default(sql`NULL`),
},
(table) => [index('user_id').on(table.userId)],
)
@ -59,9 +95,12 @@ export const transactionLinksTable = mysqlTable('transaction_links', {
id: int().autoincrement().notNull(),
userId: int().notNull(),
amount: decimal({ precision: 40, scale: 20 }).notNull(),
holdAvailableAmount: decimal('hold_available_amount', { precision: 40, scale: 20 }).notNull(),
memo: varchar({ length: 255 }).notNull(),
code: varchar({ length: 24 }).notNull(),
createdAt: datetime({ mode: 'string' }).notNull(),
deletedAt: datetime({ mode: 'string' }).default(sql`NULL`),
validUntil: datetime({ mode: 'string' }).notNull(),
redeemedAt: datetime({ mode: 'string' }).default(sql`NULL`),
redeemedBy: int().default(sql`NULL`),
})

67
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/errors.ts Normal file
View File

@ -0,0 +1,67 @@
import * as v from 'valibot'
export class NotEnoughGradidoBalanceError extends Error {
constructor(
public needed: number,
public exist: number,
) {
super(
`Not enough Gradido Balance for send coins, needed: ${needed} Gradido, exist: ${exist} Gradido`,
)
this.name = 'NotEnoughGradidoBalanceError'
}
}
export class DatabaseError extends Error {
constructor(message: string, rows: any, originalError: Error) {
const parts: string[] = [`DatabaseError in ${message}`]
// Valibot-specific
if (originalError instanceof v.ValiError) {
const flattened = v.flatten(originalError.issues)
parts.push('Validation errors:')
parts.push(JSON.stringify(flattened, null, 2))
} else {
parts.push(`Original error: ${originalError.message}`)
}
parts.push('Rows:')
parts.push(JSON.stringify(rows, null, 2))
super(parts.join('\n\n'))
this.name = 'DatabaseError'
this.cause = originalError
}
}
export class BlockchainError extends Error {
constructor(message: string, item: any, originalError: Error) {
const parts: string[] = [`BlockchainError in ${message}`]
parts.push(`Original error: ${originalError.message}`)
parts.push('Item:')
parts.push(JSON.stringify(item, null, 2))
super(parts.join('\n\n'))
this.name = 'BlockchainError'
this.cause = originalError
}
}
export class NegativeBalanceError extends Error {
constructor(
message: string,
previousBalanceString: string,
amount: string,
previousDecayedBalance: string,
) {
const parts: string[] = [`NegativeBalanceError in ${message}`]
parts.push(`Previous balance: ${previousBalanceString}`)
parts.push(`Amount: ${amount}`)
parts.push(`Previous decayed balance: ${previousDecayedBalance}`)
super(parts.join('\n'))
this.name = 'NegativeBalanceError'
}
}

62
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/index.ts Normal file
View File

@ -0,0 +1,62 @@
import { Filter, Profiler, ThreadingPolicy_Half, verifySignatures } from 'gradido-blockchain-js'
import { onShutdown } from '../../../../shared/src/helper/onShutdown'
import { exportAllCommunities } from './binaryExport'
import { bootstrap } from './bootstrap'
import { syncDbWithBlockchainContext } from './interaction/syncDbWithBlockchain/syncDbWithBlockchain.context'
// import { hello } from '../../../zig/hello.zig'
const BATCH_SIZE = 1000
async function main() {
// hello()
// return
// prepare in memory blockchains
const context = await bootstrap()
onShutdown(async (reason, error) => {
context.logger.info(`shutdown reason: ${reason}`)
if (error) {
context.logger.error(error)
}
})
// synchronize to in memory blockchain
try {
await syncDbWithBlockchainContext(context, BATCH_SIZE)
} catch (e) {
context.logger.error(e)
//context.logBlogchain(v.parse(uuidv4Schema, 'e70da33e-5976-4767-bade-aa4e4fa1c01a'))
}
const timeUsed = new Profiler()
// bulk verify transaction signatures
for (const communityContext of context.communities.values()) {
// verifySignatures(Filter.ALL_TRANSACTIONS, ThreadingPolicy_Half)
const result = verifySignatures(
Filter.ALL_TRANSACTIONS,
communityContext.communityId,
ThreadingPolicy_Half,
)
if (!result.isEmpty()) {
throw new Error(
`Verification of signatures failed for community ${communityContext.communityId}`,
)
}
}
context.logger.info(`verified in ${timeUsed.string()}`)
// write as binary file for GradidoNode
exportAllCommunities(context, BATCH_SIZE)
// log runtime statistics
context.logRuntimeStatistics()
// needed because of shutdown handler (TODO: fix shutdown handler)
process.exit(0)
}
main().catch((e) => {
// biome-ignore lint/suspicious/noConsole: maybe logger isn't initialized here
console.error(e)
process.exit(1)
})

170
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts Normal file
View File

@ -0,0 +1,170 @@
import {
AccountBalances,
Filter,
GradidoTransactionBuilder,
InMemoryBlockchain,
KeyPairEd25519,
MemoryBlockPtr,
Profiler,
SearchDirection_DESC,
} from 'gradido-blockchain-js'
import { getLogger, Logger } from 'log4js'
import { LOG4JS_BASE_CATEGORY } from '../../../../config/const'
import { deriveFromKeyPairAndIndex, deriveFromKeyPairAndUuid } from '../../../../data/deriveKeyPair'
import { Uuidv4 } from '../../../../schemas/typeGuard.schema'
import { Context } from '../../Context'
import { Balance } from '../../data/Balance'
import { CommunityContext } from '../../valibot.schema'
export type IndexType = {
date: Date
id: number
}
export let nanosBalanceForUser = 0
const lastBalanceOfUserTimeUsed = new Profiler()
export abstract class AbstractSyncRole<ItemType> {
private items: ItemType[] = []
protected lastIndex: IndexType = { date: new Date(0), id: 0 }
protected logger: Logger
protected transactionBuilder: GradidoTransactionBuilder
protected accountBalances: AccountBalances
constructor(protected readonly context: Context) {
this.logger = getLogger(
`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5.interaction.syncDbWithBlockchain`,
)
this.transactionBuilder = new GradidoTransactionBuilder()
this.accountBalances = new AccountBalances()
}
getAccountKeyPair(communityContext: CommunityContext, gradidoId: Uuidv4): KeyPairEd25519 {
return this.context.cache.getKeyPairSync(gradidoId, () => {
return deriveFromKeyPairAndIndex(
deriveFromKeyPairAndUuid(communityContext.keyPair, gradidoId),
1,
)
})
}
getLastBalanceForUser(
publicKey: MemoryBlockPtr,
blockchain: InMemoryBlockchain,
communityId: string,
): Balance {
lastBalanceOfUserTimeUsed.reset()
if (publicKey.isEmpty()) {
throw new Error('publicKey is empty')
}
const f = Filter.lastBalanceFor(publicKey)
f.setCommunityId(communityId)
const lastSenderTransaction = blockchain.findOne(f)
if (!lastSenderTransaction) {
return new Balance(publicKey, communityId)
}
const lastConfirmedTransaction = lastSenderTransaction.getConfirmedTransaction()
if (!lastConfirmedTransaction) {
throw new Error(
`invalid transaction, getConfirmedTransaction call failed for transaction nr: ${lastSenderTransaction.getTransactionNr()}`,
)
}
const senderLastAccountBalance = lastConfirmedTransaction.getAccountBalance(
publicKey,
communityId,
)
if (!senderLastAccountBalance) {
return new Balance(publicKey, communityId)
}
const result = Balance.fromAccountBalance(
senderLastAccountBalance,
lastConfirmedTransaction.getConfirmedAt().getDate(),
communityId,
)
nanosBalanceForUser += lastBalanceOfUserTimeUsed.nanos()
return result
}
logLastBalanceChangingTransactions(
publicKey: MemoryBlockPtr,
blockchain: InMemoryBlockchain,
transactionCount: number = 5,
) {
if (!this.context.logger.isDebugEnabled()) {
return
}
const f = new Filter()
f.updatedBalancePublicKey = publicKey
f.searchDirection = SearchDirection_DESC
f.pagination.size = transactionCount
const lastTransactions = blockchain.findAll(f)
for (let i = lastTransactions.size() - 1; i >= 0; i--) {
const tx = lastTransactions.get(i)
this.context.logger.debug(`${i}: ${tx?.getConfirmedTransaction()!.toJson(true)}`)
}
}
abstract getDate(): Date
// for using seek rather than offset pagination approach
abstract getLastIndex(): IndexType
abstract loadFromDb(lastIndex: IndexType, count: number): Promise<ItemType[]>
abstract pushToBlockchain(item: ItemType): void
abstract itemTypeName(): string
abstract getCommunityUuids(): Uuidv4[]
// return count of new loaded items
async ensureFilled(batchSize: number): Promise<number> {
if (this.items.length === 0) {
let timeUsed: Profiler | undefined
if (this.logger.isDebugEnabled()) {
timeUsed = new Profiler()
}
this.items = await this.loadFromDb(this.lastIndex, batchSize)
if (this.length > 0) {
this.lastIndex = this.getLastIndex()
if (timeUsed) {
this.logger.debug(
`${timeUsed.string()} for loading ${this.items.length} ${this.itemTypeName()} from db`,
)
}
}
return this.items.length
}
return 0
}
toBlockchain(): void {
if (this.isEmpty()) {
throw new Error(`[toBlockchain] No items, please call this only if isEmpty returns false`)
}
this.pushToBlockchain(this.shift())
}
peek(): ItemType {
if (this.isEmpty()) {
throw new Error(`[peek] No items, please call this only if isEmpty returns false`)
}
return this.items[0]
}
peekLast(): ItemType {
if (this.isEmpty()) {
throw new Error(`[peekLast] No items, please call this only if isEmpty returns false`)
}
return this.items[this.items.length - 1]
}
shift(): ItemType {
const item = this.items.shift()
if (!item) {
throw new Error(`[shift] No items, shift return undefined`)
}
return item
}
get length(): number {
return this.items.length
}
isEmpty(): boolean {
return this.items.length === 0
}
}

69
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts Normal file
View File

@ -0,0 +1,69 @@
import { and, asc, eq, gt, isNotNull, or } from 'drizzle-orm'
import * as v from 'valibot'
import { Context } from '../../Context'
import { ContributionStatus } from '../../data/ContributionStatus'
import { contributionLinkModerators } from '../../database'
import { contributionsTable, usersTable } from '../../drizzle.schema'
import { DatabaseError } from '../../errors'
import { toMysqlDateTime } from '../../utils'
import { CreationTransactionDb, creationTransactionDbSchema } from '../../valibot.schema'
import { IndexType } from './AbstractSync.role'
import { CreationsSyncRole } from './CreationsSync.role'
export class ContributionLinkTransactionSyncRole extends CreationsSyncRole {
constructor(readonly context: Context) {
super(context)
}
itemTypeName(): string {
return 'contributionLinkTransaction'
}
async loadFromDb(lastIndex: IndexType, count: number): Promise<CreationTransactionDb[]> {
const result = await this.context.db
.select({
contribution: contributionsTable,
user: usersTable,
})
.from(contributionsTable)
.where(
and(
isNotNull(contributionsTable.contributionLinkId),
eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED),
or(
gt(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)),
and(
eq(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)),
gt(contributionsTable.transactionId, lastIndex.id),
),
),
),
)
.innerJoin(usersTable, eq(contributionsTable.userId, usersTable.id))
.orderBy(asc(contributionsTable.confirmedAt), asc(contributionsTable.transactionId))
.limit(count)
const verifiedCreationTransactions: CreationTransactionDb[] = []
for (const row of result) {
if (!row.contribution.contributionLinkId) {
throw new Error(
`expect contributionLinkId to be set: ${JSON.stringify(row.contribution, null, 2)}`,
)
}
const item = {
...row.contribution,
user: row.user,
confirmedByUser: contributionLinkModerators.get(row.contribution.contributionLinkId),
}
if (!item.confirmedByUser || item.userId === item.confirmedByUser.id) {
this.context.logger.warn(`skipped Contribution Link Transaction ${row.contribution.id}`)
continue
}
try {
verifiedCreationTransactions.push(v.parse(creationTransactionDbSchema, item))
} catch (e) {
throw new DatabaseError('load contributions with contribution link id', item, e as Error)
}
}
return verifiedCreationTransactions
}
}

181
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts Normal file
View File

@ -0,0 +1,181 @@
import { and, asc, eq, gt, isNull, or } from 'drizzle-orm'
import { alias } from 'drizzle-orm/mysql-core'
import {
AccountBalances,
AuthenticatedEncryption,
EncryptedMemo,
Filter,
GradidoTransactionBuilder,
KeyPairEd25519,
LedgerAnchor,
MemoryBlockPtr,
SearchDirection_DESC,
TransactionType_CREATION,
TransferAmount,
} from 'gradido-blockchain-js'
import * as v from 'valibot'
import { Uuidv4 } from '../../../../schemas/typeGuard.schema'
import { addToBlockchain } from '../../blockchain'
import { Context } from '../../Context'
import { ContributionStatus } from '../../data/ContributionStatus'
import { contributionsTable, usersTable } from '../../drizzle.schema'
import { BlockchainError, DatabaseError } from '../../errors'
import { toMysqlDateTime } from '../../utils'
import {
CommunityContext,
CreationTransactionDb,
creationTransactionDbSchema,
} from '../../valibot.schema'
import { AbstractSyncRole, IndexType } from './AbstractSync.role'
export class CreationsSyncRole extends AbstractSyncRole<CreationTransactionDb> {
constructor(context: Context) {
super(context)
this.accountBalances.reserve(3)
}
getDate(): Date {
return this.peek().confirmedAt
}
getCommunityUuids(): Uuidv4[] {
return [this.peek().user.communityUuid]
}
getLastIndex(): IndexType {
const lastItem = this.peekLast()
return { date: lastItem.confirmedAt, id: lastItem.transactionId }
}
itemTypeName(): string {
return 'creationTransactions'
}
async loadFromDb(lastIndex: IndexType, count: number): Promise<CreationTransactionDb[]> {
const confirmedByUsers = alias(usersTable, 'confirmedByUser')
const result = await this.context.db
.select({
contribution: contributionsTable,
user: usersTable,
confirmedByUser: confirmedByUsers,
})
.from(contributionsTable)
.where(
and(
isNull(contributionsTable.contributionLinkId),
eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED),
or(
gt(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)),
and(
eq(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)),
gt(contributionsTable.transactionId, lastIndex.id),
),
),
),
)
.innerJoin(usersTable, eq(contributionsTable.userId, usersTable.id))
.innerJoin(confirmedByUsers, eq(contributionsTable.confirmedBy, confirmedByUsers.id))
.orderBy(asc(contributionsTable.confirmedAt), asc(contributionsTable.transactionId))
.limit(count)
return result.map((row) => {
const item = {
...row.contribution,
user: row.user,
confirmedByUser: row.confirmedByUser,
}
try {
return v.parse(creationTransactionDbSchema, item)
} catch (e) {
throw new DatabaseError('loadCreations', item, e as Error)
}
})
}
buildTransaction(
item: CreationTransactionDb,
communityContext: CommunityContext,
recipientKeyPair: KeyPairEd25519,
signerKeyPair: KeyPairEd25519,
): GradidoTransactionBuilder {
return this.transactionBuilder
.setCreatedAt(item.confirmedAt)
.setRecipientCommunity(communityContext.communityId)
.addMemo(
new EncryptedMemo(
item.memo,
new AuthenticatedEncryption(communityContext.keyPair),
new AuthenticatedEncryption(recipientKeyPair),
),
)
.setTransactionCreation(
new TransferAmount(
recipientKeyPair.getPublicKey(),
item.amount,
communityContext.communityId,
),
item.contributionDate,
)
.sign(signerKeyPair)
}
calculateAccountBalances(
item: CreationTransactionDb,
communityContext: CommunityContext,
recipientPublicKey: MemoryBlockPtr,
): AccountBalances {
this.accountBalances.clear()
const balance = this.getLastBalanceForUser(
recipientPublicKey,
communityContext.blockchain,
communityContext.communityId,
)
// calculate decay since last balance with legacy calculation method
balance.updateLegacyDecay(item.amount, item.confirmedAt)
communityContext.aufBalance.updateLegacyDecay(item.amount, item.confirmedAt)
communityContext.gmwBalance.updateLegacyDecay(item.amount, item.confirmedAt)
this.accountBalances.add(balance.getAccountBalance())
this.accountBalances.add(communityContext.aufBalance.getAccountBalance())
this.accountBalances.add(communityContext.gmwBalance.getAccountBalance())
return this.accountBalances
}
pushToBlockchain(item: CreationTransactionDb): void {
const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid)
const blockchain = communityContext.blockchain
if (item.confirmedByUser.communityUuid !== item.user.communityUuid) {
throw new Error(
`contribution was confirmed from other community: ${JSON.stringify(item, null, 2)}`,
)
}
const recipientKeyPair = this.getAccountKeyPair(communityContext, item.user.gradidoId)
const recipientPublicKey = recipientKeyPair.getPublicKey()
const signerKeyPair = this.getAccountKeyPair(communityContext, item.confirmedByUser.gradidoId)
if (!recipientKeyPair || !signerKeyPair || !recipientPublicKey) {
throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`)
}
try {
addToBlockchain(
this.buildTransaction(item, communityContext, recipientKeyPair, signerKeyPair).build(),
blockchain,
new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_CONTRIBUTION_ID),
this.calculateAccountBalances(item, communityContext, recipientPublicKey),
)
} catch (e) {
const f = new Filter()
f.transactionType = TransactionType_CREATION
f.searchDirection = SearchDirection_DESC
f.pagination.size = 1
const lastContribution = blockchain.findOne(f)
if (lastContribution) {
this.context.logger.warn(
`last contribution: ${lastContribution.getConfirmedTransaction()?.toJson(true)}`,
)
}
throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error)
}
}
}

205
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts Normal file
View File

@ -0,0 +1,205 @@
import { and, asc, eq, gt, isNotNull, lt, or } from 'drizzle-orm'
import {
AccountBalance,
AccountBalances,
Filter,
GradidoDeferredTransfer,
GradidoTransactionBuilder,
GradidoTransfer,
GradidoUnit,
KeyPairEd25519,
LedgerAnchor,
MemoryBlockPtr,
TransferAmount,
} from 'gradido-blockchain-js'
import * as v from 'valibot'
import { deriveFromCode } from '../../../../data/deriveKeyPair'
import { Uuidv4 } from '../../../../schemas/typeGuard.schema'
import { addToBlockchain } from '../../blockchain'
import { Context } from '../../Context'
import { Balance } from '../../data/Balance'
import { transactionLinksTable, usersTable } from '../../drizzle.schema'
import { BlockchainError, DatabaseError } from '../../errors'
import { toMysqlDateTime } from '../../utils'
import {
CommunityContext,
DeletedTransactionLinkDb,
deletedTransactionLinKDbSchema,
} from '../../valibot.schema'
import { AbstractSyncRole, IndexType } from './AbstractSync.role'
export class DeletedTransactionLinksSyncRole extends AbstractSyncRole<DeletedTransactionLinkDb> {
constructor(context: Context) {
super(context)
this.accountBalances.reserve(2)
}
getDate(): Date {
return this.peek().deletedAt
}
getCommunityUuids(): Uuidv4[] {
return [this.peek().user.communityUuid]
}
getLastIndex(): IndexType {
const lastItem = this.peekLast()
return { date: lastItem.deletedAt, id: lastItem.id }
}
itemTypeName(): string {
return 'deletedTransactionLinks'
}
async loadFromDb(lastIndex: IndexType, count: number): Promise<DeletedTransactionLinkDb[]> {
const result = await this.context.db
.select({
transactionLink: transactionLinksTable,
user: usersTable,
})
.from(transactionLinksTable)
.where(
and(
isNotNull(transactionLinksTable.deletedAt),
lt(transactionLinksTable.deletedAt, transactionLinksTable.validUntil),
or(
gt(transactionLinksTable.deletedAt, toMysqlDateTime(lastIndex.date)),
and(
eq(transactionLinksTable.deletedAt, toMysqlDateTime(lastIndex.date)),
gt(transactionLinksTable.id, lastIndex.id),
),
),
),
)
.innerJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id))
.orderBy(asc(transactionLinksTable.deletedAt), asc(transactionLinksTable.id))
.limit(count)
return result.map((row) => {
const item = {
...row.transactionLink,
user: row.user,
}
try {
return v.parse(deletedTransactionLinKDbSchema, item)
} catch (e) {
throw new DatabaseError('loadDeletedTransactionLinks', item, e as Error)
}
})
}
buildTransaction(
communityContext: CommunityContext,
item: DeletedTransactionLinkDb,
linkFundingTransactionNr: number,
restAmount: GradidoUnit,
senderKeyPair: KeyPairEd25519,
linkFundingPublicKey: MemoryBlockPtr,
): GradidoTransactionBuilder {
return this.transactionBuilder
.setCreatedAt(item.deletedAt)
.setSenderCommunity(communityContext.communityId)
.setRedeemDeferredTransfer(
linkFundingTransactionNr,
new GradidoTransfer(
new TransferAmount(
senderKeyPair.getPublicKey(),
restAmount,
communityContext.communityId,
),
linkFundingPublicKey,
),
)
.sign(senderKeyPair)
}
calculateBalances(
item: DeletedTransactionLinkDb,
fundingTransaction: GradidoDeferredTransfer,
senderLastBalance: Balance,
communityContext: CommunityContext,
senderPublicKey: MemoryBlockPtr,
): AccountBalances {
this.accountBalances.clear()
const fundingUserLastBalance = this.getLastBalanceForUser(
fundingTransaction.getSenderPublicKey()!,
communityContext.blockchain,
communityContext.communityId,
)
fundingUserLastBalance.updateLegacyDecay(senderLastBalance.getBalance(), item.deletedAt)
// account of link is set to zero, gdd will be send back to initiator
this.accountBalances.add(
new AccountBalance(senderPublicKey, GradidoUnit.zero(), communityContext.communityId),
)
this.accountBalances.add(fundingUserLastBalance.getAccountBalance())
return this.accountBalances
}
pushToBlockchain(item: DeletedTransactionLinkDb): void {
const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid)
const blockchain = communityContext.blockchain
const senderKeyPair = deriveFromCode(item.code)
const senderPublicKey = senderKeyPair.getPublicKey()
if (!senderKeyPair || !senderPublicKey) {
throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`)
}
const transaction = blockchain.findOne(Filter.lastBalanceFor(senderPublicKey))
if (!transaction) {
throw new Error(`expect transaction for code: ${item.code}`)
}
// should be funding transaction
if (!transaction.isDeferredTransfer()) {
throw new Error(
`expect funding transaction: ${transaction.getConfirmedTransaction()?.toJson(true)}`,
)
}
const body = transaction
.getConfirmedTransaction()
?.getGradidoTransaction()
?.getTransactionBody()
const deferredTransfer = body?.getDeferredTransfer()
if (!deferredTransfer || !deferredTransfer.getRecipientPublicKey()?.equal(senderPublicKey)) {
throw new Error(
`expect funding transaction to belong to code: ${item.code}: ${transaction.getConfirmedTransaction()?.toJson(true)}`,
)
}
const linkFundingPublicKey = deferredTransfer.getSenderPublicKey()
if (!linkFundingPublicKey) {
throw new Error(`missing sender public key of transaction link founder`)
}
const senderLastBalance = this.getLastBalanceForUser(
senderPublicKey,
communityContext.blockchain,
communityContext.communityId,
)
senderLastBalance.updateLegacyDecay(GradidoUnit.zero(), item.deletedAt)
try {
addToBlockchain(
this.buildTransaction(
communityContext,
item,
transaction.getTransactionNr(),
senderLastBalance.getBalance(),
senderKeyPair,
linkFundingPublicKey,
).build(),
blockchain,
new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_LINK_ID),
this.calculateBalances(
item,
deferredTransfer,
senderLastBalance,
communityContext,
senderPublicKey,
),
)
} catch (e) {
throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error)
}
}
}

186
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts Normal file
View File

@ -0,0 +1,186 @@
import { and, asc, eq, gt, isNotNull, isNull, or } from 'drizzle-orm'
import { alias } from 'drizzle-orm/mysql-core'
import {
AccountBalances,
AuthenticatedEncryption,
EncryptedMemo,
GradidoTransactionBuilder,
KeyPairEd25519,
LedgerAnchor,
MemoryBlockPtr,
TransferAmount,
} from 'gradido-blockchain-js'
import * as v from 'valibot'
import { Uuidv4 } from '../../../../schemas/typeGuard.schema'
import { addToBlockchain } from '../../blockchain'
import { Context } from '../../Context'
import { TransactionTypeId } from '../../data/TransactionTypeId'
import { transactionsTable, usersTable } from '../../drizzle.schema'
import {
BlockchainError,
DatabaseError,
NegativeBalanceError,
NotEnoughGradidoBalanceError,
} from '../../errors'
import { toMysqlDateTime } from '../../utils'
import { CommunityContext, TransactionDb, transactionDbSchema } from '../../valibot.schema'
import { AbstractSyncRole, IndexType } from './AbstractSync.role'
export class LocalTransactionsSyncRole extends AbstractSyncRole<TransactionDb> {
constructor(context: Context) {
super(context)
this.accountBalances.reserve(2)
}
getDate(): Date {
return this.peek().balanceDate
}
getCommunityUuids(): Uuidv4[] {
return [this.peek().user.communityUuid]
}
getLastIndex(): IndexType {
const lastItem = this.peekLast()
return { date: lastItem.balanceDate, id: lastItem.id }
}
itemTypeName(): string {
return 'localTransactions'
}
async loadFromDb(lastIndex: IndexType, count: number): Promise<TransactionDb[]> {
const linkedUsers = alias(usersTable, 'linkedUser')
const result = await this.context.db
.select({
transaction: transactionsTable,
user: usersTable,
linkedUser: linkedUsers,
})
.from(transactionsTable)
.where(
and(
eq(transactionsTable.typeId, TransactionTypeId.RECEIVE),
isNull(transactionsTable.transactionLinkId),
isNotNull(transactionsTable.linkedUserId),
eq(usersTable.foreign, 0),
eq(linkedUsers.foreign, 0),
or(
gt(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)),
and(
eq(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)),
gt(transactionsTable.id, lastIndex.id),
),
),
),
)
.innerJoin(usersTable, eq(transactionsTable.userId, usersTable.id))
.innerJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id))
.orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id))
.limit(count)
return result.map((row) => {
const item = {
...row.transaction,
user: row.user,
linkedUser: row.linkedUser,
}
try {
return v.parse(transactionDbSchema, item)
} catch (e) {
throw new DatabaseError('loadLocalTransferTransactions', item, e as Error)
}
})
}
buildTransaction(
communityContext: CommunityContext,
item: TransactionDb,
senderKeyPair: KeyPairEd25519,
recipientKeyPair: KeyPairEd25519,
): GradidoTransactionBuilder {
return this.transactionBuilder
.setCreatedAt(item.balanceDate)
.addMemo(
new EncryptedMemo(
item.memo,
new AuthenticatedEncryption(senderKeyPair),
new AuthenticatedEncryption(recipientKeyPair),
),
)
.setSenderCommunity(communityContext.communityId)
.setTransactionTransfer(
new TransferAmount(senderKeyPair.getPublicKey(), item.amount, communityContext.communityId),
recipientKeyPair.getPublicKey(),
)
.sign(senderKeyPair)
}
calculateBalances(
item: TransactionDb,
communityContext: CommunityContext,
senderPublicKey: MemoryBlockPtr,
recipientPublicKey: MemoryBlockPtr,
): AccountBalances {
this.accountBalances.clear()
const senderLastBalance = this.getLastBalanceForUser(
senderPublicKey,
communityContext.blockchain,
communityContext.communityId,
)
const recipientLastBalance = this.getLastBalanceForUser(
recipientPublicKey,
communityContext.blockchain,
communityContext.communityId,
)
try {
senderLastBalance.updateLegacyDecay(item.amount.negated(), item.balanceDate)
} catch (e) {
if (e instanceof NegativeBalanceError) {
this.logLastBalanceChangingTransactions(senderPublicKey, communityContext.blockchain)
throw e
}
}
recipientLastBalance.updateLegacyDecay(item.amount, item.balanceDate)
this.accountBalances.add(senderLastBalance.getAccountBalance())
this.accountBalances.add(recipientLastBalance.getAccountBalance())
return this.accountBalances
}
pushToBlockchain(item: TransactionDb): void {
const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid)
const blockchain = communityContext.blockchain
if (item.linkedUser.communityUuid !== item.user.communityUuid) {
throw new Error(
`transfer between user from different communities: ${JSON.stringify(item, null, 2)}`,
)
}
// I use the received transaction so user and linked user are swapped and user is recipient and linkedUser ist sender
const senderKeyPair = this.getAccountKeyPair(communityContext, item.linkedUser.gradidoId)
const senderPublicKey = senderKeyPair.getPublicKey()
const recipientKeyPair = this.getAccountKeyPair(communityContext, item.user.gradidoId)
const recipientPublicKey = recipientKeyPair.getPublicKey()
if (!senderKeyPair || !senderPublicKey || !recipientKeyPair || !recipientPublicKey) {
throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`)
}
try {
addToBlockchain(
this.buildTransaction(communityContext, item, senderKeyPair, recipientKeyPair).build(),
blockchain,
new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_ID),
this.calculateBalances(item, communityContext, senderPublicKey, recipientPublicKey),
)
} catch (e) {
if (e instanceof NotEnoughGradidoBalanceError) {
this.logLastBalanceChangingTransactions(senderPublicKey, blockchain)
}
throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error)
}
}
}

230
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts Normal file
View File

@ -0,0 +1,230 @@
import { and, asc, eq, gt, isNotNull, isNull, or } from 'drizzle-orm'
import { alias } from 'drizzle-orm/mysql-core'
import {
AccountBalance,
AccountBalances,
AuthenticatedEncryption,
EncryptedMemo,
Filter,
GradidoDeferredTransfer,
GradidoTransactionBuilder,
GradidoTransfer,
GradidoUnit,
KeyPairEd25519,
LedgerAnchor,
MemoryBlockPtr,
TransferAmount,
} from 'gradido-blockchain-js'
import * as v from 'valibot'
import { deriveFromCode } from '../../../../data/deriveKeyPair'
import { Uuidv4 } from '../../../../schemas/typeGuard.schema'
import { addToBlockchain } from '../../blockchain'
import { Context } from '../../Context'
import { transactionLinksTable, usersTable } from '../../drizzle.schema'
import { BlockchainError, DatabaseError } from '../../errors'
import { toMysqlDateTime } from '../../utils'
import {
CommunityContext,
RedeemedTransactionLinkDb,
redeemedTransactionLinkDbSchema,
} from '../../valibot.schema'
import { AbstractSyncRole, IndexType } from './AbstractSync.role'
export class RedeemTransactionLinksSyncRole extends AbstractSyncRole<RedeemedTransactionLinkDb> {
constructor(context: Context) {
super(context)
this.accountBalances.reserve(3)
}
getDate(): Date {
return this.peek().redeemedAt
}
getCommunityUuids(): Uuidv4[] {
return [this.peek().user.communityUuid]
}
getLastIndex(): IndexType {
const lastItem = this.peekLast()
return { date: lastItem.redeemedAt, id: lastItem.id }
}
itemTypeName(): string {
return 'redeemTransactionLinks'
}
async loadFromDb(lastIndex: IndexType, count: number): Promise<RedeemedTransactionLinkDb[]> {
const redeemedByUser = alias(usersTable, 'redeemedByUser')
const result = await this.context.db
.select({
transactionLink: transactionLinksTable,
user: usersTable,
redeemedBy: redeemedByUser,
})
.from(transactionLinksTable)
.where(
and(
isNull(transactionLinksTable.deletedAt),
isNotNull(transactionLinksTable.redeemedAt),
eq(usersTable.foreign, 0),
eq(redeemedByUser.foreign, 0),
or(
gt(transactionLinksTable.redeemedAt, toMysqlDateTime(lastIndex.date)),
and(
eq(transactionLinksTable.redeemedAt, toMysqlDateTime(lastIndex.date)),
gt(transactionLinksTable.id, lastIndex.id),
),
),
),
)
.innerJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id))
.innerJoin(redeemedByUser, eq(transactionLinksTable.redeemedBy, redeemedByUser.id))
.orderBy(asc(transactionLinksTable.redeemedAt), asc(transactionLinksTable.id))
.limit(count)
return result.map((row) => {
const item = {
...row.transactionLink,
redeemedBy: row.redeemedBy,
user: row.user,
}
try {
return v.parse(redeemedTransactionLinkDbSchema, item)
} catch (e) {
throw new DatabaseError('loadRedeemTransactionLinks', item, e as Error)
}
})
}
buildTransaction(
communityContext: CommunityContext,
item: RedeemedTransactionLinkDb,
linkFundingTransactionNr: number,
senderKeyPair: KeyPairEd25519,
recipientKeyPair: KeyPairEd25519,
): GradidoTransactionBuilder {
return this.transactionBuilder
.setCreatedAt(item.redeemedAt)
.addMemo(
new EncryptedMemo(
item.memo,
new AuthenticatedEncryption(senderKeyPair),
new AuthenticatedEncryption(recipientKeyPair),
),
)
.setSenderCommunity(communityContext.communityId)
.setRedeemDeferredTransfer(
linkFundingTransactionNr,
new GradidoTransfer(
new TransferAmount(
senderKeyPair.getPublicKey(),
item.amount,
communityContext.communityId,
),
recipientKeyPair.getPublicKey(),
),
)
.sign(senderKeyPair)
}
calculateBalances(
item: RedeemedTransactionLinkDb,
fundingTransaction: GradidoDeferredTransfer,
communityContext: CommunityContext,
senderPublicKey: MemoryBlockPtr,
recipientPublicKey: MemoryBlockPtr,
): AccountBalances {
this.accountBalances.clear()
const senderLastBalance = this.getLastBalanceForUser(
senderPublicKey,
communityContext.blockchain,
communityContext.communityId,
)
const fundingUserLastBalance = this.getLastBalanceForUser(
fundingTransaction.getSenderPublicKey()!,
communityContext.blockchain,
communityContext.communityId,
)
const recipientLastBalance = this.getLastBalanceForUser(
recipientPublicKey,
communityContext.blockchain,
communityContext.communityId,
)
if (senderLastBalance.getAccountBalance().getBalance().lt(item.amount)) {
throw new Error(
`sender has not enough balance (${senderLastBalance.getAccountBalance().getBalance().toString()}) to send ${item.amount.toString()} to ${recipientPublicKey.convertToHex()}`,
)
}
senderLastBalance.updateLegacyDecay(item.amount.negated(), item.redeemedAt)
fundingUserLastBalance.updateLegacyDecay(senderLastBalance.getBalance(), item.redeemedAt)
recipientLastBalance.updateLegacyDecay(item.amount, item.redeemedAt)
// account of link is set to zero, and change send back to link creator
this.accountBalances.add(
new AccountBalance(senderPublicKey, GradidoUnit.zero(), communityContext.communityId),
)
this.accountBalances.add(recipientLastBalance.getAccountBalance())
this.accountBalances.add(fundingUserLastBalance.getAccountBalance())
return this.accountBalances
}
pushToBlockchain(item: RedeemedTransactionLinkDb): void {
const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid)
const blockchain = communityContext.blockchain
const senderKeyPair = deriveFromCode(item.code)
const senderPublicKey = senderKeyPair.getPublicKey()
const recipientKeyPair = this.getAccountKeyPair(communityContext, item.redeemedBy.gradidoId)
const recipientPublicKey = recipientKeyPair.getPublicKey()
if (!senderKeyPair || !senderPublicKey || !recipientKeyPair || !recipientPublicKey) {
throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`)
}
const transaction = blockchain.findOne(Filter.lastBalanceFor(senderPublicKey))
if (!transaction) {
throw new Error(`expect transaction for code: ${item.code}`)
}
// should be funding transaction
if (!transaction.isDeferredTransfer()) {
throw new Error(
`expect funding transaction: ${transaction.getConfirmedTransaction()?.toJson(true)}`,
)
}
const body = transaction
.getConfirmedTransaction()
?.getGradidoTransaction()
?.getTransactionBody()
const deferredTransfer = body?.getDeferredTransfer()
if (!deferredTransfer || !deferredTransfer.getRecipientPublicKey()?.equal(senderPublicKey)) {
throw new Error(
`expect funding transaction to belong to code: ${item.code}: ${transaction.getConfirmedTransaction()?.toJson(true)}`,
)
}
try {
addToBlockchain(
this.buildTransaction(
communityContext,
item,
transaction.getTransactionNr(),
senderKeyPair,
recipientKeyPair,
).build(),
blockchain,
new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_LINK_ID),
this.calculateBalances(
item,
deferredTransfer,
communityContext,
senderPublicKey,
recipientPublicKey,
),
)
} catch (e) {
throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error)
}
}
}

266
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts Normal file
View File

@ -0,0 +1,266 @@
import { Decimal } from 'decimal.js-light'
import { and, asc, eq, gt, inArray, isNull, ne, or } from 'drizzle-orm'
import { alias } from 'drizzle-orm/mysql-core'
import {
AccountBalance,
AccountBalances,
AuthenticatedEncryption,
EncryptedMemo,
GradidoTransactionBuilder,
GradidoUnit,
KeyPairEd25519,
LedgerAnchor,
MemoryBlockPtr,
TransferAmount,
} from 'gradido-blockchain-js'
import * as v from 'valibot'
import { Uuidv4 } from '../../../../schemas/typeGuard.schema'
import { addToBlockchain } from '../../blockchain'
import { Context } from '../../Context'
import { TransactionTypeId } from '../../data/TransactionTypeId'
import { transactionsTable, usersTable } from '../../drizzle.schema'
import {
BlockchainError,
DatabaseError,
NegativeBalanceError,
NotEnoughGradidoBalanceError,
} from '../../errors'
import { toMysqlDateTime } from '../../utils'
import { CommunityContext, TransactionDb, transactionDbSchema, UserDb } from '../../valibot.schema'
import { AbstractSyncRole, IndexType } from './AbstractSync.role'
export class RemoteTransactionsSyncRole extends AbstractSyncRole<TransactionDb> {
constructor(context: Context) {
super(context)
this.accountBalances.reserve(1)
}
getDate(): Date {
return this.peek().balanceDate
}
getCommunityUuids(): Uuidv4[] {
const currentItem = this.peek()
return [currentItem.user.communityUuid, currentItem.linkedUser.communityUuid]
}
getLastIndex(): IndexType {
const lastItem = this.peekLast()
return { date: lastItem.balanceDate, id: lastItem.id }
}
itemTypeName(): string {
return 'remoteTransactions'
}
async loadFromDb(lastIndex: IndexType, count: number): Promise<TransactionDb[]> {
const linkedUsers = alias(usersTable, 'linkedUser')
const result = await this.context.db
.select({
transaction: transactionsTable,
user: usersTable,
linkedUser: linkedUsers,
})
.from(transactionsTable)
.where(
and(
inArray(transactionsTable.typeId, [TransactionTypeId.RECEIVE, TransactionTypeId.SEND]),
isNull(transactionsTable.transactionLinkId),
ne(usersTable.communityUuid, linkedUsers.communityUuid),
or(
gt(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)),
and(
eq(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)),
gt(transactionsTable.id, lastIndex.id),
),
),
),
)
.innerJoin(usersTable, eq(transactionsTable.userId, usersTable.id))
.innerJoin(linkedUsers, eq(transactionsTable.linkedUserGradidoId, linkedUsers.gradidoId))
.orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id))
.limit(count)
return result.map((row) => {
const item = {
...row.transaction,
user: row.user,
linkedUser: row.linkedUser,
}
if (item.typeId === TransactionTypeId.SEND && item.amount) {
item.amount = new Decimal(item.amount).neg().toString()
}
try {
return v.parse(transactionDbSchema, item)
} catch (e) {
throw new DatabaseError('loadRemoteTransferTransactions', item, e as Error)
}
})
}
buildTransaction(
item: TransactionDb,
senderKeyPair: KeyPairEd25519,
recipientKeyPair: KeyPairEd25519,
senderCommunityId: string,
recipientCommunityId: string,
): GradidoTransactionBuilder {
return this.transactionBuilder
.setCreatedAt(item.balanceDate)
.addMemo(
new EncryptedMemo(
item.memo,
new AuthenticatedEncryption(senderKeyPair),
new AuthenticatedEncryption(recipientKeyPair),
),
)
.setSenderCommunity(senderCommunityId)
.setRecipientCommunity(recipientCommunityId)
.setTransactionTransfer(
new TransferAmount(senderKeyPair.getPublicKey(), item.amount, senderCommunityId),
recipientKeyPair.getPublicKey(),
)
.sign(senderKeyPair)
}
calculateBalances(
item: TransactionDb,
communityContext: CommunityContext,
coinCommunityId: string,
amount: GradidoUnit,
publicKey: MemoryBlockPtr,
): AccountBalances {
this.accountBalances.clear()
// try to use same coins from this community
let lastBalance = this.getLastBalanceForUser(
publicKey,
communityContext.blockchain,
coinCommunityId,
)
if (
coinCommunityId !== communityContext.communityId &&
(lastBalance.getBalance().equal(GradidoUnit.zero()) ||
lastBalance.getBalance().calculateDecay(lastBalance.getDate(), item.balanceDate).lt(amount))
) {
// don't work, so we use or own coins
lastBalance = this.getLastBalanceForUser(
publicKey,
communityContext.blockchain,
communityContext.communityId,
)
}
if (
lastBalance
.getBalance()
.calculateDecay(lastBalance.getDate(), item.balanceDate)
.add(amount)
.lt(GradidoUnit.zero()) &&
communityContext.foreign
) {
this.accountBalances.add(new AccountBalance(publicKey, GradidoUnit.zero(), coinCommunityId))
return this.accountBalances
}
try {
lastBalance.updateLegacyDecay(amount, item.balanceDate)
} catch (e) {
if (e instanceof NegativeBalanceError) {
this.logLastBalanceChangingTransactions(publicKey, communityContext.blockchain, 1)
throw e
}
}
this.accountBalances.add(lastBalance.getAccountBalance())
return this.accountBalances
}
getUser(item: TransactionDb): { senderUser: UserDb; recipientUser: UserDb } {
return item.typeId === TransactionTypeId.RECEIVE
? { senderUser: item.linkedUser, recipientUser: item.user }
: { senderUser: item.user, recipientUser: item.linkedUser }
}
pushToBlockchain(item: TransactionDb): void {
const { senderUser, recipientUser } = this.getUser(item)
const ledgerAnchor = new LedgerAnchor(
item.id,
LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_ID,
)
if (senderUser.communityUuid === recipientUser.communityUuid) {
throw new Error(
`transfer between user from same community: ${JSON.stringify(item, null, 2)}, check db query`,
)
}
const senderCommunityContext = this.context.getCommunityContextByUuid(senderUser.communityUuid)
const recipientCommunityContext = this.context.getCommunityContextByUuid(
recipientUser.communityUuid,
)
const senderBlockchain = senderCommunityContext.blockchain
const recipientBlockchain = recipientCommunityContext.blockchain
// I use the received transaction so user and linked user are swapped and user is recipient and linkedUser ist sender
const senderKeyPair = this.getAccountKeyPair(senderCommunityContext, senderUser.gradidoId)
const senderPublicKey = senderKeyPair.getPublicKey()
const recipientKeyPair = this.getAccountKeyPair(
recipientCommunityContext,
recipientUser.gradidoId,
)
const recipientPublicKey = recipientKeyPair.getPublicKey()
if (!senderKeyPair || !senderPublicKey || !recipientKeyPair || !recipientPublicKey) {
throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`)
}
const transactionBuilder = this.buildTransaction(
item,
senderKeyPair,
recipientKeyPair,
senderCommunityContext.communityId,
recipientCommunityContext.communityId,
)
const outboundTransaction = transactionBuilder.buildOutbound()
try {
addToBlockchain(
outboundTransaction,
senderBlockchain,
ledgerAnchor,
this.calculateBalances(
item,
senderCommunityContext,
senderCommunityContext.communityId,
item.amount.negated(),
senderPublicKey,
),
)
} catch (e) {
if (e instanceof NotEnoughGradidoBalanceError) {
this.logLastBalanceChangingTransactions(senderPublicKey, senderBlockchain)
}
throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error)
}
transactionBuilder.setParentLedgerAnchor(ledgerAnchor)
const inboundTransaction = transactionBuilder.buildInbound()
try {
addToBlockchain(
inboundTransaction,
recipientBlockchain,
ledgerAnchor,
this.calculateBalances(
item,
recipientCommunityContext,
senderCommunityContext.communityId,
item.amount,
recipientPublicKey,
),
)
} catch (e) {
if (e instanceof NotEnoughGradidoBalanceError) {
this.logLastBalanceChangingTransactions(recipientPublicKey, recipientBlockchain)
}
throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error)
}
// this.logLastBalanceChangingTransactions(senderPublicKey, senderCommunityContext.blockchain, 1)
// this.logLastBalanceChangingTransactions(recipientPublicKey, recipientCommunityContext.blockchain, 1)
}
}

229
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts Normal file
View File

@ -0,0 +1,229 @@
import Decimal from 'decimal.js-light'
import { and, asc, eq, gt, or } from 'drizzle-orm'
import {
AccountBalance,
AccountBalances,
AuthenticatedEncryption,
DurationSeconds,
EncryptedMemo,
Filter,
GradidoTransactionBuilder,
GradidoTransfer,
GradidoUnit,
KeyPairEd25519,
LedgerAnchor,
MemoryBlockPtr,
SearchDirection_DESC,
TransferAmount,
transactionTypeToString,
} from 'gradido-blockchain-js'
import * as v from 'valibot'
import { deriveFromCode } from '../../../../data/deriveKeyPair'
import { Uuidv4 } from '../../../../schemas/typeGuard.schema'
import { addToBlockchain } from '../../blockchain'
import { Context } from '../../Context'
import { transactionLinksTable, usersTable } from '../../drizzle.schema'
import { BlockchainError, DatabaseError, NegativeBalanceError } from '../../errors'
import { reverseLegacyDecay, toMysqlDateTime } from '../../utils'
import { CommunityContext, TransactionLinkDb, transactionLinkDbSchema } from '../../valibot.schema'
import { AbstractSyncRole, IndexType } from './AbstractSync.role'
export class TransactionLinkFundingsSyncRole extends AbstractSyncRole<TransactionLinkDb> {
constructor(context: Context) {
super(context)
this.accountBalances.reserve(2)
}
getDate(): Date {
return this.peek().createdAt
}
getCommunityUuids(): Uuidv4[] {
return [this.peek().user.communityUuid]
}
getLastIndex(): IndexType {
const lastItem = this.peekLast()
return { date: lastItem.createdAt, id: lastItem.id }
}
itemTypeName(): string {
return 'transactionLinkFundings'
}
async loadFromDb(lastIndex: IndexType, count: number): Promise<TransactionLinkDb[]> {
const result = await this.context.db
.select()
.from(transactionLinksTable)
.innerJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id))
.where(
or(
gt(transactionLinksTable.createdAt, toMysqlDateTime(lastIndex.date)),
and(
eq(transactionLinksTable.createdAt, toMysqlDateTime(lastIndex.date)),
gt(transactionLinksTable.id, lastIndex.id),
),
),
)
.orderBy(asc(transactionLinksTable.createdAt), asc(transactionLinksTable.id))
.limit(count)
return result.map((row) => {
const item = {
...row.transaction_links,
user: row.users,
}
try {
return v.parse(transactionLinkDbSchema, item)
} catch (e) {
throw new DatabaseError('loadTransactionLinkFundings', item, e as Error)
}
})
}
buildTransaction(
communityContext: CommunityContext,
item: TransactionLinkDb,
blockedAmount: GradidoUnit,
duration: DurationSeconds,
senderKeyPair: KeyPairEd25519,
recipientKeyPair: KeyPairEd25519,
): GradidoTransactionBuilder {
return this.transactionBuilder
.setCreatedAt(item.createdAt)
.addMemo(
new EncryptedMemo(
item.memo,
new AuthenticatedEncryption(senderKeyPair),
new AuthenticatedEncryption(recipientKeyPair),
),
)
.setSenderCommunity(communityContext.communityId)
.setDeferredTransfer(
new GradidoTransfer(
new TransferAmount(
senderKeyPair.getPublicKey(),
blockedAmount,
communityContext.communityId,
),
recipientKeyPair.getPublicKey(),
),
duration,
)
.sign(senderKeyPair)
}
calculateBalances(
item: TransactionLinkDb,
blockedAmount: GradidoUnit,
communityContext: CommunityContext,
senderPublicKey: MemoryBlockPtr,
recipientPublicKey: MemoryBlockPtr,
): AccountBalances {
this.accountBalances.clear()
const senderLastBalance = this.getLastBalanceForUser(
senderPublicKey,
communityContext.blockchain,
communityContext.communityId,
)
try {
senderLastBalance.updateLegacyDecay(blockedAmount.negated(), item.createdAt)
} catch (e) {
if (e instanceof NegativeBalanceError) {
this.logLastBalanceChangingTransactions(senderPublicKey, communityContext.blockchain)
this.context.logger.debug(`sender public key: ${senderPublicKey.convertToHex()}`)
throw e
}
}
this.accountBalances.add(senderLastBalance.getAccountBalance())
this.accountBalances.add(
new AccountBalance(recipientPublicKey, blockedAmount, communityContext.communityId),
)
return this.accountBalances
}
pushToBlockchain(item: TransactionLinkDb): void {
const communityContext = this.context.getCommunityContextByUuid(item.user.communityUuid)
const blockchain = communityContext.blockchain
const senderKeyPair = this.getAccountKeyPair(communityContext, item.user.gradidoId)
const senderPublicKey = senderKeyPair.getPublicKey()
const recipientKeyPair = deriveFromCode(item.code)
const recipientPublicKey = recipientKeyPair.getPublicKey()
if (!senderKeyPair || !senderPublicKey || !recipientKeyPair || !recipientPublicKey) {
throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`)
}
const duration = new DurationSeconds(
(item.validUntil.getTime() - item.createdAt.getTime()) / 1000,
)
let blockedAmount = GradidoUnit.fromString(
reverseLegacyDecay(new Decimal(item.amount.toString()), duration.getSeconds()).toString(),
)
let accountBalances: AccountBalances
try {
accountBalances = this.calculateBalances(
item,
blockedAmount,
communityContext,
senderPublicKey,
recipientPublicKey,
)
} catch (e) {
if (item.deletedAt && e instanceof NegativeBalanceError) {
const senderLastBalance = this.getLastBalanceForUser(
senderPublicKey,
communityContext.blockchain,
communityContext.communityId,
)
senderLastBalance.updateLegacyDecay(GradidoUnit.zero(), item.createdAt)
const oldBlockedAmountString = blockedAmount.toString()
blockedAmount = senderLastBalance.getBalance()
accountBalances = this.calculateBalances(
item,
blockedAmount,
communityContext,
senderPublicKey,
recipientPublicKey,
)
this.context.logger.warn(
`workaround: fix founding for deleted link, reduce funding to actual sender balance: ${senderPublicKey.convertToHex()}: from ${oldBlockedAmountString} GDD to ${blockedAmount.toString()} GDD`,
)
} else {
this.context.logger.error(
`error calculate account balances for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`,
)
throw e
}
}
try {
addToBlockchain(
this.buildTransaction(
communityContext,
item,
blockedAmount,
duration,
senderKeyPair,
recipientKeyPair,
).build(),
blockchain,
new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_LINK_ID),
accountBalances,
)
} catch (e) {
if (e instanceof NegativeBalanceError) {
if (
!item.deletedAt &&
!item.redeemedAt &&
item.validUntil.getTime() < new Date().getTime()
) {
this.context.logger.warn(
`TransactionLinks: ${item.id} skipped, because else it lead to negative balance error, but it wasn't used.`,
)
return
}
}
throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error)
}
}
}

132
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/UsersSync.role.ts Normal file
View File

@ -0,0 +1,132 @@
import { and, asc, eq, gt, isNotNull, or } from 'drizzle-orm'
import {
AccountBalance,
AccountBalances,
AddressType_COMMUNITY_HUMAN,
GradidoTransactionBuilder,
GradidoUnit,
KeyPairEd25519,
LedgerAnchor,
MemoryBlockPtr,
} from 'gradido-blockchain-js'
import * as v from 'valibot'
import { deriveFromKeyPairAndUuid } from '../../../../data/deriveKeyPair'
import { Uuidv4Hash } from '../../../../data/Uuidv4Hash'
import { Uuidv4 } from '../../../../schemas/typeGuard.schema'
import { addToBlockchain } from '../../blockchain'
import { Context } from '../../Context'
import { usersTable } from '../../drizzle.schema'
import { BlockchainError, DatabaseError } from '../../errors'
import { toMysqlDateTime } from '../../utils'
import { CommunityContext, UserDb, userDbSchema } from '../../valibot.schema'
import { AbstractSyncRole, IndexType } from './AbstractSync.role'
export class UsersSyncRole extends AbstractSyncRole<UserDb> {
constructor(context: Context) {
super(context)
this.accountBalances.reserve(1)
}
getDate(): Date {
return this.peek().createdAt
}
getCommunityUuids(): Uuidv4[] {
return [this.peek().communityUuid]
}
getLastIndex(): IndexType {
const lastItem = this.peekLast()
return { date: lastItem.createdAt, id: lastItem.id }
}
itemTypeName(): string {
return 'users'
}
async loadFromDb(lastIndex: IndexType, count: number): Promise<UserDb[]> {
const result = await this.context.db
.select()
.from(usersTable)
.where(
and(
or(
gt(usersTable.createdAt, toMysqlDateTime(lastIndex.date)),
and(
eq(usersTable.createdAt, toMysqlDateTime(lastIndex.date)),
gt(usersTable.id, lastIndex.id),
),
),
isNotNull(usersTable.communityUuid),
),
)
.orderBy(asc(usersTable.createdAt), asc(usersTable.id))
.limit(count)
return result.map((row) => {
try {
return v.parse(userDbSchema, row)
} catch (e) {
throw new DatabaseError('loadUsers', row, e as Error)
}
})
}
buildTransaction(
communityContext: CommunityContext,
item: UserDb,
communityKeyPair: KeyPairEd25519,
accountKeyPair: KeyPairEd25519,
userKeyPair: KeyPairEd25519,
): GradidoTransactionBuilder {
return this.transactionBuilder
.setCreatedAt(item.createdAt)
.setSenderCommunity(communityContext.communityId)
.setRegisterAddress(
userKeyPair.getPublicKey(),
AddressType_COMMUNITY_HUMAN,
new Uuidv4Hash(item.gradidoId).getAsMemoryBlock(),
accountKeyPair.getPublicKey(),
)
.sign(communityKeyPair)
.sign(accountKeyPair)
.sign(userKeyPair)
}
calculateAccountBalances(
accountPublicKey: MemoryBlockPtr,
communityContext: CommunityContext,
): AccountBalances {
this.accountBalances.clear()
this.accountBalances.add(
new AccountBalance(accountPublicKey, GradidoUnit.zero(), communityContext.communityId),
)
return this.accountBalances
}
pushToBlockchain(item: UserDb): void {
const communityContext = this.context.getCommunityContextByUuid(item.communityUuid)
const userKeyPair = deriveFromKeyPairAndUuid(communityContext.keyPair, item.gradidoId)
const accountKeyPair = this.getAccountKeyPair(communityContext, item.gradidoId)
const accountPublicKey = accountKeyPair.getPublicKey()
if (!userKeyPair || !accountKeyPair || !accountPublicKey) {
throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`)
}
try {
addToBlockchain(
this.buildTransaction(
communityContext,
item,
communityContext.keyPair,
accountKeyPair,
userKeyPair,
).build(),
communityContext.blockchain,
new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_USER_ID),
this.calculateAccountBalances(accountPublicKey, communityContext),
)
} catch (e) {
throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error)
}
}
}

161
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts Normal file
View File

@ -0,0 +1,161 @@
import {
Abstract,
Filter,
InteractionCreateTransactionByEvent,
LedgerAnchor,
Profiler,
Timestamp,
} from 'gradido-blockchain-js'
import { Logger } from 'log4js'
import { callTime } from '../../blockchain'
import { Context } from '../../Context'
import { CommunityContext } from '../../valibot.schema'
import { nanosBalanceForUser } from './AbstractSync.role'
import { ContributionLinkTransactionSyncRole } from './ContributionLinkTransactionSync.role'
import { CreationsSyncRole } from './CreationsSync.role'
import { DeletedTransactionLinksSyncRole } from './DeletedTransactionLinksSync.role'
import { LocalTransactionsSyncRole } from './LocalTransactionsSync.role'
import { RedeemTransactionLinksSyncRole } from './RedeemTransactionLinksSync.role'
import { RemoteTransactionsSyncRole } from './RemoteTransactionsSync.role'
import { TransactionLinkFundingsSyncRole } from './TransactionLinkFundingsSync.role'
import { UsersSyncRole } from './UsersSync.role'
function processTransactionTrigger(context: CommunityContext, endDate: Date, logger: Logger) {
while (true) {
const lastTx = context.blockchain.findOne(Filter.LAST_TRANSACTION)
let confirmedAt: Timestamp | undefined
if (!lastTx) {
// no transaction, no triggers
return
} else {
const confirmedTx = lastTx.getConfirmedTransaction()
if (!confirmedTx) {
throw new Error('missing confirmed tx in transaction entry')
}
confirmedAt = confirmedTx.getConfirmedAt()
}
const triggerEvent = context.blockchain.findNextTransactionTriggerEventInRange(
confirmedAt,
new Timestamp(endDate),
)
if (!triggerEvent) {
// no trigger, we can exit here
return
}
context.blockchain.removeTransactionTriggerEvent(triggerEvent)
try {
// InMemoryBlockchain extend Abstract, but between C++ -> Swig -> TypeScript it seems the info is gone, so I need to cheat a bit here
const createTransactionByEvent = new InteractionCreateTransactionByEvent(
context.blockchain as unknown as Abstract,
)
if (
!context.blockchain.createAndAddConfirmedTransaction(
createTransactionByEvent.run(triggerEvent),
new LedgerAnchor(
triggerEvent.getLinkedTransactionId(),
LedgerAnchor.Type_NODE_TRIGGER_TRANSACTION_ID,
),
triggerEvent.getTargetDate(),
)
) {
throw new Error('Adding trigger created Transaction Failed')
}
} catch (e) {
context.blockchain.addTransactionTriggerEvent(triggerEvent)
logger.error(
`Error processing transaction trigger event for transaction: ${triggerEvent.getLinkedTransactionId()}`,
)
throw e
}
}
}
export async function syncDbWithBlockchainContext(context: Context, batchSize: number) {
const timeUsedDB = new Profiler()
const timeUsedBlockchain = new Profiler()
const timeUsedAll = new Profiler()
const timeBetweenPrints = new Profiler()
const containers = [
new UsersSyncRole(context),
new CreationsSyncRole(context),
new LocalTransactionsSyncRole(context),
new TransactionLinkFundingsSyncRole(context),
new RedeemTransactionLinksSyncRole(context),
new ContributionLinkTransactionSyncRole(context),
new DeletedTransactionLinksSyncRole(context),
new RemoteTransactionsSyncRole(context),
]
let transactionsCount = 0
let transactionsCountSinceLastLog = 0
let transactionsCountSinceLastPrint = 0
let available = containers
const isDebug = context.logger.isDebugEnabled()
let lastPrintedCallTime = 0
while (true) {
timeUsedDB.reset()
const results = await Promise.all(available.map((c) => c.ensureFilled(batchSize)))
const loadedItemsCount = results.reduce((acc, c) => acc + c, 0)
// log only, if at least one new item was loaded
if (loadedItemsCount && isDebug) {
context.logger.debug(`${loadedItemsCount} new items loaded from db in ${timeUsedDB.string()}`)
}
// remove empty containers
available = available.filter((c) => !c.isEmpty())
if (available.length === 0) {
break
}
// sort by date, to ensure container on index 0 is the one with the smallest date
if (available.length > 1) {
// const sortTime = new Profiler()
available.sort((a, b) => a.getDate().getTime() - b.getDate().getTime())
// context.logger.debug(`sorted ${available.length} containers in ${sortTime.string()}`)
}
const communityUuids = available[0].getCommunityUuids()
for (let i = 0; i < communityUuids.length; ++i) {
processTransactionTrigger(
context.getCommunityContextByUuid(communityUuids[i]),
available[0].getDate(),
context.logger,
)
}
available[0].toBlockchain()
transactionsCount++
if (isDebug) {
if (timeBetweenPrints.millis() > 100) {
process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`)
timeBetweenPrints.reset()
}
transactionsCountSinceLastLog++
if (transactionsCountSinceLastLog >= batchSize) {
context.logger.debug(
`${transactionsCountSinceLastLog} transactions added to blockchain in ${timeUsedBlockchain.string()}`,
)
context.logger.info(
`Time for createAndConfirm: ${((callTime - lastPrintedCallTime) / 1000 / 1000).toFixed(2)} milliseconds`,
)
lastPrintedCallTime = callTime
timeUsedBlockchain.reset()
transactionsCountSinceLastLog = 0
}
} else {
transactionsCountSinceLastPrint++
if (transactionsCountSinceLastPrint >= 100) {
process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`)
transactionsCountSinceLastPrint = 0
}
}
}
process.stdout.write(`successfully added to blockchain: ${transactionsCount}\n`)
context.logger.info(
`Synced ${transactionsCount} transactions to blockchain in ${timeUsedAll.string()}`,
)
context.logger.info(
`Time for createAndConfirm: ${(callTime / 1000 / 1000 / 1000).toFixed(2)} seconds`,
)
context.logger.info(
`Time for call lastBalance of user: ${(nanosBalanceForUser / 1000 / 1000 / 1000).toFixed(2)} seconds`,
)
}

70
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/utils.ts Normal file
View File

@ -0,0 +1,70 @@
import Decimal from 'decimal.js-light'
import { crypto_generichash_batch, crypto_generichash_KEYBYTES } from 'sodium-native'
export function bytesToMbyte(bytes: number): string {
return (bytes / 1024 / 1024).toFixed(4)
}
export function bytesToKbyte(bytes: number): string {
return (bytes / 1024).toFixed(0)
}
export function bytesString(bytes: number): string {
if (bytes > 1024 * 1024) {
return `${bytesToMbyte(bytes)} MB`
} else if (bytes > 1024) {
return `${bytesToKbyte(bytes)} KB`
}
return `${bytes.toFixed(0)} Bytes`
}
export function toMysqlDateTime(date: Date): string {
return date.toISOString().slice(0, 23).replace('T', ' ')
}
export function calculateOneHashStep(hash: Buffer, data: Buffer): Buffer<ArrayBuffer> {
const outputHash = Buffer.alloc(crypto_generichash_KEYBYTES, 0)
crypto_generichash_batch(outputHash, [hash, data])
return outputHash
}
export const DECAY_START_TIME = new Date('2021-05-13T17:46:31Z')
export const SECONDS_PER_YEAR_GREGORIAN_CALENDER = 31556952.0
const FACTOR = new Decimal('0.99999997803504048973201202316767079413460520837376')
export function legacyDecayFormula(value: Decimal, seconds: number): Decimal {
// TODO why do we need to convert this here to a string to work properly?
// chatgpt: We convert to string here to avoid precision loss:
// .pow(seconds) can internally round the result, especially for large values of `seconds`.
// Using .toString() ensures full precision is preserved in the multiplication.
return value.mul(FACTOR.pow(seconds).toString())
}
export function reverseLegacyDecay(result: Decimal, seconds: number): Decimal {
return result.div(FACTOR.pow(seconds).toString())
}
export function legacyCalculateDecay(amount: Decimal, from: Date, to: Date): Decimal {
const fromMs = from.getTime()
const toMs = to.getTime()
const startBlockMs = DECAY_START_TIME.getTime()
if (toMs < fromMs) {
throw new Error(
`calculateDecay: to (${to.toISOString()}) < from (${from.toISOString()}), reverse decay calculation is invalid`,
)
}
// decay started after end date; no decay
if (startBlockMs > toMs) {
return amount
}
// decay started before start date; decay for full duration
let duration = (toMs - fromMs) / 1000
// decay started between start and end date; decay from decay start till end date
if (startBlockMs >= fromMs) {
duration = (toMs - startBlockMs) / 1000
}
return legacyDecayFormula(amount, duration)
}

174
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/valibot.schema.ts Normal file
View File

@ -0,0 +1,174 @@
import Decimal from 'decimal.js-light'
import { GradidoUnit, InMemoryBlockchain, KeyPairEd25519 } from 'gradido-blockchain-js'
import * as v from 'valibot'
import { booleanSchema, dateSchema } from '../../schemas/typeConverter.schema'
import {
gradidoAmountSchema,
identifierSeedSchema,
memoSchema,
uuidv4Schema,
} from '../../schemas/typeGuard.schema'
import { Balance } from './data/Balance'
import { TransactionTypeId } from './data/TransactionTypeId'
const positiveNumberSchema = v.pipe(v.number(), v.minValue(1))
export const userDbSchema = v.object({
id: positiveNumberSchema,
gradidoId: uuidv4Schema,
communityUuid: uuidv4Schema,
createdAt: dateSchema,
})
/*
declare const validLegacyAmount: unique symbol
export type LegacyAmount = string & { [validLegacyAmount]: true }
export const legacyAmountSchema = v.pipe(
v.string(),
v.regex(/^-?[0-9]+(\.[0-9]+)?$/),
v.transform<string, LegacyAmount>((input: string) => input as LegacyAmount),
)
declare const validGradidoAmount: unique symbol
export type GradidoAmount = GradidoUnit & { [validGradidoAmount]: true }
export const gradidoAmountSchema = v.pipe(
v.union([legacyAmountSchema, v.instance(GradidoUnit, 'expect GradidoUnit type')]),
v.transform<LegacyAmount | GradidoUnit, GradidoAmount>((input: LegacyAmount | GradidoUnit) => {
if (input instanceof GradidoUnit) {
return input as GradidoAmount
}
// round floor with decimal js beforehand
const rounded = new Decimal(input).toDecimalPlaces(4, Decimal.ROUND_FLOOR).toString()
return GradidoUnit.fromString(rounded) as GradidoAmount
}),
)
*/
export const transactionBaseSchema = v.object({
id: positiveNumberSchema,
amount: gradidoAmountSchema,
memo: memoSchema,
user: userDbSchema,
})
export const transactionDbSchema = v.pipe(
v.object({
...transactionBaseSchema.entries,
typeId: v.enum(TransactionTypeId),
balanceDate: dateSchema,
linkedUser: userDbSchema,
}),
v.custom((value: any) => {
if (
value.user &&
value.linkedUser &&
!value.transactionLinkCode &&
value.user.gradidoId === value.linkedUser.gradidoId
) {
throw new Error(
`expect user to be different from linkedUser: ${JSON.stringify(value, null, 2)}`,
)
}
// check that user and linked user exist before transaction balance date
const balanceDate = new Date(value.balanceDate)
if (
value.user.createdAt.getTime() >= balanceDate.getTime() ||
value.linkedUser?.createdAt.getTime() >= balanceDate.getTime()
) {
throw new Error(
`at least one user was created after transaction balance date, logic error! ${JSON.stringify(value, null, 2)}`,
)
}
return value
}),
)
export const creationTransactionDbSchema = v.pipe(
v.object({
...transactionBaseSchema.entries,
contributionDate: dateSchema,
confirmedAt: dateSchema,
confirmedByUser: userDbSchema,
transactionId: positiveNumberSchema,
}),
v.custom((value: any) => {
if (
value.user &&
value.confirmedByUser &&
value.user.gradidoId === value.confirmedByUser.gradidoId
) {
throw new Error(
`expect user to be different from confirmedByUser: ${JSON.stringify(value, null, 2)}`,
)
}
// check that user and confirmedByUser exist before transaction balance date
const confirmedAt = new Date(value.confirmedAt)
if (
value.user.createdAt.getTime() >= confirmedAt.getTime() ||
value.confirmedByUser?.createdAt.getTime() >= confirmedAt.getTime()
) {
throw new Error(
`at least one user was created after transaction confirmedAt date, logic error! ${JSON.stringify(value, null, 2)}`,
)
}
return value
}),
)
export const transactionLinkDbSchema = v.object({
...transactionBaseSchema.entries,
code: identifierSeedSchema,
createdAt: dateSchema,
validUntil: dateSchema,
holdAvailableAmount: gradidoAmountSchema,
redeemedAt: v.nullish(dateSchema),
deletedAt: v.nullish(dateSchema),
})
export const redeemedTransactionLinkDbSchema = v.object({
...transactionLinkDbSchema.entries,
redeemedAt: dateSchema,
redeemedBy: userDbSchema,
})
export const deletedTransactionLinKDbSchema = v.object({
id: positiveNumberSchema,
user: userDbSchema,
code: identifierSeedSchema,
deletedAt: dateSchema,
})
export const communityDbSchema = v.object({
id: positiveNumberSchema,
foreign: booleanSchema,
communityUuid: uuidv4Schema,
name: v.string(),
creationDate: dateSchema,
userMinCreatedAt: v.nullish(dateSchema),
})
export const communityContextSchema = v.object({
communityId: v.string(),
foreign: booleanSchema,
blockchain: v.instance(InMemoryBlockchain, 'expect InMemoryBlockchain type'),
keyPair: v.instance(KeyPairEd25519),
folder: v.pipe(
v.string(),
v.minLength(1, 'expect string length >= 1'),
v.maxLength(512, 'expect string length <= 512'),
v.regex(/^[a-zA-Z0-9-_]+$/, 'expect string to be a valid (alphanumeric, _, -) folder name'),
),
gmwBalance: v.instance(Balance),
aufBalance: v.instance(Balance),
})
export type TransactionDb = v.InferOutput<typeof transactionDbSchema>
export type CreationTransactionDb = v.InferOutput<typeof creationTransactionDbSchema>
export type UserDb = v.InferOutput<typeof userDbSchema>
export type TransactionLinkDb = v.InferOutput<typeof transactionLinkDbSchema>
export type RedeemedTransactionLinkDb = v.InferOutput<typeof redeemedTransactionLinkDbSchema>
export type DeletedTransactionLinkDb = v.InferOutput<typeof deletedTransactionLinKDbSchema>
export type CommunityDb = v.InferOutput<typeof communityDbSchema>
export type CommunityContext = v.InferOutput<typeof communityContextSchema>

1
dlt-connector/src/schemas/account.schema.ts
View File

@ -11,6 +11,7 @@ export type IdentifierCommunityAccount = v.InferOutput<typeof identifierCommunit
export const identifierKeyPairSchema = v.object({
communityTopicId: hieroIdSchema,
communityId: uuidv4Schema,
account: v.optional(identifierCommunityAccountSchema),
seed: v.optional(identifierSeedSchema),
})

19
dlt-connector/src/schemas/transaction.schema.test.ts
View File

@ -34,8 +34,12 @@ const transactionLinkCode = (date: Date): string => {
}
let topic: HieroId
const topicString = '0.0.261'
let communityUuid: Uuidv4
const communityUuidString = 'fcd48487-6d31-4f4c-be9b-b3c8ca853912'
beforeAll(() => {
topic = v.parse(hieroIdSchema, topicString)
communityUuid = v.parse(uuidv4Schema, communityUuidString)
})
describe('transaction schemas', () => {
@ -55,6 +59,7 @@ describe('transaction schemas', () => {
registerAddress = {
user: {
communityTopicId: topicString,
communityId: communityUuidString,
account: { userUuid: userUuidString },
},
type: InputTransactionType.REGISTER_ADDRESS,
@ -66,6 +71,7 @@ describe('transaction schemas', () => {
expect(v.parse(transactionSchema, registerAddress)).toEqual({
user: {
communityTopicId: topic,
communityId: communityUuid,
account: {
userUuid,
accountNr: 0,
@ -80,6 +86,7 @@ describe('transaction schemas', () => {
expect(v.parse(registerAddressTransactionSchema, registerAddress)).toEqual({
user: {
communityTopicId: topic,
communityId: communityUuid,
account: {
userUuid,
accountNr: 0,
@ -101,10 +108,12 @@ describe('transaction schemas', () => {
const gradidoTransfer: TransactionInput = {
user: {
communityTopicId: topicString,
communityId: communityUuidString,
account: { userUuid: userUuidString },
},
linkedUser: {
communityTopicId: topicString,
communityId: communityUuidString,
account: { userUuid: userUuidString },
},
amount: '100',
@ -115,6 +124,7 @@ describe('transaction schemas', () => {
expect(v.parse(transactionSchema, gradidoTransfer)).toEqual({
user: {
communityTopicId: topic,
communityId: communityUuid,
account: {
userUuid,
accountNr: 0,
@ -122,6 +132,7 @@ describe('transaction schemas', () => {
},
linkedUser: {
communityTopicId: topic,
communityId: communityUuid,
account: {
userUuid,
accountNr: 0,
@ -138,10 +149,12 @@ describe('transaction schemas', () => {
const gradidoCreation: TransactionInput = {
user: {
communityTopicId: topicString,
communityId: communityUuidString,
account: { userUuid: userUuidString },
},
linkedUser: {
communityTopicId: topicString,
communityId: communityUuidString,
account: { userUuid: userUuidString },
},
amount: '1000',
@ -153,10 +166,12 @@ describe('transaction schemas', () => {
expect(v.parse(transactionSchema, gradidoCreation)).toEqual({
user: {
communityTopicId: topic,
communityId: communityUuid,
account: { userUuid, accountNr: 0 },
},
linkedUser: {
communityTopicId: topic,
communityId: communityUuid,
account: { userUuid, accountNr: 0 },
},
amount: v.parse(gradidoAmountSchema, gradidoCreation.amount!),
@ -172,12 +187,14 @@ describe('transaction schemas', () => {
const gradidoTransactionLink: TransactionInput = {
user: {
communityTopicId: topicString,
communityId: communityUuidString,
account: {
userUuid: userUuidString,
},
},
linkedUser: {
communityTopicId: topicString,
communityId: communityUuidString,
seed,
},
amount: '100',
@ -189,6 +206,7 @@ describe('transaction schemas', () => {
expect(v.parse(transactionSchema, gradidoTransactionLink)).toEqual({
user: {
communityTopicId: topic,
communityId: communityUuid,
account: {
userUuid,
accountNr: 0,
@ -196,6 +214,7 @@ describe('transaction schemas', () => {
},
linkedUser: {
communityTopicId: topic,
communityId: communityUuid,
seed: seedParsed,
},
amount: v.parse(gradidoAmountSchema, gradidoTransactionLink.amount!),

2
dlt-connector/src/schemas/transaction.schema.ts
View File

@ -43,12 +43,14 @@ export type Transaction = v.InferOutput<typeof transactionSchema>
// if the account is identified by seed
export const seedAccountSchema = v.object({
communityTopicId: hieroIdSchema,
communityId: uuidv4Schema,
seed: identifierSeedSchema,
})
// if the account is identified by userUuid and accountNr
export const userAccountSchema = v.object({
communityTopicId: hieroIdSchema,
communityId: uuidv4Schema,
account: identifierCommunityAccountSchema,
})

15
dlt-connector/src/schemas/typeConverter.schema.test.ts
View File

@ -2,7 +2,7 @@
import { describe, expect, it } from 'bun:test'
import { TypeCompiler } from '@sinclair/typebox/compiler'
import { Static, TypeBoxFromValibot } from '@sinclair/typemap'
import { AddressType_COMMUNITY_AUF } from 'gradido-blockchain-js'
import { AddressType_COMMUNITY_AUF, InMemoryBlockchainProvider } from 'gradido-blockchain-js'
import * as v from 'valibot'
import { AccountType } from '../data/AccountType.enum'
import {
@ -96,12 +96,15 @@ describe('basic.schema', () => {
})
it('confirmedTransactionSchema', () => {
const confirmedTransaction = v.parse(
confirmedTransactionSchema,
'CAcS5AEKZgpkCiCBZwMplGmI7fRR9MQkaR2Dz1qQQ5BCiC1btyJD71Ue9BJABODQ9sS70th9yHn8X3K+SNv2gsiIdX/V09baCvQCb+yEj2Dd/fzShIYqf3pooIMwJ01BkDJdNGBZs5MDzEAkChJ6ChkIAhIVRGFua2UgZnVlciBkZWluIFNlaW4hEggIgMy5/wUQABoDMy41IAAyTAooCiDbDtYSWhTwMKvtG/yDHgohjPn6v87n7NWBwMDniPAXxxCUmD0aABIgJE0o18xb6P6PsNjh0bkN52AzhggteTzoh09jV+blMq0aCAjC8rn/BRAAIgMzLjUqICiljeEjGHifWe4VNzoe+DN9oOLNZvJmv3VlkP+1RH7MMiAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADomCiDbDtYSWhTwMKvtG/yDHgohjPn6v87n7NWBwMDniPAXxxDAhD06JwogJE0o18xb6P6PsNjh0bkN52AzhggteTzoh09jV+blMq0Q65SlBA==',
)
// create blockchain in native module
const communityId = 'fcd48487-6d31-4f4c-be9b-b3c8ca853912'
InMemoryBlockchainProvider.getInstance().getBlockchain(communityId)
const confirmedTransaction = v.parse(confirmedTransactionSchema, {
base64:
'CAcS4AEKZgpkCiCBZwMplGmI7fRR9MQkaR2Dz1qQQ5BCiC1btyJD71Ue9BJABODQ9sS70th9yHn8X3K+SNv2gsiIdX/V09baCvQCb+zo7nEQgCUXOEe/tN7YaRppwt6TDcXBPxkwnw4gfpCODhJ0ChkIAhIVRGFua2UgZnVlciBkZWluIFNlaW4hEgYIgMy5/wUaAzMuNTJKCiYKINsO1hJaFPAwq+0b/IMeCiGM+fq/zufs1YHAwOeI8BfHEJSYPRIgJE0o18xb6P6PsNjh0bkN52AzhggteTzoh09jV+blMq0aABoGCMLyuf8FIgMzLjcqIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMhUIAhoRCgkIqemnUhD+4wESBBj8sgc6Jgog2w7WEloU8DCr7Rv8gx4KIYz5+r/O5+zVgcDA54jwF8cQwIQ9OicKICRNKNfMW+j+j7DY4dG5DedgM4YILXk86IdPY1fm5TKtEOuUpQRAAg==',
communityId,
})
expect(confirmedTransaction.getId()).toBe(7)
expect(confirmedTransaction.getConfirmedAt().getSeconds()).toBe(1609464130)
expect(confirmedTransaction.getVersionNumber()).toBe('3.5')
})
})

21
dlt-connector/src/schemas/typeConverter.schema.ts
View File

@ -8,6 +8,7 @@ import {
toAccountType,
toAddressType,
} from '../utils/typeConverter'
import { Uuidv4, uuidv4Schema } from './typeGuard.schema'
/**
* dateSchema for creating a date from string or Date object
@ -72,17 +73,23 @@ export const accountTypeSchema = v.pipe(
export const confirmedTransactionSchema = v.pipe(
v.union([
v.instance(ConfirmedTransaction, 'expect ConfirmedTransaction'),
v.pipe(
v.string('expect confirmed Transaction base64 as string type'),
v.base64('expect to be valid base64'),
),
v.object({
base64: v.pipe(
v.string('expect confirmed Transaction base64 as string type'),
v.base64('expect to be valid base64'),
),
communityId: uuidv4Schema,
}),
]),
v.transform<string | ConfirmedTransaction, ConfirmedTransaction>(
(data: string | ConfirmedTransaction) => {
v.transform<ConfirmedTransaction | { base64: string; communityId: Uuidv4 }, ConfirmedTransaction>(
(data: string | ConfirmedTransaction | { base64: string; communityId: Uuidv4 }) => {
if (data instanceof ConfirmedTransaction) {
return data
}
return confirmedTransactionFromBase64(data)
if (typeof data === 'object' && 'base64' in data && 'communityId' in data) {
return confirmedTransactionFromBase64(data.base64, data.communityId)
}
throw new Error("invalid data, community id missing, couldn't deserialize")
},
),
)

12
dlt-connector/src/schemas/typeGuard.schema.test.ts
View File

@ -1,7 +1,7 @@
import { describe, expect, it } from 'bun:test'
import { v4 as uuidv4 } from 'uuid'
import * as v from 'valibot'
import { memoSchema, uuidv4Schema } from './typeGuard.schema'
import { MEMO_MAX_CHARS, MEMO_MIN_CHARS, memoSchema, uuidv4Schema } from './typeGuard.schema'
describe('typeGuard.schema', () => {
describe('Uuidv4', () => {
@ -20,18 +20,20 @@ describe('typeGuard.schema', () => {
expect(memoValueParsed.toString()).toBe(memoValue)
})
it('max length', () => {
const memoValue = 's'.repeat(255)
const memoValue = 's'.repeat(MEMO_MAX_CHARS)
const memoValueParsed = v.parse(memoSchema, memoValue)
expect(memoValueParsed.toString()).toBe(memoValue)
})
it('to short', () => {
const memoValue = 'memo'
expect(() => v.parse(memoSchema, memoValue)).toThrow(new Error('expect string length >= 5'))
expect(() => v.parse(memoSchema, memoValue)).toThrow(
new Error(`expect string length >= ${MEMO_MIN_CHARS}`),
)
})
it('to long', () => {
const memoValue = 's'.repeat(256)
const memoValue = 's'.repeat(MEMO_MAX_CHARS + 1)
expect(() => v.parse(memoSchema, memoValue)).toThrow(
new Error('expect string length <= 255'),
new Error(`expect string length <= ${MEMO_MAX_CHARS}`),
)
})
})

2
dlt-connector/src/schemas/typeGuard.schema.ts
View File

@ -163,7 +163,7 @@ export type HieroTransactionIdInput = v.InferInput<typeof hieroTransactionIdStri
* memo string inside bounds [5, 255]
*/
export const MEMO_MIN_CHARS = 5
export const MEMO_MAX_CHARS = 255
export const MEMO_MAX_CHARS = 512
declare const validMemo: unique symbol
export type Memo = string & { [validMemo]: true }

11
dlt-connector/src/server/index.test.ts
View File

@ -1,6 +1,11 @@
import { beforeAll, describe, expect, it, mock } from 'bun:test'
import { AccountId, Timestamp, TransactionId } from '@hashgraph/sdk'
import { GradidoTransaction, KeyPairEd25519, MemoryBlock } from 'gradido-blockchain-js'
import {
GradidoTransaction,
InMemoryBlockchainProvider,
KeyPairEd25519,
MemoryBlock,
} from 'gradido-blockchain-js'
import * as v from 'valibot'
import { KeyPairCacheManager } from '../cache/KeyPairCacheManager'
import { HieroId, hieroIdSchema } from '../schemas/typeGuard.schema'
@ -55,9 +60,13 @@ beforeAll(() => {
describe('Server', () => {
it('send register address transaction', async () => {
// create blockchain in native module
const communityId = '1e88a0f4-d4fc-4cae-a7e8-a88e613ce324'
InMemoryBlockchainProvider.getInstance().getBlockchain(communityId)
const transaction = {
user: {
communityTopicId: '0.0.21732',
communityId,
account: {
userUuid,
accountNr: 0,

12
dlt-connector/src/server/index.ts
View File

@ -69,9 +69,10 @@ export const appRoutes = new Elysia()
// check if account exists by user, call example:
// GET /isAccountExist/by-user/0.0.21732/408780b2-59b3-402a-94be-56a4f4f4e8ec/0
.get(
'/isAccountExist/by-user/:communityTopicId/:userUuid/:accountNr',
async ({ params: { communityTopicId, userUuid, accountNr } }) => ({
'/isAccountExist/by-user/:communityId/:communityTopicId/:userUuid/:accountNr',
async ({ params: { communityId, communityTopicId, userUuid, accountNr } }) => ({
exists: await isAccountExist({
communityId,
communityTopicId,
account: { userUuid, accountNr },
}),
@ -84,9 +85,10 @@ export const appRoutes = new Elysia()
// check if account exists by seed, call example:
// GET /isAccountExist/by-seed/0.0.21732/0c4676adfd96519a0551596c
.get(
'/isAccountExist/by-seed/:communityTopicId/:seed',
async ({ params: { communityTopicId, seed } }) => ({
'/isAccountExist/by-seed/:communityId/:communityTopicId/:seed',
async ({ params: { communityId, communityTopicId, seed } }) => ({
exists: await isAccountExist({
communityId,
communityTopicId,
seed,
}),
@ -145,7 +147,7 @@ async function isAccountExist(identifierAccount: IdentifierAccountInput): Promis
// ask gradido node server for account type, if type !== NONE account exist
const addressType = await GradidoNodeClient.getInstance().getAddressType(
publicKey.convertToHex(),
identifierAccountParsed.communityTopicId,
identifierAccountParsed.communityId,
)
const exists = addressType !== AddressType_NONE
const endTime = Date.now()

2
dlt-connector/src/server/input.schema.ts
View File

@ -3,6 +3,7 @@ import { t } from 'elysia'
import { hieroIdSchema, uuidv4Schema } from '../schemas/typeGuard.schema'
export const accountIdentifierUserTypeBoxSchema = t.Object({
communityId: TypeBoxFromValibot(uuidv4Schema),
communityTopicId: TypeBoxFromValibot(hieroIdSchema),
userUuid: TypeBoxFromValibot(uuidv4Schema),
accountNr: t.Number({ min: 0 }),
@ -10,6 +11,7 @@ export const accountIdentifierUserTypeBoxSchema = t.Object({
// identifier for a gradido account created by transaction link / deferred transfer
export const accountIdentifierSeedTypeBoxSchema = t.Object({
communityId: TypeBoxFromValibot(uuidv4Schema),
communityTopicId: TypeBoxFromValibot(hieroIdSchema),
seed: TypeBoxFromValibot(uuidv4Schema),
})

6
dlt-connector/src/utils/filesystem.ts
View File

@ -9,7 +9,7 @@ export function checkFileExist(filePath: string): boolean {
fs.accessSync(filePath, fs.constants.R_OK | fs.constants.W_OK)
return true
} catch (_err) {
// logger.debug(`file ${filePath} does not exist: ${_err}`)
logger.debug(`file ${filePath} does not exist: ${_err}`)
return false
}
}
@ -28,3 +28,7 @@ export function checkPathExist(path: string, createIfMissing: boolean = false):
}
return false
}
export function toFolderName(name: string): string {
return name.toLowerCase().replace(/[^a-z0-9]/g, '_')
}

2
dlt-connector/src/utils/network.ts
View File

@ -29,7 +29,7 @@ export async function isPortOpen(
socket.destroy()
const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.network.isPortOpen`)
logger.addContext('url', url)
logger.error(`${err.message}: ${err.code}`)
logger.debug(`${err.message}: ${err.code}`)
resolve(false)
})
})

8
dlt-connector/src/utils/typeConverter.ts
View File

@ -6,14 +6,18 @@ import {
} from 'gradido-blockchain-js'
import { AccountType } from '../data/AccountType.enum'
import { AddressType } from '../data/AddressType.enum'
import { Uuidv4 } from '../schemas/typeGuard.schema'
export const confirmedTransactionFromBase64 = (base64: string): ConfirmedTransaction => {
export const confirmedTransactionFromBase64 = (
base64: string,
communityId: Uuidv4,
): ConfirmedTransaction => {
const confirmedTransactionBinaryPtr = MemoryBlock.createPtr(MemoryBlock.fromBase64(base64))
const deserializer = new InteractionDeserialize(
confirmedTransactionBinaryPtr,
DeserializeType_CONFIRMED_TRANSACTION,
)
deserializer.run()
deserializer.run(communityId)
const confirmedTransaction = deserializer.getConfirmedTransaction()
if (!confirmedTransaction) {
throw new Error("invalid data, couldn't deserialize")

2
inspector

@ -1 +1 @@
Subproject commit 0c14b7eea29b8911cbe3cb303f5b0b61ce9bf6f4
Subproject commit e1d13e3336199eae615557d11d6671c034860326

2
package.json
View File

@ -34,7 +34,7 @@
"auto-changelog": "^2.4.0",
"cross-env": "^7.0.3",
"jose": "^4.14.4",
"turbo": "^2.5.0",
"turbo": "^2.8.12",
"uuid": "^8.3.2"
},
"devDependencies": {

3
shared/src/const/index.ts
View File

@ -1,4 +1,7 @@
import Decimal from 'decimal.js-light'
export const DECAY_START_TIME = new Date('2021-05-13T17:46:31Z')
export const DECAY_FACTOR = new Decimal('0.99999997803504048973201202316767079413460520837376')
export const SECONDS_PER_YEAR_GREGORIAN_CALENDER = 31556952.0
export const LOG4JS_BASE_CATEGORY_NAME = 'shared'
export const REDEEM_JWT_TOKEN_EXPIRATION = '10m'

15
shared/src/logic/decay.ts
View File

@ -1,6 +1,6 @@
import { Decimal } from 'decimal.js-light'
import { DECAY_START_TIME, SECONDS_PER_YEAR_GREGORIAN_CALENDER } from '../const'
import { DECAY_FACTOR, DECAY_START_TIME, SECONDS_PER_YEAR_GREGORIAN_CALENDER } from '../const'
Decimal.set({
precision: 25,
@ -16,21 +16,28 @@ export interface Decay {
duration: number | null
}
// legacy decay formula
export function decayFormula(value: Decimal, seconds: number): Decimal {
// TODO why do we need to convert this here to a string to work properly?
// chatgpt: We convert to string here to avoid precision loss:
// .pow(seconds) can internally round the result, especially for large values of `seconds`.
// Using .toString() ensures full precision is preserved in the multiplication.
return value.mul(
new Decimal('0.99999997803504048973201202316767079413460520837376').pow(seconds).toString(),
)
return value.mul(DECAY_FACTOR.pow(seconds).toString())
}
// legacy reverse decay formula
export function reverseLegacyDecay(result: Decimal, seconds: number): Decimal {
return result.div(DECAY_FACTOR.pow(seconds).toString())
}
// fast and more correct decay formula
export function decayFormulaFast(value: Decimal, seconds: number): Decimal {
return value.mul(
new Decimal(2).pow(new Decimal(-seconds).div(new Decimal(SECONDS_PER_YEAR_GREGORIAN_CALENDER))),
)
}
// compound interest formula, the reverse decay formula for decayFormulaFast
export function compoundInterest(value: Decimal, seconds: number): Decimal {
return value.mul(
new Decimal(2).pow(new Decimal(seconds).div(new Decimal(SECONDS_PER_YEAR_GREGORIAN_CALENDER))),