diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/blockchain.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/blockchain.ts index 22fa9b400..4154b0ba9 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/blockchain.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/blockchain.ts @@ -32,6 +32,7 @@ import { CommunityDb, loadCommunities, TransactionDb, TransactionTypeId, UserDb import { LOG4JS_BASE_CATEGORY } from '../../config/const' import { Community, Transaction } from '../../schemas/transaction.schema' import { communityDbToCommunity, userDbToTransaction } from './convert' +import { TransferTransactionRole } from '../../interactions/sendToHiero/TransferTransaction.role' const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`) export const defaultHieroAccount = new HieroAccountId(0, 0, 2) @@ -60,39 +61,27 @@ export async function addRegisterAddressTransaction(blockchain: InMemoryBlockcha throw new Error(`Register Address Transaction not added for user ${transaction.user.account!.userUuid}`) } } -/* - -export async function addTransaction(blockchain: InMemoryBlockchain, transactionDb: TransactionDb): Promise { - - let transactionRole: AbstractTransactionRole - switch (transactionDb.typeId) { - case TransactionTypeId.CREATION: - transactionRole = new CreationTransactionRole({ - user: { - communityTopicId: transactionDb.user.communityTopicId, - account: { - userUuid: transactionDb.user.gradidoId, - accountNr: 0, - }, - }, - linkedUser: { - communityTopicId: transactionDb.linkedUser.communityTopicId, - account: { - userUuid: transactionDb.linkedUser.gradidoId, - accountNr: 0, - }, - }, - amount: v.parse(gradidoAmountSchema, transactionDb.amount), - memo: v.parse(memoSchema, transactionDb.memo), - createdAt: transactionDb.creationDate, - targetDate: transactionDb.balanceDate, - type: getInputTransactionTypeFromTypeId(transactionDb.typeId), - }) - if(addToBlockchain(await transactionRole.getGradidoTransactionBuilder(), blockchain, new Timestamp(transactionDb.creationDate))) { - logger.info(`Transaction added for user ${transactionDb.user.gradidoId}`) - } else { - throw new Error(`Transaction not added for user ${transactionDb.user.gradidoId}`) +export async function addTransaction( + senderBlockchain: InMemoryBlockchain, + _recipientBlockchain: InMemoryBlockchain, + transaction: Transaction +): Promise { + const createdAtTimestamp = new Timestamp(transaction.createdAt) + if (transaction.type === InputTransactionType.GRADIDO_CREATION) { + const creationTransactionRole = new CreationTransactionRole(transaction) + if(addToBlockchain(await creationTransactionRole.getGradidoTransactionBuilder(), senderBlockchain, createdAtTimestamp)) { + logger.info(`Creation Transaction added for user ${transaction.user.account!.userUuid}`) + } else { + throw new Error(`Creation Transaction not added for user ${transaction.user.account!.userUuid}`) + } + } else if (transaction.type === InputTransactionType.GRADIDO_TRANSFER) { + const transferTransactionRole = new TransferTransactionRole(transaction) + // will crash with cross group transaction + if(addToBlockchain(await transferTransactionRole.getGradidoTransactionBuilder(), senderBlockchain, createdAtTimestamp)) { + logger.info(`Transfer Transaction added for user ${transaction.user.account!.userUuid}`) + } else { + throw new Error(`Transfer Transaction not added for user ${transaction.user.account!.userUuid}`) + } } } -*/ \ No newline at end of file diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/convert.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/convert.ts index f182664bc..e8052df88 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/convert.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/convert.ts @@ -39,7 +39,11 @@ export function userDbToTransaction(userDb: CreatedUserDb, communityTopicId: Hie }) } -export function transactionDbToTransaction(transactionDb: TransactionDb, communityTopicId: HieroId, recipientCommunityTopicId: HieroId): Transaction { +export function transactionDbToTransaction( + transactionDb: TransactionDb, + communityTopicId: HieroId, + recipientCommunityTopicId: HieroId +): Transaction { if ( transactionDb.typeId !== TransactionTypeId.CREATION && transactionDb.typeId !== TransactionTypeId.SEND @@ -48,11 +52,11 @@ export function transactionDbToTransaction(transactionDb: TransactionDb, communi } const user = { communityTopicId: communityTopicId, - account: { userUuid: transactionDb.user.gradidoId }, + account: { userUuid: transactionDb.user.gradidoId, accountNr: 0 }, } const linkedUser = { communityTopicId: recipientCommunityTopicId, - account: { userUuid: transactionDb.linkedUser.gradidoId }, + account: { userUuid: transactionDb.linkedUser.gradidoId, accountNr: 0 }, } const transaction: TransactionInput = { user, diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/database.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/database.ts index 71561e33d..ce990a76f 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/database.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/database.ts @@ -1,5 +1,5 @@ import { SQL } from 'bun' -import { amountSchema, memoSchema, uuidv4Schema, identifierSeedSchema } from '../../schemas/typeGuard.schema' +import { amountSchema, memoSchema, uuidv4Schema, identifierSeedSchema, gradidoAmountSchema } from '../../schemas/typeGuard.schema' import { dateSchema, booleanSchema } from '../../schemas/typeConverter.schema' import * as v from 'valibot' import { GradidoUnit } from 'gradido-blockchain-js' @@ -26,7 +26,7 @@ export enum TransactionTypeId { export const transactionDbSchema = v.object({ typeId: v.enum(TransactionTypeId), - amount: amountSchema, + amount: gradidoAmountSchema, balanceDate: dateSchema, memo: memoSchema, creationDate: v.nullish(dateSchema), @@ -38,7 +38,7 @@ export const transactionDbSchema = v.object({ export const transactionLinkDbSchema = v.object({ userUuid: uuidv4Schema, code: identifierSeedSchema, - amount: amountSchema, + amount: gradidoAmountSchema, memo: memoSchema, createdAt: dateSchema, validUntil: dateSchema, @@ -85,7 +85,7 @@ export async function loadCommunities(db: SQL): Promise { export async function loadUsers(db: SQL, offset: number, count: number): Promise { const result = await db` - SELECT id, gradido_id as gradidoId, community_uuid as communityUuid, created_at as createdAt FROM users + SELECT gradido_id as gradidoId, community_uuid as communityUuid, created_at as createdAt FROM users ORDER by created_at ASC LIMIT ${offset}, ${count} ` @@ -100,38 +100,47 @@ export async function loadUsers(db: SQL, offset: number, count: number): Promise export async function loadTransactions(db: SQL, offset: number, count: number): Promise { const result = await db` - SELECT type_id, amount, balance_date, memo, creation_date, + SELECT t.type_id, t.amount, t.balance_date, t.memo, t.creation_date, u.gradido_id AS user_gradido_id, u.community_uuid AS user_community_uuid, lu.gradido_id AS linked_user_gradido_id, lu.community_uuid AS linked_user_community_uuid, tl.code as transaction_link_code - FROM transactions - LEFT JOIN users u ON transactions.user_id = u.id - LEFT JOIN users lu ON transactions.linked_user_id = lu.id - LEFT JOIN transaction_links tl ON transactions.transaction_link_id = tl.id + FROM transactions as t + LEFT JOIN users u ON t.user_id = u.id + LEFT JOIN users lu ON t.linked_user_id = lu.id + LEFT JOIN transaction_links tl ON t.transaction_link_id = tl.id ORDER by balance_date ASC LIMIT ${offset}, ${count} ` return result.map((row: any) => { + // console.log(row) let amount = GradidoUnit.fromString(row.amount) if (row.type_id === TransactionTypeId.SEND) { amount = amount.mul(new GradidoUnit(-1)) } - return v.parse(transactionDbSchema, { - typeId: row.type_id, - amount, - balanceDate: new Date(row.balance_date), - memo: row.memo, - creationDate: new Date(row.creation_date), - transactionLinkCode: row.transaction_link_code, - user: { - gradidoId: row.user_gradido_id, - communityUuid: row.user_community_uuid - }, - linkedUser: { - gradidoId: row.linked_user_gradido_id, - communityUuid: row.linked_user_community_uuid + try { + return v.parse(transactionDbSchema, { + typeId: row.type_id, + amount, + balanceDate: new Date(row.balance_date), + memo: row.memo, + creationDate: new Date(row.creation_date), + transactionLinkCode: row.transaction_link_code, + user: { + gradidoId: row.user_gradido_id, + communityUuid: row.user_community_uuid + }, + linkedUser: { + gradidoId: row.linked_user_gradido_id, + communityUuid: row.linked_user_community_uuid + } + }) + } catch (e) { + if (e instanceof v.ValiError) { + console.error(v.flatten(e.issues)) + } else { + throw e } - }) + } }) } diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/index.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/index.ts index 201526fbb..2aaa02366 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/index.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/index.ts @@ -9,7 +9,7 @@ import { import { Logger } from 'log4js' import { CreatedUserDb, loadDeletedTransactionLinks, loadTransactionLinks, loadTransactions, loadUsers, TransactionDb, TransactionLinkDb } from './database' -import { addRegisterAddressTransaction, defaultHieroAccount } from './blockchain' +import { addRegisterAddressTransaction, addTransaction, defaultHieroAccount } from './blockchain' import { generateKeyPairUserAccount } from './keyPair' import { transactionDbToTransaction, userDbToTransaction } from './convert' import { Orderable, OrderedContainer } from './OrderedContainer' @@ -17,20 +17,35 @@ import { Context } from './Context' import { bootstrap } from './bootstrap' import { RegisterAddressTransactionRole } from '../../interactions/sendToHiero/RegisterAddressTransaction.role' +const publicKeyUserIdMap = new Map() + async function main() { // prepare in memory blockchains const context = await bootstrap() + const startTime = Date.now() + await getNextUsers(context, 0, 110) + const endTime = Date.now() + console.log(`getNextUsers took ${endTime - startTime} ms`) + // synchronize to blockchain const BATCH_SIZE = 10 const users = new OrderedContainer( - getNextUsers, + (context: Context, offset: number, count: number) => loadUsers(context.db, offset, count), (user: CreatedUserDb) => user.createdAt, (context: Context, user: CreatedUserDb) => pushRegisterAddressTransaction(context, user) ) + const transactions = new OrderedContainer( + getNextTransactions, + (transaction: TransactionDb) => transaction.balanceDate, + (context: Context, transaction: TransactionDb) => pushTransaction(context, transaction) + ) - await synchronizeToBlockchain(context, [users], BATCH_SIZE) + // const transactionLinks = new OrderedContainer(getNextTransactionLinks, (transactionLink) => transactionLink.createdAt) + // const deletedTransactionLinks = new OrderedContainer(getNextDeletedTransactionLinks, (transactionLink) => transactionLink.balanceDate) + + await synchronizeToBlockchain(context, [users, transactions], BATCH_SIZE) // log blockchains for(const community of context.communities.values()) { @@ -60,7 +75,16 @@ async function synchronizeToBlockchain( // console.log(`smallest date: ${available[0].getDate()}`) if(rounds >= 0) { - await available[0].pushToBlockchain(context) + try { + await available[0].pushToBlockchain(context) + } catch (e) { + console.error(e) + logBlogchain(context.logger, context.communities.values().next().value!.blockchain) + // for(const [key, value] of publicKeyUserIdMap.entries()) { + // console.log(`${key}: ${value}`) + // } + throw e + } } else { const user = (available[0] as any as OrderedContainer).shift() console.log(JSON.stringify(user, null, 2)) @@ -91,28 +115,16 @@ async function synchronizeToBlockchain( } } -async function fillBlockchains(context: Context): Promise { - const BATCH_SIZE = 10 - - const users = new OrderedContainer( - getNextUsers, - (user: CreatedUserDb) => user.createdAt, - (context: Context, user: CreatedUserDb) => pushRegisterAddressTransaction(context, user) - ) - /*const transactions = new OrderedContainer(getNextTransactions, (transaction) => transaction.balanceDate) - const transactionLinks = new OrderedContainer(getNextTransactionLinks, (transactionLink) => transactionLink.createdAt) - const deletedTransactionLinks = new OrderedContainer(getNextDeletedTransactionLinks, (transactionLink) => transactionLink.balanceDate) -*/ - await synchronizeToBlockchain(context, [users], BATCH_SIZE) -} - // ---------------- load from db graiddo backend transactions format ----------------------------------------------- + /// load next max ${count} users and calculate key pair for calculating signatures later async function getNextUsers(context: Context, offset: number, count: number): Promise { const users = await loadUsers(context.db, offset, count) for (const user of users) { const communityContext = context.getCommunityContextByUuid(user.communityUuid) - generateKeyPairUserAccount(user, context.cache, communityContext.topicId) + const { userKeyPair, accountKeyPair } = await generateKeyPairUserAccount(user, context.cache, communityContext.topicId) + publicKeyUserIdMap.set(userKeyPair.convertToHex(), user.gradidoId) + publicKeyUserIdMap.set(accountKeyPair.convertToHex(), user.gradidoId) } return users } @@ -140,16 +152,18 @@ async function pushRegisterAddressTransaction(context: Context, user: CreatedUse return await addRegisterAddressTransaction(communityContext.blockchain, transaction) } -/* + async function pushTransaction(context: Context, transactionDb: TransactionDb): Promise { const senderCommunityContext = context.getCommunityContextByUuid(transactionDb.user.communityUuid) const recipientCommunityContext = context.getCommunityContextByUuid(transactionDb.linkedUser.communityUuid) // CreationTransactionRole will check that community topic id belongs to home community context.cache.setHomeCommunityTopicId(senderCommunityContext.topicId) const transaction = transactionDbToTransaction(transactionDb, senderCommunityContext.topicId, recipientCommunityContext.topicId) - await addTransaction(senderCommunityContext.blockchain, transaction) + await addTransaction(senderCommunityContext.blockchain, recipientCommunityContext.blockchain, transaction) } -*/ + +// ---------------- utils ---------------------------------------------------------------------- + function logBlogchain(logger: Logger, blockchain: InMemoryBlockchain) { const f = new Filter() f.pagination.size = 0 diff --git a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/keyPair.ts b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/keyPair.ts index 67337a656..d5d471210 100644 --- a/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/keyPair.ts +++ b/dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/keyPair.ts @@ -1,7 +1,7 @@ import { CommunityDb, UserDb } from './database' import { KeyPairCacheManager } from '../../cache/KeyPairCacheManager' import { KeyPairIdentifierLogic } from '../../data/KeyPairIdentifier.logic' -import { KeyPairEd25519, MemoryBlock } from 'gradido-blockchain-js' +import { KeyPairEd25519, MemoryBlock, MemoryBlockPtr } from 'gradido-blockchain-js' import { getLogger } from 'log4js' import { LOG4JS_BASE_CATEGORY } from '../../config/const' import { CONFIG } from '../../config' @@ -31,19 +31,23 @@ export function generateKeyPairCommunity(community: CommunityDb, cache: KeyPairC logger.info(`Community Key Pair added with key: ${communityKeyPairKey}`) } -export function generateKeyPairUserAccount(user: UserDb, cache: KeyPairCacheManager, communityTopicId: HieroId): void { +export async function generateKeyPairUserAccount( + user: UserDb, + cache: KeyPairCacheManager, + communityTopicId: HieroId +): Promise<{userKeyPair: MemoryBlockPtr, accountKeyPair: MemoryBlockPtr}> { const communityKeyPair = cache.findKeyPair(communityTopicId)! - const userKeyPair = new UserKeyPairRole(user.gradidoId, communityKeyPair).generateKeyPair() + const userKeyPairRole = new UserKeyPairRole(user.gradidoId, communityKeyPair) const userKeyPairKey = new KeyPairIdentifierLogic({ communityTopicId: communityTopicId, account: { userUuid: user.gradidoId, accountNr: 0 } - }).getKey() - cache.addKeyPair(userKeyPairKey, userKeyPair) + }).getKey() + const userKeyPair = await cache.getKeyPair(userKeyPairKey, () => Promise.resolve(userKeyPairRole.generateKeyPair())) - const accountKeyPair = new AccountKeyPairRole(1, userKeyPair).generateKeyPair() + const accountKeyPairRole = new AccountKeyPairRole(1, userKeyPair) const accountKeyPairKey = new KeyPairIdentifierLogic({ communityTopicId: communityTopicId, account: { @@ -51,6 +55,10 @@ export function generateKeyPairUserAccount(user: UserDb, cache: KeyPairCacheMana accountNr: 1 } }).getKey() - cache.addKeyPair(accountKeyPairKey, accountKeyPair) + const accountKeyPair = await cache.getKeyPair(accountKeyPairKey, () => Promise.resolve(accountKeyPairRole.generateKeyPair())) logger.info(`Key Pairs for user and account added, user: ${userKeyPairKey}, account: ${accountKeyPairKey}`) + return { + userKeyPair: userKeyPair.getPublicKey()!, + accountKeyPair: accountKeyPair.getPublicKey()! + } } \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d2d88cc9c..c8e23557b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,6 +71,8 @@ services: context: ./inspector dockerfile: Dockerfile target: production + profiles: + - dlt networks: - internal-net ports: diff --git a/nginx/gradido.conf b/nginx/gradido.conf index bbfd8db51..edaf12d22 100644 --- a/nginx/gradido.conf +++ b/nginx/gradido.conf @@ -82,4 +82,30 @@ server { proxy_redirect off; } + # Inspector + location /inspector { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + + proxy_pass http://inspector:3100; + proxy_redirect off; + } + + # Gradido Node + location /dlt { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + + proxy_pass http://dlt-connector:8340/api; + proxy_redirect off; + } + }