IT4Change/gradido
3
0
Fork 0
mirror of https://github.com/IT4Change/gradido.git synced 2026-02-06 09:56:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

refine migration code structure

Browse Source
This commit is contained in:
einhornimmond 2025-10-31 10:59:09 +01:00
parent f54097e6ba
commit 74bc448b96
7 changed files with 142 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/blockchain.ts
View File

@ -32,6 +32,7 @@ import { CommunityDb, loadCommunities, TransactionDb, TransactionTypeId, UserDb
import { LOG4JS_BASE_CATEGORY } from '../../config/const'
import { Community, Transaction } from '../../schemas/transaction.schema'
import { communityDbToCommunity, userDbToTransaction } from './convert'
import { TransferTransactionRole } from '../../interactions/sendToHiero/TransferTransaction.role'
const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`)
export const defaultHieroAccount = new HieroAccountId(0, 0, 2)
@ -60,39 +61,27 @@ export async function addRegisterAddressTransaction(blockchain: InMemoryBlockcha
throw new Error(`Register Address Transaction not added for user ${transaction.user.account!.userUuid}`)
}
}
/*
export async function addTransaction(blockchain: InMemoryBlockchain, transactionDb: TransactionDb): Promise<void> {
let transactionRole: AbstractTransactionRole
switch (transactionDb.typeId) {
case TransactionTypeId.CREATION:
transactionRole = new CreationTransactionRole({
user: {
communityTopicId: transactionDb.user.communityTopicId,
account: {
userUuid: transactionDb.user.gradidoId,
accountNr: 0,
},
},
linkedUser: {
communityTopicId: transactionDb.linkedUser.communityTopicId,
account: {
userUuid: transactionDb.linkedUser.gradidoId,
accountNr: 0,
},
},
amount: v.parse(gradidoAmountSchema, transactionDb.amount),
memo: v.parse(memoSchema, transactionDb.memo),
createdAt: transactionDb.creationDate,
targetDate: transactionDb.balanceDate,
type: getInputTransactionTypeFromTypeId(transactionDb.typeId),
})
if(addToBlockchain(await transactionRole.getGradidoTransactionBuilder(), blockchain, new Timestamp(transactionDb.creationDate))) {
logger.info(`Transaction added for user ${transactionDb.user.gradidoId}`)
} else {
throw new Error(`Transaction not added for user ${transactionDb.user.gradidoId}`)
export async function addTransaction(
senderBlockchain: InMemoryBlockchain,
_recipientBlockchain: InMemoryBlockchain,
transaction: Transaction
): Promise<void> {
const createdAtTimestamp = new Timestamp(transaction.createdAt)
if (transaction.type === InputTransactionType.GRADIDO_CREATION) {
const creationTransactionRole = new CreationTransactionRole(transaction)
if(addToBlockchain(await creationTransactionRole.getGradidoTransactionBuilder(), senderBlockchain, createdAtTimestamp)) {
logger.info(`Creation Transaction added for user ${transaction.user.account!.userUuid}`)
} else {
throw new Error(`Creation Transaction not added for user ${transaction.user.account!.userUuid}`)
}
} else if (transaction.type === InputTransactionType.GRADIDO_TRANSFER) {
const transferTransactionRole = new TransferTransactionRole(transaction)
// will crash with cross group transaction
if(addToBlockchain(await transferTransactionRole.getGradidoTransactionBuilder(), senderBlockchain, createdAtTimestamp)) {
logger.info(`Transfer Transaction added for user ${transaction.user.account!.userUuid}`)
} else {
throw new Error(`Transfer Transaction not added for user ${transaction.user.account!.userUuid}`)
}
}
}
*/

10
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/convert.ts
View File

@ -39,7 +39,11 @@ export function userDbToTransaction(userDb: CreatedUserDb, communityTopicId: Hie
})
}
export function transactionDbToTransaction(transactionDb: TransactionDb, communityTopicId: HieroId, recipientCommunityTopicId: HieroId): Transaction {
export function transactionDbToTransaction(
transactionDb: TransactionDb,
communityTopicId: HieroId,
recipientCommunityTopicId: HieroId
): Transaction {
if (
transactionDb.typeId !== TransactionTypeId.CREATION
&& transactionDb.typeId !== TransactionTypeId.SEND
@ -48,11 +52,11 @@ export function transactionDbToTransaction(transactionDb: TransactionDb, communi
}
const user = {
communityTopicId: communityTopicId,
account: { userUuid: transactionDb.user.gradidoId },
account: { userUuid: transactionDb.user.gradidoId, accountNr: 0 },
}
const linkedUser = {
communityTopicId: recipientCommunityTopicId,
account: { userUuid: transactionDb.linkedUser.gradidoId },
account: { userUuid: transactionDb.linkedUser.gradidoId, accountNr: 0 },
}
const transaction: TransactionInput = {
user,

57
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/database.ts
View File

@ -1,5 +1,5 @@
import { SQL } from 'bun'
import { amountSchema, memoSchema, uuidv4Schema, identifierSeedSchema } from '../../schemas/typeGuard.schema'
import { amountSchema, memoSchema, uuidv4Schema, identifierSeedSchema, gradidoAmountSchema } from '../../schemas/typeGuard.schema'
import { dateSchema, booleanSchema } from '../../schemas/typeConverter.schema'
import * as v from 'valibot'
import { GradidoUnit } from 'gradido-blockchain-js'
@ -26,7 +26,7 @@ export enum TransactionTypeId {
export const transactionDbSchema = v.object({
typeId: v.enum(TransactionTypeId),
amount: amountSchema,
amount: gradidoAmountSchema,
balanceDate: dateSchema,
memo: memoSchema,
creationDate: v.nullish(dateSchema),
@ -38,7 +38,7 @@ export const transactionDbSchema = v.object({
export const transactionLinkDbSchema = v.object({
userUuid: uuidv4Schema,
code: identifierSeedSchema,
amount: amountSchema,
amount: gradidoAmountSchema,
memo: memoSchema,
createdAt: dateSchema,
validUntil: dateSchema,
@ -85,7 +85,7 @@ export async function loadCommunities(db: SQL): Promise<CommunityDb[]> {
export async function loadUsers(db: SQL, offset: number, count: number): Promise<CreatedUserDb[]> {
const result = await db`
SELECT id, gradido_id as gradidoId, community_uuid as communityUuid, created_at as createdAt FROM users
SELECT gradido_id as gradidoId, community_uuid as communityUuid, created_at as createdAt FROM users
ORDER by created_at ASC
LIMIT ${offset}, ${count}
`
@ -100,38 +100,47 @@ export async function loadUsers(db: SQL, offset: number, count: number): Promise
export async function loadTransactions(db: SQL, offset: number, count: number): Promise<TransactionDb[]> {
const result = await db`
SELECT type_id, amount, balance_date, memo, creation_date,
SELECT t.type_id, t.amount, t.balance_date, t.memo, t.creation_date,
u.gradido_id AS user_gradido_id, u.community_uuid AS user_community_uuid,
lu.gradido_id AS linked_user_gradido_id, lu.community_uuid AS linked_user_community_uuid,
tl.code as transaction_link_code
FROM transactions
LEFT JOIN users u ON transactions.user_id = u.id
LEFT JOIN users lu ON transactions.linked_user_id = lu.id
LEFT JOIN transaction_links tl ON transactions.transaction_link_id = tl.id
FROM transactions as t
LEFT JOIN users u ON t.user_id = u.id
LEFT JOIN users lu ON t.linked_user_id = lu.id
LEFT JOIN transaction_links tl ON t.transaction_link_id = tl.id
ORDER by balance_date ASC
LIMIT ${offset}, ${count}
`
return result.map((row: any) => {
// console.log(row)
let amount = GradidoUnit.fromString(row.amount)
if (row.type_id === TransactionTypeId.SEND) {
amount = amount.mul(new GradidoUnit(-1))
}
return v.parse(transactionDbSchema, {
typeId: row.type_id,
amount,
balanceDate: new Date(row.balance_date),
memo: row.memo,
creationDate: new Date(row.creation_date),
transactionLinkCode: row.transaction_link_code,
user: {
gradidoId: row.user_gradido_id,
communityUuid: row.user_community_uuid
},
linkedUser: {
gradidoId: row.linked_user_gradido_id,
communityUuid: row.linked_user_community_uuid
try {
return v.parse(transactionDbSchema, {
typeId: row.type_id,
amount,
balanceDate: new Date(row.balance_date),
memo: row.memo,
creationDate: new Date(row.creation_date),
transactionLinkCode: row.transaction_link_code,
user: {
gradidoId: row.user_gradido_id,
communityUuid: row.user_community_uuid
},
linkedUser: {
gradidoId: row.linked_user_gradido_id,
communityUuid: row.linked_user_community_uuid
}
})
} catch (e) {
if (e instanceof v.ValiError) {
console.error(v.flatten(e.issues))
} else {
throw e
}
})
}
})
}

60
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/index.ts
View File

@ -9,7 +9,7 @@ import {
import { Logger } from 'log4js'
import { CreatedUserDb, loadDeletedTransactionLinks, loadTransactionLinks, loadTransactions, loadUsers, TransactionDb, TransactionLinkDb } from './database'
import { addRegisterAddressTransaction, defaultHieroAccount } from './blockchain'
import { addRegisterAddressTransaction, addTransaction, defaultHieroAccount } from './blockchain'
import { generateKeyPairUserAccount } from './keyPair'
import { transactionDbToTransaction, userDbToTransaction } from './convert'
import { Orderable, OrderedContainer } from './OrderedContainer'
@ -17,20 +17,35 @@ import { Context } from './Context'
import { bootstrap } from './bootstrap'
import { RegisterAddressTransactionRole } from '../../interactions/sendToHiero/RegisterAddressTransaction.role'
const publicKeyUserIdMap = new Map<string, string>()
async function main() {
// prepare in memory blockchains
const context = await bootstrap()
const startTime = Date.now()
await getNextUsers(context, 0, 110)
const endTime = Date.now()
console.log(`getNextUsers took ${endTime - startTime} ms`)
// synchronize to blockchain
const BATCH_SIZE = 10
const users = new OrderedContainer(
getNextUsers,
(context: Context, offset: number, count: number) => loadUsers(context.db, offset, count),
(user: CreatedUserDb) => user.createdAt,
(context: Context, user: CreatedUserDb) => pushRegisterAddressTransaction(context, user)
)
const transactions = new OrderedContainer(
getNextTransactions,
(transaction: TransactionDb) => transaction.balanceDate,
(context: Context, transaction: TransactionDb) => pushTransaction(context, transaction)
)
await synchronizeToBlockchain(context, [users], BATCH_SIZE)
// const transactionLinks = new OrderedContainer(getNextTransactionLinks, (transactionLink) => transactionLink.createdAt)
// const deletedTransactionLinks = new OrderedContainer(getNextDeletedTransactionLinks, (transactionLink) => transactionLink.balanceDate)
await synchronizeToBlockchain(context, [users, transactions], BATCH_SIZE)
// log blockchains
for(const community of context.communities.values()) {
@ -60,7 +75,16 @@ async function synchronizeToBlockchain(
// console.log(`smallest date: ${available[0].getDate()}`)
if(rounds >= 0) {
await available[0].pushToBlockchain(context)
try {
await available[0].pushToBlockchain(context)
} catch (e) {
console.error(e)
logBlogchain(context.logger, context.communities.values().next().value!.blockchain)
// for(const [key, value] of publicKeyUserIdMap.entries()) {
// console.log(`${key}: ${value}`)
// }
throw e
}
} else {
const user = (available[0] as any as OrderedContainer<any, Context>).shift()
console.log(JSON.stringify(user, null, 2))
@ -91,28 +115,16 @@ async function synchronizeToBlockchain(
}
}
async function fillBlockchains(context: Context): Promise<void> {
const BATCH_SIZE = 10
const users = new OrderedContainer(
getNextUsers,
(user: CreatedUserDb) => user.createdAt,
(context: Context, user: CreatedUserDb) => pushRegisterAddressTransaction(context, user)
)
/*const transactions = new OrderedContainer(getNextTransactions, (transaction) => transaction.balanceDate)
const transactionLinks = new OrderedContainer(getNextTransactionLinks, (transactionLink) => transactionLink.createdAt)
const deletedTransactionLinks = new OrderedContainer(getNextDeletedTransactionLinks, (transactionLink) => transactionLink.balanceDate)
*/
await synchronizeToBlockchain(context, [users], BATCH_SIZE)
}
// ---------------- load from db graiddo backend transactions format -----------------------------------------------
/// load next max ${count} users and calculate key pair for calculating signatures later
async function getNextUsers(context: Context, offset: number, count: number): Promise<CreatedUserDb[]> {
const users = await loadUsers(context.db, offset, count)
for (const user of users) {
const communityContext = context.getCommunityContextByUuid(user.communityUuid)
generateKeyPairUserAccount(user, context.cache, communityContext.topicId)
const { userKeyPair, accountKeyPair } = await generateKeyPairUserAccount(user, context.cache, communityContext.topicId)
publicKeyUserIdMap.set(userKeyPair.convertToHex(), user.gradidoId)
publicKeyUserIdMap.set(accountKeyPair.convertToHex(), user.gradidoId)
}
return users
}
@ -140,16 +152,18 @@ async function pushRegisterAddressTransaction(context: Context, user: CreatedUse
return await addRegisterAddressTransaction(communityContext.blockchain, transaction)
}
/*
async function pushTransaction(context: Context, transactionDb: TransactionDb): Promise<void> {
const senderCommunityContext = context.getCommunityContextByUuid(transactionDb.user.communityUuid)
const recipientCommunityContext = context.getCommunityContextByUuid(transactionDb.linkedUser.communityUuid)
// CreationTransactionRole will check that community topic id belongs to home community
context.cache.setHomeCommunityTopicId(senderCommunityContext.topicId)
const transaction = transactionDbToTransaction(transactionDb, senderCommunityContext.topicId, recipientCommunityContext.topicId)
await addTransaction(senderCommunityContext.blockchain, transaction)
await addTransaction(senderCommunityContext.blockchain, recipientCommunityContext.blockchain, transaction)
}
*/
// ---------------- utils ----------------------------------------------------------------------
function logBlogchain(logger: Logger, blockchain: InMemoryBlockchain) {
const f = new Filter()
f.pagination.size = 0

22
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.6/keyPair.ts
View File

@ -1,7 +1,7 @@
import { CommunityDb, UserDb } from './database'
import { KeyPairCacheManager } from '../../cache/KeyPairCacheManager'
import { KeyPairIdentifierLogic } from '../../data/KeyPairIdentifier.logic'
import { KeyPairEd25519, MemoryBlock } from 'gradido-blockchain-js'
import { KeyPairEd25519, MemoryBlock, MemoryBlockPtr } from 'gradido-blockchain-js'
import { getLogger } from 'log4js'
import { LOG4JS_BASE_CATEGORY } from '../../config/const'
import { CONFIG } from '../../config'
@ -31,19 +31,23 @@ export function generateKeyPairCommunity(community: CommunityDb, cache: KeyPairC
logger.info(`Community Key Pair added with key: ${communityKeyPairKey}`)
}
export function generateKeyPairUserAccount(user: UserDb, cache: KeyPairCacheManager, communityTopicId: HieroId): void {
export async function generateKeyPairUserAccount(
user: UserDb,
cache: KeyPairCacheManager,
communityTopicId: HieroId
): Promise<{userKeyPair: MemoryBlockPtr, accountKeyPair: MemoryBlockPtr}> {
const communityKeyPair = cache.findKeyPair(communityTopicId)!
const userKeyPair = new UserKeyPairRole(user.gradidoId, communityKeyPair).generateKeyPair()
const userKeyPairRole = new UserKeyPairRole(user.gradidoId, communityKeyPair)
const userKeyPairKey = new KeyPairIdentifierLogic({
communityTopicId: communityTopicId,
account: {
userUuid: user.gradidoId,
accountNr: 0
}
}).getKey()
cache.addKeyPair(userKeyPairKey, userKeyPair)
}).getKey()
const userKeyPair = await cache.getKeyPair(userKeyPairKey, () => Promise.resolve(userKeyPairRole.generateKeyPair()))
const accountKeyPair = new AccountKeyPairRole(1, userKeyPair).generateKeyPair()
const accountKeyPairRole = new AccountKeyPairRole(1, userKeyPair)
const accountKeyPairKey = new KeyPairIdentifierLogic({
communityTopicId: communityTopicId,
account: {
@ -51,6 +55,10 @@ export function generateKeyPairUserAccount(user: UserDb, cache: KeyPairCacheMana
accountNr: 1
}
}).getKey()
cache.addKeyPair(accountKeyPairKey, accountKeyPair)
const accountKeyPair = await cache.getKeyPair(accountKeyPairKey, () => Promise.resolve(accountKeyPairRole.generateKeyPair()))
logger.info(`Key Pairs for user and account added, user: ${userKeyPairKey}, account: ${accountKeyPairKey}`)
return {
userKeyPair: userKeyPair.getPublicKey()!,
accountKeyPair: accountKeyPair.getPublicKey()!
}
}

2
docker-compose.yml
View File

@ -71,6 +71,8 @@ services:
context: ./inspector
dockerfile: Dockerfile
target: production
profiles:
- dlt
networks:
- internal-net
ports:

26
nginx/gradido.conf
View File

@ -82,4 +82,30 @@ server {
proxy_redirect off;
}
# Inspector
location /inspector {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_pass http://inspector:3100;
proxy_redirect off;
}
# Gradido Node
location /dlt {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_pass http://dlt-connector:8340/api;
proxy_redirect off;
}
}