IT4Change/gradido
3
0
Fork 0
mirror of https://github.com/IT4Change/gradido.git synced 2026-04-06 01:25:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor migration code, fix holdAvailable Amount from transaction links on the fly

Browse Source
This commit is contained in:
einhornimmond 2025-12-29 19:54:17 +01:00
parent e1a94e3c93
commit 6d78179391
30 changed files with 281 additions and 939 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dlt-connector/bun.lock
View File

@ -1,6 +1,5 @@
{
"lockfileVersion": 1,
"configVersion": 0,
"workspaces": {
"": {
"name": "dlt-connector",
@ -23,7 +22,6 @@
"decimal.js-light": "^2.5.1",
"dotenv": "^10.0.0",
"drizzle-orm": "^0.44.7",
"drizzle-valibot": "^0.4.2",
"elysia": "1.3.8",
"graphql-request": "^7.2.0",
"jose": "^5.2.2",
@ -467,8 +465,6 @@
"drizzle-orm": ["drizzle-orm@0.44.7", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@prisma/client": "*", "@tidbcloud/serverless": "*", "@types/better-sqlite3": "*", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=7", "bun-types": "*", "expo-sqlite": ">=14.0.0", "gel": ">=2", "knex": "*", "kysely": "*", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@tidbcloud/serverless", "@types/better-sqlite3", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "gel", "knex", "kysely", "mysql2", "pg", "postgres", "sql.js", "sqlite3"] }, "sha512-quIpnYznjU9lHshEOAYLoZ9s3jweleHlZIAWR/jX9gAWNg/JhQ1wj0KGRf7/Zm+obRrYd9GjPVJg790QY9N5AQ=="],
"drizzle-valibot": ["drizzle-valibot@0.4.2", "", { "peerDependencies": { "drizzle-orm": ">=0.36.0", "valibot": ">=1.0.0-beta.7" } }, "sha512-tzjT7g0Di/HX7426marIy8IDtWODjPgrwvgscdevLQRUe5rzYzRhx6bDsYLdDFF9VI/eaYgnjNeF/fznWJoUjg=="],
"dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="],
"ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="],

1
dlt-connector/package.json
View File

@ -36,7 +36,6 @@
"decimal.js-light": "^2.5.1",
"dotenv": "^10.0.0",
"drizzle-orm": "^0.44.7",
"drizzle-valibot": "^0.4.2",
"elysia": "1.3.8",
"graphql-request": "^7.2.0",
"jose": "^5.2.2",

9
dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts
View File

@ -1,8 +1,8 @@
import {
GradidoTransaction,
HieroTransactionId,
InteractionSerialize,
InteractionValidate,
LedgerAnchor,
ValidateType_SINGLE,
} from 'gradido-blockchain-js'
import { getLogger } from 'log4js'
@ -61,11 +61,8 @@ export async function SendToHieroContext(
role.getSenderCommunityTopicId(),
)
// serialize Hiero transaction ID and attach it to the builder for the inbound transaction
const transactionIdSerializer = new InteractionSerialize(
new HieroTransactionId(outboundHieroTransactionIdString),
)
builder.setParentMessageId(transactionIdSerializer.run())
// attach Hiero transaction ID to the builder for the inbound transaction
builder.setParentLedgerAnchor(new LedgerAnchor(new HieroTransactionId(outboundHieroTransactionIdString)))
// build and validate inbound transaction
const inboundTransaction = builder.buildInbound()

39
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/binaryExport.ts
View File

@ -9,12 +9,13 @@ import {
} from 'gradido-blockchain-js'
import { CONFIG } from '../../config'
import { Context } from './Context'
import { bytesToKbyte, calculateOneHashStep } from './utils'
import { bytesString, calculateOneHashStep } from './utils'
import { CommunityContext } from './valibot.schema'
export function exportAllCommunities(context: Context, batchSize: number) {
const timeUsed = new Profiler()
for (const communityContext of context.communities.values()) {
context.logger.info(`exporting community ${communityContext.communityId} to binary file`)
exportCommunity(communityContext, context, batchSize)
}
context.logger.info(`time used for exporting communities to binary file: ${timeUsed.string()}`)
@ -25,36 +26,66 @@ export function exportCommunity(
context: Context,
batchSize: number,
) {
const timeUsed = new Profiler()
// write as binary file for GradidoNode
const f = new Filter()
f.pagination.size = batchSize
f.pagination.page = 1
f.searchDirection = SearchDirection_ASC
const binFilePath = prepareFolder(communityContext)
let count = 0
let printCount = 0
let lastTransactionCount = 0
let triggeredTransactionsCount = 0
let hash = Buffer.alloc(32, 0)
const isDebug = context.logger.isDebugEnabled()
const printConsole = () => {
if (triggeredTransactionsCount > 0) {
process.stdout.write(`exported ${count} transactions + ${triggeredTransactionsCount} triggered from timeouted transaction links\r`)
} else {
process.stdout.write(`exported ${count} transactions\r`)
}
}
do {
const transactions = communityContext.blockchain.findAll(f)
lastTransactionCount = transactions.size()
for (let i = 0; i < lastTransactionCount; i++) {
const confirmedTransaction = transactions.get(i)?.getConfirmedTransaction()
const transactionNr = f.pagination.page * batchSize + i
const transactionNr = (f.pagination.page - 2) * batchSize + i
if (!confirmedTransaction) {
throw new Error(`invalid TransactionEntry at index: ${transactionNr} `)
}
hash = exportTransaction(confirmedTransaction, hash, binFilePath)
if (confirmedTransaction?.getGradidoTransaction()?.getTransactionBody()?.isTimeoutDeferredTransfer()) {
triggeredTransactionsCount++
} else {
count++
}
if (isDebug) {
printConsole()
} else {
printCount++
if (printCount >= 100) {
printConsole()
printCount = 0
}
}
}
f.pagination.page++
f.pagination.page++
} while (lastTransactionCount === batchSize)
printConsole()
process.stdout.write(`\n`)
fs.appendFileSync(binFilePath, hash!)
context.logger.info(
`binary file for community ${communityContext.communityId} written to ${binFilePath}`,
)
const sumTransactionsCount = ((f.pagination.page - 2) * batchSize) + lastTransactionCount
const fileSize = fs.statSync(binFilePath).size
context.logger.info(
`transactions count: ${(f.pagination.page - 1) * batchSize + lastTransactionCount}, size: ${bytesToKbyte(fs.statSync(binFilePath).size)} KByte`,
`exported ${sumTransactionsCount} transactions (${bytesString(fileSize)}) in ${timeUsed.string()}`,
)
}

111
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/blockchain.ts
View File

@ -1,36 +1,20 @@
import * as fs from 'node:fs'
import {
AccountBalances,
Filter,
GradidoTransactionBuilder,
HieroAccountId,
InMemoryBlockchain,
InteractionSerialize,
TransactionType_DEFERRED_TRANSFER,
} from 'gradido-blockchain-js'
import { getLogger } from 'log4js'
import * as v from 'valibot'
import { LOG4JS_BASE_CATEGORY } from '../../config/const'
import { InputTransactionType } from '../../data/InputTransactionType.enum'
import { LinkedTransactionKeyPairRole } from '../../interactions/resolveKeyPair/LinkedTransactionKeyPair.role'
import { AbstractTransactionRole } from '../../interactions/sendToHiero/AbstractTransaction.role'
import { CommunityRootTransactionRole } from '../../interactions/sendToHiero/CommunityRootTransaction.role'
import { CreationTransactionRole } from '../../interactions/sendToHiero/CreationTransaction.role'
import { DeferredTransferTransactionRole } from '../../interactions/sendToHiero/DeferredTransferTransaction.role'
import { RedeemDeferredTransferTransactionRole } from '../../interactions/sendToHiero/RedeemDeferredTransferTransaction.role'
import { RegisterAddressTransactionRole } from '../../interactions/sendToHiero/RegisterAddressTransaction.role'
import { TransferTransactionRole } from '../../interactions/sendToHiero/TransferTransaction.role'
import { Community, Transaction } from '../../schemas/transaction.schema'
import { identifierSeedSchema } from '../../schemas/typeGuard.schema'
import { Community } from '../../schemas/transaction.schema'
import { NotEnoughGradidoBalanceError } from './errors'
const logger = getLogger(
`${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.6.blockchain`,
)
export const defaultHieroAccount = new HieroAccountId(0, 0, 2)
let transactionAddedToBlockchainSum = 0
let addToBlockchainSum = 0
const sizeBuffer = Buffer.alloc(2)
export function addToBlockchain(
builder: GradidoTransactionBuilder,
@ -39,30 +23,12 @@ export function addToBlockchain(
accountBalances: AccountBalances,
): boolean {
const transaction = builder.build()
const transactionSerializer = new InteractionSerialize(transaction)
const binTransaction = transactionSerializer.run()
if (!binTransaction) {
logger.error(`Failed to serialize transaction ${transaction.toJson(true)}`)
return false
}
const filePath = `${blockchain.getCommunityId()}.bin`
if (!addToBlockchainSum) {
// clear file
fs.writeFileSync(filePath, Buffer.alloc(0))
}
sizeBuffer.writeUInt16LE(binTransaction.size(), 0)
fs.appendFileSync(filePath, sizeBuffer)
fs.appendFileSync(filePath, binTransaction.data())
//*/
try {
const result = blockchain.createAndAddConfirmedTransactionExtern(
transaction,
transactionId,
accountBalances,
)
// logger.info(`${transactionTypeToString(transaction.getTransactionBody()?.getTransactionType()!)} Transaction added in ${timeUsed.string()}`)
addToBlockchainSum++
return result
} catch (error) {
if (error instanceof Error) {
@ -98,78 +64,3 @@ export async function addCommunityRootTransaction(
}
}
export async function addTransaction(
senderBlockchain: InMemoryBlockchain,
_recipientBlockchain: InMemoryBlockchain,
transaction: Transaction,
transactionId: number,
accountBalances: AccountBalances,
): Promise<void> {
let debugTmpStr = ''
let role: AbstractTransactionRole
if (transaction.type === InputTransactionType.GRADIDO_CREATION) {
role = new CreationTransactionRole(transaction)
} else if (transaction.type === InputTransactionType.GRADIDO_TRANSFER) {
role = new TransferTransactionRole(transaction)
} else if (transaction.type === InputTransactionType.REGISTER_ADDRESS) {
role = new RegisterAddressTransactionRole(transaction)
} else if (transaction.type === InputTransactionType.GRADIDO_DEFERRED_TRANSFER) {
role = new DeferredTransferTransactionRole(transaction)
} else if (transaction.type === InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER) {
const seedKeyPairRole = new LinkedTransactionKeyPairRole(
v.parse(identifierSeedSchema, transaction.user.seed),
)
const f = new Filter()
f.involvedPublicKey = seedKeyPairRole.generateKeyPair().getPublicKey()
f.transactionType = TransactionType_DEFERRED_TRANSFER
const deferredTransactions = senderBlockchain.findAll(f)
if (!deferredTransactions) {
throw new Error(
`redeem deferred transfer: couldn't find parent deferred transfer on Gradido Node for ${JSON.stringify(transaction, null, 2)} and public key from seed: ${f.involvedPublicKey?.convertToHex()}`,
)
}
if (deferredTransactions.size() !== 1) {
logger.error(
`redeem deferred transfer: found ${deferredTransactions.size()} parent deferred transfer on Gradido Node for ${JSON.stringify(transaction, null, 2)} and public key from seed: ${f.involvedPublicKey?.convertToHex()}`,
)
for(let i = 0; i < deferredTransactions.size(); i++) {
logger.error(`deferred transaction ${i}: ${deferredTransactions.get(i)?.getConfirmedTransaction()?.toJson(true)}`)
}
throw new Error(
`redeem deferred transfer: found ${deferredTransactions.size()} parent deferred transfer on Gradido Node for ${JSON.stringify(transaction, null, 2)} and public key from seed: ${f.involvedPublicKey?.convertToHex()}`,
)
}
const deferredTransaction = deferredTransactions.get(0)!
const confirmedDeferredTransaction = deferredTransaction.getConfirmedTransaction()
if (!confirmedDeferredTransaction) {
throw new Error('redeem deferred transfer: invalid TransactionEntry')
}
debugTmpStr += `\nconfirmed deferred transaction: ${confirmedDeferredTransaction.toJson(true)} with filter: ${f.toJson(true)}`
role = new RedeemDeferredTransferTransactionRole(
transaction,
confirmedDeferredTransaction,
)
} else {
throw new Error(`Transaction type ${transaction.type} not supported`)
}
const involvedUser = transaction.user.account
? transaction.user.account.userUuid
: transaction.linkedUser?.account?.userUuid
try {
if (addToBlockchain(await role.getGradidoTransactionBuilder(), senderBlockchain, transactionId, accountBalances)) {
// logger.debug(`${transaction.type} Transaction added for user ${involvedUser}`)
transactionAddedToBlockchainSum++
} else {
logger.error(debugTmpStr)
logger.error(`transaction: ${JSON.stringify(transaction, null, 2)}`)
throw new Error(`${transaction.type} Transaction not added for user ${involvedUser}, after ${transactionAddedToBlockchainSum} transactions`)
}
} catch(e) {
if (e instanceof NotEnoughGradidoBalanceError) {
throw e
}
logger.error(`error adding transaction: ${JSON.stringify(transaction, null, 2)}`)
throw e
}
}

136
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/convert.ts
View File

@ -1,136 +0,0 @@
import * as v from 'valibot'
import { AccountType } from '../../data/AccountType.enum'
import { InputTransactionType } from '../../data/InputTransactionType.enum'
import {
Community,
communitySchema,
Transaction,
TransactionInput,
transactionSchema,
} from '../../schemas/transaction.schema'
import {
gradidoAmountSchema,
HieroId,
memoSchema,
timeoutDurationSchema,
} from '../../schemas/typeGuard.schema'
import { TransactionTypeId } from './data/TransactionTypeId'
import { CommunityDb, CreatedUserDb, TransactionDb, TransactionLinkDb } from './valibot.schema'
export function getInputTransactionTypeFromTypeId(typeId: TransactionTypeId): InputTransactionType {
switch (typeId) {
case TransactionTypeId.CREATION:
return InputTransactionType.GRADIDO_CREATION
case TransactionTypeId.SEND:
return InputTransactionType.GRADIDO_TRANSFER
case TransactionTypeId.RECEIVE:
throw new Error('not used')
default:
throw new Error('not implemented')
}
}
export function communityDbToCommunity(
topicId: HieroId,
communityDb: CommunityDb,
creationDate: Date,
): Community {
return v.parse(communitySchema, {
hieroTopicId: topicId,
uuid: communityDb.communityUuid,
foreign: communityDb.foreign,
creationDate,
})
}
export function userDbToTransaction(userDb: CreatedUserDb, communityTopicId: HieroId): Transaction {
return v.parse(transactionSchema, {
user: {
communityTopicId: communityTopicId,
account: { userUuid: userDb.gradidoId },
},
type: InputTransactionType.REGISTER_ADDRESS,
accountType: AccountType.COMMUNITY_HUMAN,
createdAt: userDb.createdAt,
})
}
export function transactionDbToTransaction(
transactionDb: TransactionDb,
communityTopicId: HieroId,
recipientCommunityTopicId: HieroId,
): Transaction {
if (
transactionDb.typeId !== TransactionTypeId.CREATION &&
transactionDb.typeId !== TransactionTypeId.SEND &&
transactionDb.typeId !== TransactionTypeId.RECEIVE
) {
throw new Error('not implemented')
}
const user = {
communityTopicId: communityTopicId,
account: { userUuid: transactionDb.user.gradidoId },
}
const linkedUser = {
communityTopicId: recipientCommunityTopicId,
account: { userUuid: transactionDb.linkedUser.gradidoId },
}
const transaction: TransactionInput = {
user,
linkedUser,
amount: v.parse(gradidoAmountSchema, transactionDb.amount),
memo: v.parse(memoSchema, transactionDb.memo),
type: InputTransactionType.GRADIDO_TRANSFER,
createdAt: transactionDb.balanceDate,
}
if (transactionDb.typeId === TransactionTypeId.CREATION) {
if (!transactionDb.creationDate) {
throw new Error('contribution transaction without creation date')
}
transaction.targetDate = transactionDb.creationDate
transaction.type = InputTransactionType.GRADIDO_CREATION
} else if (transactionDb.typeId === TransactionTypeId.RECEIVE) {
transaction.user = linkedUser
transaction.linkedUser = user
}
if (transactionDb.transactionLinkCode) {
if (transactionDb.typeId !== TransactionTypeId.RECEIVE) {
throw new Error(
"linked transaction which isn't receive, send will taken care of on link creation",
)
}
transaction.user = {
communityTopicId: recipientCommunityTopicId,
seed: transactionDb.transactionLinkCode,
}
transaction.type = InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER
}
return v.parse(transactionSchema, transaction)
}
export function transactionLinkDbToTransaction(
transactionLinkDb: TransactionLinkDb,
communityTopicId: HieroId,
): Transaction {
return v.parse(transactionSchema, {
user: {
communityTopicId: communityTopicId,
account: { userUuid: transactionLinkDb.user.gradidoId },
},
linkedUser: {
communityTopicId: communityTopicId,
seed: transactionLinkDb.code,
},
type: InputTransactionType.GRADIDO_DEFERRED_TRANSFER,
amount: v.parse(gradidoAmountSchema, transactionLinkDb.amount),
memo: v.parse(memoSchema, transactionLinkDb.memo),
createdAt: transactionLinkDb.createdAt,
timeoutDuration: v.parse(
timeoutDurationSchema,
Math.round(
(transactionLinkDb.validUntil.getTime() - transactionLinkDb.createdAt.getTime()) / 1000,
),
),
})
}

324
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/database.ts
View File

@ -1,36 +1,19 @@
import { and, asc, count, eq, gt, inArray, isNotNull, isNull, lt, sql } from 'drizzle-orm'
import { alias } from 'drizzle-orm/mysql-core'
import { asc, eq, isNotNull, sql } from 'drizzle-orm'
import { MySql2Database } from 'drizzle-orm/mysql2'
import { getLogger } from 'log4js'
import * as v from 'valibot'
import { LOG4JS_BASE_CATEGORY } from '../../config/const'
import { ContributionStatus } from './data/ContributionStatus'
import { TransactionTypeId } from './data/TransactionTypeId'
import {
communitiesTable,
contributionsTable,
eventsTable,
TransactionSelect,
transactionLinksTable,
transactionSelectSchema,
transactionsTable,
UserSelect,
userRolesTable,
userSelectSchema,
usersTable
} from './drizzle.schema'
import { DatabaseError } from './errors'
import {
CommunityDb,
UserDb,
CreationTransactionDb,
communityDbSchema,
userDbSchema,
creationTransactionDbSchema,
TransactionDb,
TransactionLinkDb,
transactionDbSchema,
transactionLinkDbSchema,
} from './valibot.schema'
const logger = getLogger(
@ -93,21 +76,6 @@ export async function loadCommunities(db: MySql2Database): Promise<CommunityDb[]
})
}
export async function loadUsers(
db: MySql2Database,
offset: number,
count: number,
): Promise<UserDb[]> {
const result = await db
.select()
.from(usersTable)
.orderBy(asc(usersTable.createdAt), asc(usersTable.id))
.limit(count)
.offset(offset)
return result.map((row: any) => v.parse(userDbSchema, row))
}
export async function loadUserByGradidoId(db: MySql2Database, gradidoId: string): Promise<UserDb | null> {
const result = await db
.select()
@ -118,293 +86,3 @@ export async function loadUserByGradidoId(db: MySql2Database, gradidoId: string)
return result.length ? v.parse(userDbSchema, result[0]) : null
}
export async function loadLocalTransferTransactions(
db: MySql2Database,
offset: number,
count: number,
): Promise<TransactionDb[]> {
const linkedUsers = alias(usersTable, 'linkedUser')
const result = await db
.select({
transaction: transactionsTable,
user: usersTable,
linkedUser: linkedUsers,
})
.from(transactionsTable)
.where(
and(
eq(transactionsTable.typeId, TransactionTypeId.RECEIVE),
isNull(transactionsTable.transactionLinkId),
isNotNull(transactionsTable.linkedUserId),
eq(usersTable.foreign, 0),
eq(linkedUsers.foreign, 0),
)
)
.leftJoin(usersTable, eq(transactionsTable.userId, usersTable.id))
.leftJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id))
.orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id))
.limit(count)
.offset(offset)
return result.map((row: any) => {
const item = {
...row.transaction,
user: row.user,
linkedUser: row.linkedUser,
}
try {
return v.parse(transactionDbSchema, item)
} catch (e) {
throw new DatabaseError('loadLocalTransferTransactions', item, e as Error)
}
})
}
export async function loadTransactions(
db: MySql2Database,
offset: number,
count: number,
): Promise<TransactionDb[]> {
const linkedUsers = alias(usersTable, 'linkedUser')
const result = await db
.select({
transaction: transactionsTable,
user: usersTable,
linkedUser: linkedUsers,
transactionLink: {
id: transactionLinksTable.id,
code: transactionLinksTable.code
},
})
.from(transactionsTable)
.where(
and(
inArray(transactionsTable.typeId, [TransactionTypeId.CREATION, TransactionTypeId.RECEIVE]),
isNotNull(transactionsTable.linkedUserId),
eq(usersTable.foreign, 0)
)
)
.leftJoin(usersTable, eq(transactionsTable.userId, usersTable.id))
.leftJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id))
.leftJoin(
transactionLinksTable,
eq(transactionsTable.transactionLinkId, transactionLinksTable.id),
)
.orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id))
.limit(count)
.offset(offset)
return result.map((row: any) => {
// console.log(row)
try {
/*if (transactionIdSet.has(row.transaction.id)) {
throw new Error(`transaction ${row.transaction.id} already loaded`)
}
transactionIdSet.add(row.transaction.id)
*/
return v.parse(transactionDbSchema, {
...row.transaction,
transactionLinkCode: row.transactionLink ? row.transactionLink.code : null,
user: row.user,
linkedUser: row.linkedUser,
})
} catch (e) {
logger.error(`table row: ${JSON.stringify(row, null, 2)}`)
if (e instanceof v.ValiError) {
logger.error(v.flatten(e.issues))
}
throw e
}
})
}
export async function loadCreations(
db: MySql2Database,
offset: number,
count: number,
): Promise<CreationTransactionDb[]> {
const confirmedByUsers = alias(usersTable, 'confirmedByUser')
const result = await db
.select({
contribution: contributionsTable,
user: usersTable,
confirmedByUser: confirmedByUsers,
})
.from(contributionsTable)
.where(and(
isNull(contributionsTable.contributionLinkId),
eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED),
))
.leftJoin(usersTable, eq(contributionsTable.userId, usersTable.id))
.leftJoin(confirmedByUsers, eq(contributionsTable.confirmedBy, confirmedByUsers.id))
.orderBy(asc(contributionsTable.confirmedAt), asc(contributionsTable.id))
.limit(count)
.offset(offset)
return result.map((row) => {
const creationTransactionDb = {
...row.contribution,
user: row.user,
confirmedByUser: row.confirmedByUser,
}
try {
return v.parse(creationTransactionDbSchema, creationTransactionDb)
} catch (e) {
throw new DatabaseError('loadCreations', creationTransactionDb, e as Error)
}
})
}
export async function loadInvalidContributionTransactions(
db: MySql2Database,
offset: number,
count: number,
): Promise<{ id: number, balanceDate: Date }[]> {
const result = await db
.select({
id: transactionsTable.id,
balanceDate: transactionsTable.balanceDate,
})
.from(transactionsTable)
.where(
and(
eq(transactionsTable.typeId, TransactionTypeId.CREATION),
sql`NOT EXISTS (SELECT 1 FROM contributions WHERE contributions.transaction_id = transactions.id)`,
)
)
.orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id))
.limit(count)
.offset(offset)
return result.map((row: any) => {
return {
id: row.id,
balanceDate: new Date(row.balanceDate),
}
})
}
export async function loadDoubleLinkedTransactions(
db: MySql2Database,
offset: number,
rowsCount: number,
): Promise<{ id: number, balanceDate: Date }[]> {
const result = await db
.select({
id: transactionsTable.id,
balanceDate: transactionsTable.balanceDate,
transactionLinkId: transactionsTable.transactionLinkId,
cnt: count(),
})
.from(transactionsTable)
.where(
and(
eq(transactionsTable.typeId, TransactionTypeId.RECEIVE),
isNotNull(transactionsTable.transactionLinkId),
)
)
.groupBy(transactionsTable.transactionLinkId)
.having(gt(count(), 1))
.orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id))
.limit(rowsCount)
.offset(offset)
// logger.info(`loadDoubleLinkedTransactions ${result.length}: ${timeUsed.string()}`)
return result.map((row: any) => {
return {
id: row.transactionLinkId,
balanceDate: new Date(row.balanceDate),
}
})
}
export async function loadContributionLinkTransactions(
db: MySql2Database,
offset: number,
count: number,
): Promise<{ transaction: TransactionSelect, user: UserSelect, contributionLinkId: number }[]> {
const result = await db
.select({
transaction: transactionsTable,
user: usersTable,
contributionLinkId: contributionsTable.contributionLinkId,
})
.from(contributionsTable)
.where(
and(
isNotNull(contributionsTable.contributionLinkId),
isNull(transactionsTable.linkedUserId)
)
)
.leftJoin(transactionsTable, eq(contributionsTable.transactionId, transactionsTable.id))
.leftJoin(usersTable, eq(transactionsTable.userId, usersTable.id))
.orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id))
.limit(count)
.offset(offset)
return result.map((row: any) => {
if (transactionIdSet.has(row.transaction.id)) {
throw new Error(`transaction ${row.transaction.id} already loaded`)
}
transactionIdSet.add(row.transaction.id)
return {
transaction: v.parse(transactionSelectSchema, row.transaction),
user: v.parse(userSelectSchema, row.user),
contributionLinkId: row.contributionLinkId,
}
})
}
export async function loadTransactionLinks(
db: MySql2Database,
offset: number,
count: number,
): Promise<TransactionLinkDb[]> {
const result = await db
.select()
.from(transactionLinksTable)
.leftJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id))
.orderBy(asc(transactionLinksTable.createdAt), asc(transactionLinksTable.id))
.limit(count)
.offset(offset)
return result.map((row: any) => {
return v.parse(transactionLinkDbSchema, {
...row.transaction_links,
user: row.users,
})
})
}
export async function loadDeletedTransactionLinks(
db: MySql2Database,
offset: number,
count: number,
): Promise<TransactionDb[]> {
const result = await db
.select()
.from(transactionLinksTable)
.leftJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id))
.where(and(
isNotNull(transactionLinksTable.deletedAt),
lt(transactionLinksTable.deletedAt, transactionLinksTable.validUntil)
))
.orderBy(asc(transactionLinksTable.deletedAt), asc(transactionLinksTable.id))
.limit(count)
.offset(offset)
return result.map((row: any) => {
return v.parse(transactionDbSchema, {
id: row.transaction_links.id,
typeId: TransactionTypeId.RECEIVE,
amount: row.transaction_links.amount,
balanceDate: new Date(row.transaction_links.deletedAt),
memo: row.transaction_links.memo,
transactionLinkCode: row.transaction_links.code,
user: row.users,
linkedUser: row.users,
})
})
}

9
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/drizzle.schema.ts
View File

@ -10,8 +10,6 @@ import {
unique,
varchar,
} from 'drizzle-orm/mysql-core'
import { createSelectSchema } from 'drizzle-valibot'
import * as v from 'valibot'
// use only fields needed in the migration, after update the rest of the project, import database instead
export const communitiesTable = mysqlTable(
@ -62,9 +60,6 @@ export const usersTable = mysqlTable(
(table) => [unique('uuid_key').on(table.gradidoId, table.communityUuid)],
)
export const userSelectSchema = createSelectSchema(usersTable)
export type UserSelect = v.InferOutput<typeof userSelectSchema>
export const userRolesTable = mysqlTable('user_roles', {
id: int().autoincrement().notNull(),
userId: int('user_id').notNull(),
@ -94,13 +89,11 @@ export const transactionsTable = mysqlTable(
(table) => [index('user_id').on(table.userId)],
)
export const transactionSelectSchema = createSelectSchema(transactionsTable)
export type TransactionSelect = v.InferOutput<typeof transactionSelectSchema>
export const transactionLinksTable = mysqlTable('transaction_links', {
id: int().autoincrement().notNull(),
userId: int().notNull(),
amount: decimal({ precision: 40, scale: 20 }).notNull(),
holdAvailableAmount: decimal("hold_available_amount", { precision: 40, scale: 20 }).notNull(),
memo: varchar({ length: 255 }).notNull(),
code: varchar({ length: 24 }).notNull(),
createdAt: datetime({ mode: 'string' }).notNull(),

3
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/errors.ts
View File

@ -26,6 +26,7 @@ export class DatabaseError extends Error {
super(parts.join('\n\n'))
this.name = 'DatabaseError'
this.cause = originalError
}
}
@ -40,7 +41,7 @@ export class BlockchainError extends Error {
super(parts.join('\n\n'))
this.name = 'BlockchainError'
this.stack = originalError.stack
this.cause = originalError
}
}

4
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/index.ts
View File

@ -1,11 +1,9 @@
import * as v from 'valibot'
import { onShutdown } from '../../../../shared/src/helper/onShutdown'
import { uuidv4Schema } from '../../schemas/typeGuard.schema'
import { exportAllCommunities } from './binaryExport'
import { bootstrap } from './bootstrap'
import { syncDbWithBlockchainContext } from './interaction/syncDbWithBlockchain/syncDbWithBlockchain.context'
const BATCH_SIZE = 500
const BATCH_SIZE = 1000
async function main() {
// prepare in memory blockchains

38
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/AbstractBalances.role.ts
View File

@ -1,38 +0,0 @@
import { AccountBalances, Filter, InMemoryBlockchain, SearchDirection_DESC } from 'gradido-blockchain-js'
import { KeyPairIdentifierLogic } from '../../../../data/KeyPairIdentifier.logic'
import { ResolveKeyPair } from '../../../../interactions/resolveKeyPair/ResolveKeyPair.context'
import { IdentifierAccount } from '../../../../schemas/account.schema'
import { Transaction } from '../../../../schemas/transaction.schema'
import { Context } from '../../Context'
import { Balance } from '../../data/Balance'
export abstract class AbstractBalancesRole {
public constructor(protected transaction: Transaction) {}
abstract getAccountBalances(context: Context): Promise<AccountBalances>
async getLastBalanceForUser(identifierAccount: IdentifierAccount, blockchain: InMemoryBlockchain): Promise<Balance> {
const userKeyPair = await ResolveKeyPair(
new KeyPairIdentifierLogic(identifierAccount),
)
const f = new Filter()
f.involvedPublicKey = userKeyPair.getPublicKey()
f.pagination.size = 1
f.searchDirection = SearchDirection_DESC
const lastSenderTransaction = blockchain.findOne(f)
if (!lastSenderTransaction) {
throw new Error(`no last transaction found for user: ${JSON.stringify(identifierAccount, null, 2)}`)
}
const lastConfirmedTransaction = lastSenderTransaction.getConfirmedTransaction()
if (!lastConfirmedTransaction) {
throw new Error(`invalid transaction, getConfirmedTransaction call failed for transaction nr: ${lastSenderTransaction.getTransactionNr()}`)
}
const senderLastAccountBalance = lastConfirmedTransaction.getAccountBalance(userKeyPair.getPublicKey(), '')
if (!senderLastAccountBalance) {
throw new Error(
`no sender account balance found for transaction nr: ${lastSenderTransaction.getTransactionNr()} and public key: ${userKeyPair.getPublicKey()?.convertToHex()}`
)
}
return Balance.fromAccountBalance(senderLastAccountBalance, lastConfirmedTransaction.getConfirmedAt().getDate())
}
}

33
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/CreationBalances.role.ts
View File

@ -1,33 +0,0 @@
import { AccountBalances } from 'gradido-blockchain-js'
import { Transaction } from '../../../../schemas/transaction.schema'
import { Context } from '../../Context'
import { TransactionDb } from '../../valibot.schema'
import { AbstractBalancesRole } from './AbstractBalances.role'
export class CreationBalancesRole extends AbstractBalancesRole {
constructor(transaction: Transaction, protected dbTransaction: TransactionDb) {
super(transaction)
}
async getAccountBalances(context: Context): Promise<AccountBalances> {
if (this.dbTransaction.linkedUser.communityUuid !== this.dbTransaction.user.communityUuid) {
throw new Error('creation: both recipient and signer must belong to same community')
}
const accountBalances = new AccountBalances()
const communityContext = context.getCommunityContextByUuid(this.dbTransaction.user.communityUuid)
const balance = await this.getLastBalanceForUser(this.transaction.user, communityContext.blockchain)
// calculate decay since last balance with legacy calculation method
balance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate)
communityContext.aufBalance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate)
communityContext.gmwBalance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate)
accountBalances.add(balance.getAccountBalance())
accountBalances.add(communityContext.aufBalance.getAccountBalance())
accountBalances.add(communityContext.gmwBalance.getAccountBalance())
return accountBalances
}
}

33
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/DeferredTransferBalances.role.ts
View File

@ -1,33 +0,0 @@
import { AccountBalance, AccountBalances, GradidoUnit } from 'gradido-blockchain-js'
import { KeyPairIdentifierLogic } from '../../../../data/KeyPairIdentifier.logic'
import { ResolveKeyPair } from '../../../../interactions/resolveKeyPair/ResolveKeyPair.context'
import { Transaction } from '../../../../schemas/transaction.schema'
import { Context } from '../../Context'
import { TransactionLinkDb } from '../../valibot.schema'
import { AbstractBalancesRole } from './AbstractBalances.role'
export class DeferredTransferBalancesRole extends AbstractBalancesRole {
constructor(transaction: Transaction, protected dbTransactionLink: TransactionLinkDb) {
super(transaction)
}
async getAccountBalances(context: Context): Promise<AccountBalances> {
const senderCommunityContext = context.getCommunityContextByUuid(this.dbTransactionLink.user.communityUuid)
const accountBalances = new AccountBalances()
const seededIdentifier = new KeyPairIdentifierLogic(this.transaction.linkedUser!)
if (!seededIdentifier.isSeedKeyPair()) {
throw new Error(`linked user is not a seed: ${JSON.stringify(this.transaction, null, 2)}`)
}
const seedKeyPair = await ResolveKeyPair(seededIdentifier)
const senderAccountBalance = await this.getLastBalanceForUser(this.transaction.user, senderCommunityContext.blockchain)
let amount = GradidoUnit.fromString(this.dbTransactionLink.amount.toString())
amount = amount.calculateCompoundInterest((this.dbTransactionLink.validUntil.getTime() - this.dbTransactionLink.createdAt.getTime()) / 60000)
senderAccountBalance.updateLegacyDecay(amount.negated(), this.dbTransactionLink.createdAt)
accountBalances.add(senderAccountBalance.getAccountBalance())
accountBalances.add(new AccountBalance(seedKeyPair.getPublicKey(), amount, ''))
return accountBalances
}
}

30
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RedeemDeferredTransferBalances.role.ts
View File

@ -1,30 +0,0 @@
import { AccountBalances } from 'gradido-blockchain-js'
import { Transaction } from '../../../../schemas/transaction.schema'
import { Context } from '../../Context'
import { TransactionDb } from '../../valibot.schema'
import { AbstractBalancesRole } from './AbstractBalances.role'
export class RedeemDeferredTransferBalancesRole extends AbstractBalancesRole {
constructor(transaction: Transaction, protected dbTransaction: TransactionDb) {
super(transaction)
}
async getAccountBalances(context: Context): Promise<AccountBalances> {
// I use the receiving part of transaction pair, so the user is the recipient and the linked user the sender and amount is positive
const senderCommunityContext = context.getCommunityContextByUuid(this.dbTransaction.linkedUser.communityUuid)
const recipientCommunityContext = context.getCommunityContextByUuid(this.dbTransaction.user.communityUuid)
const accountBalances = new AccountBalances()
context.cache.setHomeCommunityTopicId(senderCommunityContext.topicId)
const senderLastBalance = await this.getLastBalanceForUser(this.transaction.linkedUser!, senderCommunityContext.blockchain)
context.cache.setHomeCommunityTopicId(recipientCommunityContext.topicId)
const recipientLastBalance = await this.getLastBalanceForUser(this.transaction.user, recipientCommunityContext.blockchain)
senderLastBalance.updateLegacyDecay(this.dbTransaction.amount.negate(), this.dbTransaction.balanceDate)
recipientLastBalance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate)
accountBalances.add(senderLastBalance.getAccountBalance())
accountBalances.add(recipientLastBalance.getAccountBalance())
return accountBalances
}
}

17
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/RegisterAddressBalances.role.ts
View File

@ -1,17 +0,0 @@
import { AccountBalance, AccountBalances, GradidoUnit } from 'gradido-blockchain-js'
import { KeyPairIdentifierLogic } from '../../../../data/KeyPairIdentifier.logic'
import { ResolveKeyPair } from '../../../../interactions/resolveKeyPair/ResolveKeyPair.context'
import { Context } from '../../Context'
import { AbstractBalancesRole } from './AbstractBalances.role'
export class RegisterAddressBalancesRole extends AbstractBalancesRole {
async getAccountBalances(_context: Context): Promise<AccountBalances> {
const accountBalances = new AccountBalances()
const recipientKeyPair = await ResolveKeyPair(
new KeyPairIdentifierLogic(this.transaction.user),
)
accountBalances.add(new AccountBalance(recipientKeyPair.getPublicKey(), GradidoUnit.zero(), ''))
return accountBalances
}
}

30
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/TransferBalances.role.ts
View File

@ -1,30 +0,0 @@
import { AccountBalances } from 'gradido-blockchain-js'
import { Transaction } from '../../../../schemas/transaction.schema'
import { Context } from '../../Context'
import { TransactionDb } from '../../valibot.schema'
import { AbstractBalancesRole } from './AbstractBalances.role'
export class TransferBalancesRole extends AbstractBalancesRole {
constructor(transaction: Transaction, protected dbTransaction: TransactionDb) {
super(transaction)
}
async getAccountBalances(context: Context): Promise<AccountBalances> {
// I use the receiving part of transaction pair, so the user is the recipient and the linked user the sender and amount is positive
const senderCommunityContext = context.getCommunityContextByUuid(this.dbTransaction.linkedUser.communityUuid)
const recipientCommunityContext = context.getCommunityContextByUuid(this.dbTransaction.user.communityUuid)
const accountBalances = new AccountBalances()
context.cache.setHomeCommunityTopicId(senderCommunityContext.topicId)
const senderLastBalance = await this.getLastBalanceForUser(this.transaction.linkedUser!, senderCommunityContext.blockchain)
context.cache.setHomeCommunityTopicId(recipientCommunityContext.topicId)
const recipientLastBalance = await this.getLastBalanceForUser(this.transaction.user, recipientCommunityContext.blockchain)
senderLastBalance.updateLegacyDecay(this.dbTransaction.amount.negate(), this.dbTransaction.balanceDate)
recipientLastBalance.updateLegacyDecay(this.dbTransaction.amount, this.dbTransaction.balanceDate)
accountBalances.add(senderLastBalance.getAccountBalance())
accountBalances.add(recipientLastBalance.getAccountBalance())
return accountBalances
}
}

32
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/accountBalances/accountBalances.context.ts
View File

@ -1,32 +0,0 @@
import { AccountBalances } from 'gradido-blockchain-js'
import * as v from 'valibot'
import { InputTransactionType } from '../../../../data/InputTransactionType.enum'
import { Transaction } from '../../../../schemas/transaction.schema'
import { Context } from '../../Context'
import { TransactionDb, TransactionLinkDb, transactionDbSchema, transactionLinkDbSchema } from '../../valibot.schema'
import { AbstractBalancesRole } from './AbstractBalances.role'
import { CreationBalancesRole } from './CreationBalances.role'
import { DeferredTransferBalancesRole } from './DeferredTransferBalances.role'
import { RedeemDeferredTransferBalancesRole } from './RedeemDeferredTransferBalances.role'
import { RegisterAddressBalancesRole } from './RegisterAddressBalances.role'
import { TransferBalancesRole } from './TransferBalances.role'
export async function accountBalancesContext(transaction: Transaction, item: TransactionDb | TransactionLinkDb, context: Context): Promise<AccountBalances> {
let role: AbstractBalancesRole | null = null
if (InputTransactionType.GRADIDO_CREATION === transaction.type) {
role = new CreationBalancesRole(transaction, v.parse(transactionDbSchema, item))
} else if (InputTransactionType.GRADIDO_TRANSFER === transaction.type) {
role = new TransferBalancesRole(transaction, v.parse(transactionDbSchema, item))
} else if (InputTransactionType.GRADIDO_DEFERRED_TRANSFER === transaction.type) {
role = new DeferredTransferBalancesRole(transaction, v.parse(transactionLinkDbSchema, item))
} else if (InputTransactionType.GRADIDO_REDEEM_DEFERRED_TRANSFER === transaction.type) {
role = new RedeemDeferredTransferBalancesRole(transaction, v.parse(transactionDbSchema, item))
} else if (InputTransactionType.REGISTER_ADDRESS === transaction.type) {
role = new RegisterAddressBalancesRole(transaction)
}
if (!role) {
throw new Error(`No role found for transaction type ${transaction.type}`)
}
return await role.getAccountBalances(context)
}

43
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/AbstractSync.role.ts
View File

@ -7,9 +7,14 @@ import { Context } from '../../Context'
import { Balance } from '../../data/Balance'
import { CommunityContext } from '../../valibot.schema'
export abstract class AbstractSyncRole<T> {
private items: T[] = []
private offset = 0
export type IndexType = {
date: Date
id: number
}
export abstract class AbstractSyncRole<ItemType> {
private items: ItemType[] = []
protected lastIndex: IndexType = { date: new Date(0), id: 0 }
protected logger: Logger
constructor(protected readonly context: Context) {
@ -54,13 +59,15 @@ export abstract class AbstractSyncRole<T> {
const lastTransactions = blockchain.findAll(f)
for (let i = lastTransactions.size() - 1; i >= 0; i--) {
const tx = lastTransactions.get(i)
this.context.logger.debug(`${tx?.getConfirmedTransaction()!.toJson(true)}`)
this.context.logger.debug(`${i}: ${tx?.getConfirmedTransaction()!.toJson(true)}`)
}
}
abstract getDate(): Date
abstract loadFromDb(offset: number, count: number): Promise<T[]>
abstract pushToBlockchain(item: T): void
// for using seek rather than offset pagination approach
abstract getLastIndex(): IndexType
abstract loadFromDb(lastIndex: IndexType, count: number): Promise<ItemType[]>
abstract pushToBlockchain(item: ItemType): void
abstract itemTypeName(): string
// return count of new loaded items
@ -70,12 +77,14 @@ export abstract class AbstractSyncRole<T> {
if (this.logger.isDebugEnabled()) {
timeUsed = new Profiler()
}
this.items = await this.loadFromDb(this.offset, batchSize)
this.offset += this.items.length
if (timeUsed && this.items.length) {
this.logger.debug(
`${timeUsed.string()} for loading ${this.items.length} ${this.itemTypeName()} from db`,
)
this.items = await this.loadFromDb(this.lastIndex, batchSize)
if (this.length > 0) {
this.lastIndex = this.getLastIndex()
if (timeUsed) {
this.logger.debug(
`${timeUsed.string()} for loading ${this.items.length} ${this.itemTypeName()} from db`,
)
}
}
return this.items.length
}
@ -89,14 +98,20 @@ export abstract class AbstractSyncRole<T> {
this.pushToBlockchain(this.shift())
}
peek(): T {
peek(): ItemType {
if (this.isEmpty()) {
throw new Error(`[peek] No items, please call this only if isEmpty returns false`)
}
return this.items[0]
}
peekLast(): ItemType {
if (this.isEmpty()) {
throw new Error(`[peekLast] No items, please call this only if isEmpty returns false`)
}
return this.items[this.items.length - 1]
}
shift(): T {
shift(): ItemType {
const item = this.items.shift()
if (!item) {
throw new Error(`[shift] No items, shift return undefined`)

14
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/ContributionLinkTransactionSync.role.ts
View File

@ -1,4 +1,4 @@
import { and, asc, eq, isNotNull } from 'drizzle-orm'
import { and, asc, eq, gt, isNotNull, or } from 'drizzle-orm'
import * as v from 'valibot'
import { Context } from '../../Context'
import { contributionLinkModerators } from '../../database'
@ -7,6 +7,8 @@ import { CreationsSyncRole } from './CreationsSync.role'
import { contributionsTable, usersTable } from '../../drizzle.schema'
import { ContributionStatus } from '../../data/ContributionStatus'
import { DatabaseError } from '../../errors'
import { IndexType } from './AbstractSync.role'
import { toMysqlDateTime } from '../../utils'
export class ContributionLinkTransactionSyncRole extends CreationsSyncRole {
constructor(readonly context: Context) {
@ -16,7 +18,7 @@ export class ContributionLinkTransactionSyncRole extends CreationsSyncRole {
return 'contributionLinkTransaction'
}
async loadFromDb(offset: number, count: number): Promise<CreationTransactionDb[]> {
async loadFromDb(lastIndex: IndexType, count: number): Promise<CreationTransactionDb[]> {
const result = await this.context.db
.select({
contribution: contributionsTable,
@ -26,11 +28,17 @@ export class ContributionLinkTransactionSyncRole extends CreationsSyncRole {
.where(and(
isNotNull(contributionsTable.contributionLinkId),
eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED),
or(
gt(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)),
and(
eq(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)),
gt(contributionsTable.transactionId, lastIndex.id)
)
)
))
.innerJoin(usersTable, eq(contributionsTable.userId, usersTable.id))
.orderBy(asc(contributionsTable.confirmedAt), asc(contributionsTable.transactionId))
.limit(count)
.offset(offset)
const verifiedCreationTransactions: CreationTransactionDb[] = []
for(const row of result) {

31
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/CreationsSync.role.ts
View File

@ -1,12 +1,15 @@
import { and, asc, eq, isNull } from 'drizzle-orm'
import { and, asc, eq, isNull, gt, or } from 'drizzle-orm'
import { alias } from 'drizzle-orm/mysql-core'
import {
AccountBalances,
AuthenticatedEncryption,
EncryptedMemo,
Filter,
GradidoTransactionBuilder,
KeyPairEd25519,
MemoryBlockPtr,
SearchDirection_DESC,
TransactionType_CREATION,
TransferAmount
} from 'gradido-blockchain-js'
import * as v from 'valibot'
@ -18,7 +21,8 @@ import {
} from '../../drizzle.schema'
import { BlockchainError, DatabaseError } from '../../errors'
import { CommunityContext, CreationTransactionDb, creationTransactionDbSchema } from '../../valibot.schema'
import { AbstractSyncRole } from './AbstractSync.role'
import { AbstractSyncRole, IndexType } from './AbstractSync.role'
import { toMysqlDateTime } from '../../utils'
export class CreationsSyncRole extends AbstractSyncRole<CreationTransactionDb> {
@ -26,11 +30,16 @@ export class CreationsSyncRole extends AbstractSyncRole<CreationTransactionDb> {
return this.peek().confirmedAt
}
getLastIndex(): IndexType {
const lastItem = this.peekLast()
return { date: lastItem.confirmedAt, id: lastItem.transactionId }
}
itemTypeName(): string {
return 'creationTransactions'
}
async loadFromDb(offset: number, count: number): Promise<CreationTransactionDb[]> {
async loadFromDb(lastIndex: IndexType, count: number): Promise<CreationTransactionDb[]> {
const confirmedByUsers = alias(usersTable, 'confirmedByUser')
const result = await this.context.db
.select({
@ -42,12 +51,18 @@ export class CreationsSyncRole extends AbstractSyncRole<CreationTransactionDb> {
.where(and(
isNull(contributionsTable.contributionLinkId),
eq(contributionsTable.contributionStatus, ContributionStatus.CONFIRMED),
or(
gt(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)),
and(
eq(contributionsTable.confirmedAt, toMysqlDateTime(lastIndex.date)),
gt(contributionsTable.transactionId, lastIndex.id)
)
)
))
.innerJoin(usersTable, eq(contributionsTable.userId, usersTable.id))
.innerJoin(confirmedByUsers, eq(contributionsTable.confirmedBy, confirmedByUsers.id))
.orderBy(asc(contributionsTable.confirmedAt), asc(contributionsTable.transactionId))
.limit(count)
.offset(offset)
return result.map((row) => {
const item = {
@ -126,6 +141,14 @@ export class CreationsSyncRole extends AbstractSyncRole<CreationTransactionDb> {
this.calculateAccountBalances(item, communityContext, recipientPublicKey),
)
} catch(e) {
const f= new Filter()
f.transactionType = TransactionType_CREATION
f.searchDirection = SearchDirection_DESC
f.pagination.size = 1
const lastContribution = blockchain.findOne(f)
if (lastContribution) {
this.context.logger.warn(`last contribution: ${lastContribution.getConfirmedTransaction()?.toJson(true)}`)
}
throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error)
}
}

22
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts
View File

@ -1,7 +1,7 @@
import { CommunityContext, DeletedTransactionLinkDb, deletedTransactionLinKDbSchema } from '../../valibot.schema'
import { AbstractSyncRole } from './AbstractSync.role'
import { AbstractSyncRole, IndexType } from './AbstractSync.role'
import { transactionLinksTable, usersTable } from '../../drizzle.schema'
import { and, lt, asc, isNotNull, eq } from 'drizzle-orm'
import { and, lt, asc, isNotNull, eq, or, gt } from 'drizzle-orm'
import * as v from 'valibot'
import {
AccountBalance,
@ -19,17 +19,23 @@ import { deriveFromCode } from '../../../../data/deriveKeyPair'
import { addToBlockchain } from '../../blockchain'
import { BlockchainError, DatabaseError } from '../../errors'
import { Balance } from '../../data/Balance'
import { toMysqlDateTime } from '../../utils'
export class DeletedTransactionLinksSyncRole extends AbstractSyncRole<DeletedTransactionLinkDb> {
getDate(): Date {
return this.peek().deletedAt
}
getLastIndex(): IndexType {
const lastItem = this.peekLast()
return { date: lastItem.deletedAt, id: lastItem.id }
}
itemTypeName(): string {
return 'deletedTransactionLinks'
}
async loadFromDb(offset: number, count: number): Promise<DeletedTransactionLinkDb[]> {
async loadFromDb(lastIndex: IndexType, count: number): Promise<DeletedTransactionLinkDb[]> {
const result = await this.context.db
.select({
transactionLink: transactionLinksTable,
@ -39,13 +45,19 @@ export class DeletedTransactionLinksSyncRole extends AbstractSyncRole<DeletedTra
.where(
and(
isNotNull(transactionLinksTable.deletedAt),
lt(transactionLinksTable.deletedAt, transactionLinksTable.validUntil)
lt(transactionLinksTable.deletedAt, transactionLinksTable.validUntil),
or(
gt(transactionLinksTable.deletedAt, toMysqlDateTime(lastIndex.date)),
and(
eq(transactionLinksTable.deletedAt, toMysqlDateTime(lastIndex.date)),
gt(transactionLinksTable.id, lastIndex.id)
)
)
)
)
.innerJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id))
.orderBy(asc(transactionLinksTable.deletedAt), asc(transactionLinksTable.id))
.limit(count)
.offset(offset)
return result.map((row) => {
const item = {

27
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/DoubleLinkedTransactions.role.ts
View File

@ -1,27 +0,0 @@
import { Context } from '../../Context'
import { loadDoubleLinkedTransactions } from '../../database'
import { AbstractSyncRole } from './AbstractSync.role'
export class DoubleLinkedTransactionsSyncRole extends AbstractSyncRole<{ id: number, balanceDate: Date }> {
static allTransactionIds: number[] = []
constructor(readonly context: Context) {
super(context)
}
itemTypeName(): string {
return 'doubleLinkedTransaction'
}
async loadFromDb(offset: number, count: number): Promise<{ id: number, balanceDate: Date }[]> {
const result = await loadDoubleLinkedTransactions(this.context.db, offset, count)
DoubleLinkedTransactionsSyncRole.allTransactionIds.push(...result.map((r) => r.id))
return result
}
getDate(): Date {
return this.peek().balanceDate
}
async pushToBlockchain(item: { id: number, balanceDate: Date }): Promise<void> {
this.logger.warn(`Double transaction_links ${item.id} found.`)
}
}

27
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/InvalidContributionTransactionSync.role.ts
View File

@ -1,27 +0,0 @@
import { Context } from '../../Context'
import { loadInvalidContributionTransactions } from '../../database'
import { AbstractSyncRole } from './AbstractSync.role'
export class InvalidContributionTransactionSyncRole extends AbstractSyncRole<{ id: number, balanceDate: Date }> {
static allTransactionIds: number[] = []
constructor(readonly context: Context) {
super(context)
}
itemTypeName(): string {
return 'invalidContributionTransaction'
}
async loadFromDb(offset: number, count: number): Promise<{ id: number, balanceDate: Date }[]> {
const result = await loadInvalidContributionTransactions(this.context.db, offset, count)
InvalidContributionTransactionSyncRole.allTransactionIds.push(...result.map((r) => r.id))
return result
}
getDate(): Date {
return this.peek().balanceDate
}
async pushToBlockchain(item: { id: number, balanceDate: Date }): Promise<void> {
this.logger.warn(`Invalid contribution transaction ${item.id} found.`)
}
}

23
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts
View File

@ -1,4 +1,4 @@
import { and, asc, eq, isNotNull, isNull } from 'drizzle-orm'
import { and, asc, eq, isNotNull, isNull, gt, or } from 'drizzle-orm'
import { alias } from 'drizzle-orm/mysql-core'
import {
AccountBalances,
@ -17,7 +17,8 @@ import { TransactionTypeId } from '../../data/TransactionTypeId'
import { transactionsTable, usersTable } from '../../drizzle.schema'
import { BlockchainError, DatabaseError, NegativeBalanceError, NotEnoughGradidoBalanceError } from '../../errors'
import { CommunityContext, TransactionDb, transactionDbSchema } from '../../valibot.schema'
import { AbstractSyncRole } from './AbstractSync.role'
import { AbstractSyncRole, IndexType } from './AbstractSync.role'
import { toMysqlDateTime } from '../../utils'
export class LocalTransactionsSyncRole extends AbstractSyncRole<TransactionDb> {
@ -25,11 +26,16 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole<TransactionDb> {
return this.peek().balanceDate
}
getLastIndex(): IndexType {
const lastItem = this.peekLast()
return { date: lastItem.balanceDate, id: lastItem.id }
}
itemTypeName(): string {
return 'localTransactions'
}
async loadFromDb(offset: number, count: number): Promise<TransactionDb[]> {
async loadFromDb(lastIndex: IndexType, count: number): Promise<TransactionDb[]> {
const linkedUsers = alias(usersTable, 'linkedUser')
const result = await this.context.db
.select({
@ -45,13 +51,19 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole<TransactionDb> {
isNotNull(transactionsTable.linkedUserId),
eq(usersTable.foreign, 0),
eq(linkedUsers.foreign, 0),
or(
gt(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)),
and(
eq(transactionsTable.balanceDate, toMysqlDateTime(lastIndex.date)),
gt(transactionsTable.id, lastIndex.id)
)
)
)
)
.innerJoin(usersTable, eq(transactionsTable.userId, usersTable.id))
.innerJoin(linkedUsers, eq(transactionsTable.linkedUserId, linkedUsers.id))
.orderBy(asc(transactionsTable.balanceDate), asc(transactionsTable.id))
.limit(count)
.offset(offset)
return result.map((row) => {
const item = {
@ -74,8 +86,7 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole<TransactionDb> {
): GradidoTransactionBuilder {
return new GradidoTransactionBuilder()
.setCreatedAt(item.balanceDate)
.addMemo(
new EncryptedMemo(
.addMemo(new EncryptedMemo(
item.memo,
new AuthenticatedEncryption(senderKeyPair),
new AuthenticatedEncryption(recipientKeyPair),

22
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts
View File

@ -1,4 +1,4 @@
import { and, asc, eq, isNotNull, isNull } from 'drizzle-orm'
import { and, asc, eq, isNotNull, isNull, or, gt } from 'drizzle-orm'
import {
AccountBalance,
AccountBalances,
@ -18,20 +18,26 @@ import { addToBlockchain } from '../../blockchain'
import { transactionLinksTable, usersTable } from '../../drizzle.schema'
import { BlockchainError, DatabaseError } from '../../errors'
import { CommunityContext, RedeemedTransactionLinkDb, redeemedTransactionLinkDbSchema } from '../../valibot.schema'
import { AbstractSyncRole } from './AbstractSync.role'
import { AbstractSyncRole, IndexType } from './AbstractSync.role'
import { deriveFromCode } from '../../../../data/deriveKeyPair'
import { alias } from 'drizzle-orm/mysql-core'
import { toMysqlDateTime } from '../../utils'
export class RedeemTransactionLinksSyncRole extends AbstractSyncRole<RedeemedTransactionLinkDb> {
getDate(): Date {
return this.peek().redeemedAt
}
getLastIndex(): IndexType {
const lastItem = this.peekLast()
return { date: lastItem.redeemedAt, id: lastItem.id }
}
itemTypeName(): string {
return 'redeemTransactionLinks'
}
async loadFromDb(offset: number, count: number): Promise<RedeemedTransactionLinkDb[]> {
async loadFromDb(lastIndex: IndexType, count: number): Promise<RedeemedTransactionLinkDb[]> {
const redeemedByUser = alias(usersTable, 'redeemedByUser')
const result = await this.context.db
.select({
@ -45,14 +51,20 @@ export class RedeemTransactionLinksSyncRole extends AbstractSyncRole<RedeemedTra
isNull(transactionLinksTable.deletedAt),
isNotNull(transactionLinksTable.redeemedAt),
eq(usersTable.foreign, 0),
eq(redeemedByUser.foreign, 0)
eq(redeemedByUser.foreign, 0),
or(
gt(transactionLinksTable.redeemedAt, toMysqlDateTime(lastIndex.date)),
and(
eq(transactionLinksTable.redeemedAt, toMysqlDateTime(lastIndex.date)),
gt(transactionLinksTable.id, lastIndex.id)
)
)
)
)
.innerJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id))
.innerJoin(redeemedByUser, eq(transactionLinksTable.redeemedBy, redeemedByUser.id))
.orderBy(asc(transactionLinksTable.redeemedAt), asc(transactionLinksTable.id))
.limit(count)
.offset(offset)
return result.map((row) => {
const item = {

69
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/TransactionLinkFundingsSync.role.ts
View File

@ -1,17 +1,15 @@
import { asc, eq } from 'drizzle-orm'
import { asc, eq, or, gt, and, isNull } from 'drizzle-orm'
import {
AccountBalance,
AccountBalances,
AuthenticatedEncryption,
DurationSeconds,
EncryptedMemo,
Filter,
GradidoTransactionBuilder,
GradidoTransfer,
GradidoUnit,
KeyPairEd25519,
MemoryBlockPtr,
SearchDirection_DESC,
TransferAmount
} from 'gradido-blockchain-js'
import * as v from 'valibot'
@ -19,9 +17,9 @@ import { addToBlockchain } from '../../blockchain'
import { transactionLinksTable, usersTable } from '../../drizzle.schema'
import { BlockchainError, DatabaseError, NegativeBalanceError } from '../../errors'
import { CommunityContext, TransactionLinkDb, transactionLinkDbSchema } from '../../valibot.schema'
import { AbstractSyncRole } from './AbstractSync.role'
import { AbstractSyncRole, IndexType } from './AbstractSync.role'
import { deriveFromCode } from '../../../../data/deriveKeyPair'
import { legacyCalculateDecay } from '../../utils'
import { calculateEffectiveSeconds, reverseLegacyDecay, toMysqlDateTime } from '../../utils'
import Decimal from 'decimal.js-light'
export class TransactionLinkFundingsSyncRole extends AbstractSyncRole<TransactionLinkDb> {
@ -29,19 +27,30 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole<Transactio
return this.peek().createdAt
}
getLastIndex(): IndexType {
const lastItem = this.peekLast()
return { date: lastItem.createdAt, id: lastItem.id }
}
itemTypeName(): string {
return 'transactionLinkFundings'
}
async loadFromDb(offset: number, count: number): Promise<TransactionLinkDb[]> {
async loadFromDb(lastIndex: IndexType, count: number): Promise<TransactionLinkDb[]> {
const result = await this.context.db
.select()
.from(transactionLinksTable)
.innerJoin(usersTable, eq(transactionLinksTable.userId, usersTable.id))
.where(or(
gt(transactionLinksTable.createdAt, toMysqlDateTime(lastIndex.date)),
and(
eq(transactionLinksTable.createdAt, toMysqlDateTime(lastIndex.date)),
gt(transactionLinksTable.id, lastIndex.id)
)
))
.orderBy(asc(transactionLinksTable.createdAt), asc(transactionLinksTable.id))
.limit(count)
.offset(offset)
return result.map((row) => {
const item = {
...row.transaction_links,
@ -90,7 +99,15 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole<Transactio
): AccountBalances {
const accountBalances = new AccountBalances()
let senderLastBalance = this.getLastBalanceForUser(senderPublicKey, communityContext.blockchain)
senderLastBalance.updateLegacyDecay(blockedAmount.negated(), item.createdAt)
try {
senderLastBalance.updateLegacyDecay(blockedAmount.negated(), item.createdAt)
} catch(e) {
if (e instanceof NegativeBalanceError) {
this.logLastBalanceChangingTransactions(senderPublicKey, communityContext.blockchain)
this.context.logger.debug(`sender public key: ${senderPublicKey.convertToHex()}`)
throw e
}
}
accountBalances.add(senderLastBalance.getAccountBalance())
accountBalances.add(new AccountBalance(recipientPublicKey, blockedAmount, ''))
@ -109,9 +126,32 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole<Transactio
if (!senderKeyPair || !senderPublicKey || !recipientKeyPair || !recipientPublicKey) {
throw new Error(`missing key for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`)
}
const duration = new DurationSeconds((item.validUntil.getTime() - item.createdAt.getTime()) / 1000)
let blockedAmount = item.amount.calculateCompoundInterest(duration.getSeconds())
let endDateTime: number = item.validUntil.getTime()
if (item.redeemedAt) {
endDateTime = item.redeemedAt.getTime() + (1000 * 120)
} else if (item.deletedAt) {
endDateTime = item.deletedAt.getTime() + (1000 * 120)
} else {
const duration = new DurationSeconds((endDateTime - item.createdAt.getTime()) / 1000)
const blockedAmount = GradidoUnit.fromString(reverseLegacyDecay(new Decimal(item.amount.toString()), duration.getSeconds()).toString())
const secondsDiff = calculateEffectiveSeconds(
new Decimal(item.holdAvailableAmount.toString()),
new Decimal(blockedAmount.toString())
)
endDateTime = endDateTime - secondsDiff.toNumber() * 1000
}
if (endDateTime > item.validUntil.getTime()) {
endDateTime = item.validUntil.getTime()
}
let duration = new DurationSeconds((endDateTime - item.createdAt.getTime()) / 1000)
const hourInSeconds = 60 * 60
if (duration.getSeconds() < hourInSeconds) {
duration = new DurationSeconds(hourInSeconds)
}
let blockedAmount = GradidoUnit.fromString(reverseLegacyDecay(new Decimal(item.amount.toString()), duration.getSeconds()).toString())
blockedAmount = blockedAmount.add(GradidoUnit.fromGradidoCent(1))
// let blockedAmount = decayedAmount.calculateCompoundInterest(duration.getSeconds())
let accountBalances: AccountBalances
try {
accountBalances = this.calculateBalances(item, blockedAmount, communityContext, senderPublicKey, recipientPublicKey)
@ -119,9 +159,14 @@ export class TransactionLinkFundingsSyncRole extends AbstractSyncRole<Transactio
if (item.deletedAt && e instanceof NegativeBalanceError) {
const senderLastBalance = this.getLastBalanceForUser(senderPublicKey, communityContext.blockchain)
senderLastBalance.updateLegacyDecay(GradidoUnit.zero(), item.createdAt)
const oldBlockedAmountString = blockedAmount.toString()
blockedAmount = senderLastBalance.getBalance()
accountBalances = this.calculateBalances(item, blockedAmount, communityContext, senderPublicKey, recipientPublicKey)
this.context.logger.warn(
`workaround: fix founding for deleted link, reduce funding to actual sender balance: ${senderPublicKey.convertToHex()}: from ${oldBlockedAmountString} GDD to ${blockedAmount.toString()} GDD`
)
} else {
this.context.logger.error(`error calculate account balances for ${this.itemTypeName()}: ${JSON.stringify(item, null, 2)}`)
throw e
}
}

20
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/UsersSync.role.ts
View File

@ -1,4 +1,4 @@
import { asc } from 'drizzle-orm'
import { asc, and, gt, eq, or } from 'drizzle-orm'
import {
AccountBalance,
AccountBalances,
@ -15,7 +15,8 @@ import { addToBlockchain } from '../../blockchain'
import { usersTable } from '../../drizzle.schema'
import { BlockchainError, DatabaseError } from '../../errors'
import { UserDb, userDbSchema } from '../../valibot.schema'
import { AbstractSyncRole } from './AbstractSync.role'
import { AbstractSyncRole, IndexType } from './AbstractSync.role'
import { toMysqlDateTime } from '../../utils'
export class UsersSyncRole extends AbstractSyncRole<UserDb> {
@ -23,17 +24,28 @@ export class UsersSyncRole extends AbstractSyncRole<UserDb> {
return this.peek().createdAt
}
getLastIndex(): IndexType {
const lastItem = this.peekLast()
return { date: lastItem.createdAt, id: lastItem.id }
}
itemTypeName(): string {
return 'users'
}
async loadFromDb(offset: number, count: number): Promise<UserDb[]> {
async loadFromDb(lastIndex: IndexType, count: number): Promise<UserDb[]> {
const result = await this.context.db
.select()
.from(usersTable)
.where(or(
gt(usersTable.createdAt, toMysqlDateTime(lastIndex.date)),
and(
eq(usersTable.createdAt, toMysqlDateTime(lastIndex.date)),
gt(usersTable.id, lastIndex.id)
)
))
.orderBy(asc(usersTable.createdAt), asc(usersTable.id))
.limit(count)
.offset(offset)
return result.map((row) => {
try {

39
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/interaction/syncDbWithBlockchain/syncDbWithBlockchain.context.ts
View File

@ -1,7 +1,6 @@
import { Profiler } from 'gradido-blockchain-js'
import { Context } from '../../Context'
import { CreationsSyncRole } from './CreationsSync.role'
import { InvalidContributionTransactionSyncRole } from './InvalidContributionTransactionSync.role'
import { LocalTransactionsSyncRole } from './LocalTransactionsSync.role'
import { UsersSyncRole } from './UsersSync.role'
import { TransactionLinkFundingsSyncRole } from './TransactionLinkFundingsSync.role'
@ -24,13 +23,15 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n
]
let transactionsCount = 0
let transactionsCountSinceLastLog = 0
let transactionsCountSinceLastPrint = 0
let available = containers
const isDebug = context.logger.isDebugEnabled()
while (true) {
timeUsedDB.reset()
const results = await Promise.all(available.map((c) => c.ensureFilled(batchSize)))
const loadedItemsCount = results.reduce((acc, c) => acc + c, 0)
// log only, if at least one new item was loaded
if (loadedItemsCount && context.logger.isDebugEnabled()) {
if (loadedItemsCount && isDebug) {
context.logger.debug(`${loadedItemsCount} new items loaded from db in ${timeUsedDB.string()}`)
}
@ -46,24 +47,24 @@ export async function syncDbWithBlockchainContext(context: Context, batchSize: n
available.sort((a, b) => a.getDate().getTime() - b.getDate().getTime())
// context.logger.debug(`sorted ${available.length} containers in ${sortTime.string()}`)
}
available[0].toBlockchain()
process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`)
available[0].toBlockchain()
transactionsCount++
transactionsCountSinceLastLog++
if (transactionsCountSinceLastLog >= batchSize) {
context.logger.debug(`${transactionsCountSinceLastLog} transactions added to blockchain in ${timeUsedBlockchain.string()}`)
timeUsedBlockchain.reset()
transactionsCountSinceLastLog = 0
if (isDebug) {
process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`)
transactionsCountSinceLastLog++
if (transactionsCountSinceLastLog >= batchSize) {
context.logger.debug(`${transactionsCountSinceLastLog} transactions added to blockchain in ${timeUsedBlockchain.string()}`)
timeUsedBlockchain.reset()
transactionsCountSinceLastLog = 0
}
} else {
transactionsCountSinceLastPrint++
if (transactionsCountSinceLastPrint >= 100) {
process.stdout.write(`successfully added to blockchain: ${transactionsCount}\r`)
transactionsCountSinceLastPrint = 0
}
}
}
process.stdout.write(`\n`)
context.logger.info(`Synced ${transactionsCount} transactions to blockchain in ${(timeUsedAll.seconds() / 60).toFixed(2)} minutes`)
context.logger.info(`Invalid contribution transactions: ${InvalidContributionTransactionSyncRole.allTransactionIds.length}`)
if (context.logger.isDebugEnabled()) {
context.logger.debug(InvalidContributionTransactionSyncRole.allTransactionIds.join(', '))
}
/*context.logger.info(`Double linked transactions: ${TransactionsSyncRole.doubleTransactionLinkCodes.length}`)
if (context.logger.isDebugEnabled()) {
context.logger.debug(TransactionsSyncRole.doubleTransactionLinkCodes.join(', '))
}*/
process.stdout.write(`successfully added to blockchain: ${transactionsCount}\n`)
context.logger.info(`Synced ${transactionsCount} transactions to blockchain in ${timeUsedAll.string()}`)
}

28
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/utils.ts
View File

@ -9,6 +9,19 @@ export function bytesToKbyte(bytes: number): string {
return (bytes / 1024).toFixed(0)
}
export function bytesString(bytes: number): string {
if (bytes > (1024 * 1024)) {
return `${bytesToMbyte(bytes)} MB`
} else if (bytes > 1024) {
return `${bytesToKbyte(bytes)} KB`
}
return `${bytes.toFixed(0)} Bytes`
}
export function toMysqlDateTime(date: Date): string {
return date.toISOString().slice(0, 23).replace('T', ' ')
}
export function calculateOneHashStep(hash: Buffer, data: Buffer): Buffer<ArrayBuffer> {
const outputHash = Buffer.alloc(crypto_generichash_KEYBYTES, 0)
crypto_generichash_batch(outputHash, [hash, data])
@ -17,15 +30,22 @@ export function calculateOneHashStep(hash: Buffer, data: Buffer): Buffer<ArrayBu
export const DECAY_START_TIME = new Date('2021-05-13T17:46:31Z')
export const SECONDS_PER_YEAR_GREGORIAN_CALENDER = 31556952.0
const FACTOR = new Decimal('0.99999997803504048973201202316767079413460520837376')
export function legacyDecayFormula(value: Decimal, seconds: number): Decimal {
// TODO why do we need to convert this here to a string to work properly?
// chatgpt: We convert to string here to avoid precision loss:
// .pow(seconds) can internally round the result, especially for large values of `seconds`.
// Using .toString() ensures full precision is preserved in the multiplication.
return value.mul(
new Decimal('0.99999997803504048973201202316767079413460520837376').pow(seconds).toString(),
)
return value.mul(FACTOR.pow(seconds).toString())
}
export function reverseLegacyDecay(result: Decimal, seconds: number): Decimal {
return result.div(FACTOR.pow(seconds).toString())
}
export function calculateEffectiveSeconds(holdOriginal: Decimal, holdCorrected: Decimal): Decimal {
return holdOriginal.div(holdCorrected).ln().div(FACTOR.ln());
}
export function legacyCalculateDecay(amount: Decimal, from: Date, to: Date): Decimal {
@ -34,7 +54,7 @@ export function legacyCalculateDecay(amount: Decimal, from: Date, to: Date): Dec
const startBlockMs = DECAY_START_TIME.getTime()
if (toMs < fromMs) {
throw new Error('calculateDecay: to < from, reverse decay calculation is invalid')
throw new Error(`calculateDecay: to (${to.toISOString()}) < from (${from.toISOString()}), reverse decay calculation is invalid`)
}
// decay started after end date; no decay

2
dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.5/valibot.schema.ts
View File

@ -54,6 +54,7 @@ export const creationTransactionDbSchema = v.pipe(v.object({
contributionDate: dateSchema,
confirmedAt: dateSchema,
confirmedByUser: userDbSchema,
transactionId: positiveNumberSchema,
}), v.custom((value: any) => {
if (value.user && value.confirmedByUser && value.user.gradidoId === value.confirmedByUser.gradidoId) {
throw new Error(`expect user to be different from confirmedByUser: ${JSON.stringify(value, null, 2)}`)
@ -77,6 +78,7 @@ export const transactionLinkDbSchema = v.object({
code: identifierSeedSchema,
createdAt: dateSchema,
validUntil: dateSchema,
holdAvailableAmount: gradidoAmountSchema,
redeemedAt: v.nullish(dateSchema),
deletedAt: v.nullish(dateSchema),
})