From 15cd2a6e7fc983fd833e18639cbda1fd75c0bf78 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Sat, 29 Nov 2025 11:04:35 +0100 Subject: [PATCH] speed up seeding --- database/src/seeds/factory/creation.ts | 53 +++++++- database/src/seeds/factory/transaction.ts | 6 +- database/src/seeds/factory/transactionLink.ts | 20 ++- database/src/seeds/factory/user.ts | 80 +++++++++--- database/src/seeds/index.ts | 117 ++++++++---------- 5 files changed, 187 insertions(+), 89 deletions(-) diff --git a/database/src/seeds/factory/creation.ts b/database/src/seeds/factory/creation.ts index ddf5224de..618cbeebd 100644 --- a/database/src/seeds/factory/creation.ts +++ b/database/src/seeds/factory/creation.ts @@ -1,9 +1,10 @@ -import { Contribution, User } from '../../entity' +import { Contribution, Transaction, User } from '../../entity' import { Decimal } from 'decimal.js-light' import { CreationInterface } from '../creation/CreationInterface' import { ContributionType, ContributionStatus, TransactionTypeId } from '../../enum' import { findUserByIdentifier } from '../../queries' import { createTransaction } from './transaction' +import { AppDatabase } from '../../AppDatabase' export function nMonthsBefore(date: Date, months = 1): string { return new Date(date.getFullYear(), date.getMonth() - months, 1).toISOString() @@ -28,11 +29,42 @@ export async function creationFactory( if (!moderatorUser) { throw new Error('Moderator user not found') } - contribution = await confirmTransaction(contribution, moderatorUser) + await confirmTransaction(contribution, moderatorUser) } return contribution } +export async function creationFactoryBulk( + creations: CreationInterface[], + userCreationIndexedByEmail: Map, + moderatorUser: User, +): Promise { + const lastTransaction = await Transaction.findOne({ order: { id: 'DESC' }, select: ['id'], where: {} }) + let transactionId = lastTransaction ? lastTransaction.id + 1 : 1 + const dbContributions: Contribution[] = [] + const dbTransactions: Transaction[] = [] + + for (const creation of creations) { + const user = userCreationIndexedByEmail.get(creation.email) + if (!user) { + throw new Error(`User ${creation.email} not found`) + } + let contribution = await createContribution(creation, user, false) + if (creation.confirmed) { + const { contribution: _, transaction } = await confirmTransaction(contribution, moderatorUser, transactionId, false) + dbTransactions.push(transaction) + transactionId++ + } + dbContributions.push(contribution) + } + const dataSource = AppDatabase.getInstance().getDataSource() + await dataSource.transaction(async (transaction) => { + await dataSource.getRepository(Contribution).insert(dbContributions) + await dataSource.getRepository(Transaction).insert(dbTransactions) + }) + return dbContributions +} + export async function createContribution(creation: CreationInterface, user: User, store: boolean = true): Promise { const contribution = new Contribution() contribution.user = user @@ -47,7 +79,12 @@ export async function createContribution(creation: CreationInterface, user: User return store ? contribution.save() : contribution } -export async function confirmTransaction(contribution: Contribution, moderatorUser: User, store: boolean = true): Promise { +export async function confirmTransaction( + contribution: Contribution, + moderatorUser: User, + transactionId?: number, + store: boolean = true +): Promise<{ contribution: Contribution, transaction: Transaction }> { const now = new Date() const transaction = await createTransaction( contribution.amount, @@ -57,7 +94,8 @@ export async function confirmTransaction(contribution: Contribution, moderatorUs TransactionTypeId.CREATION, now, contribution.contributionDate, - true, + transactionId, + store, ) contribution.confirmedAt = now contribution.confirmedBy = moderatorUser.id @@ -65,7 +103,12 @@ export async function confirmTransaction(contribution: Contribution, moderatorUs contribution.transaction = transaction contribution.contributionStatus = ContributionStatus.CONFIRMED - return store ? contribution.save() : contribution + if (store) { + await contribution.save() + await transaction.save() + } + + return { contribution, transaction } } function getContributionDate(creation: CreationInterface): Date { diff --git a/database/src/seeds/factory/transaction.ts b/database/src/seeds/factory/transaction.ts index 1782c4f18..02a56723e 100644 --- a/database/src/seeds/factory/transaction.ts +++ b/database/src/seeds/factory/transaction.ts @@ -12,7 +12,8 @@ export async function createTransaction( linkedUser: User, type: TransactionTypeId, balanceDate: Date, - creationDate?: Date, + creationDate?: Date, + id?: number, store: boolean = true, ): Promise { @@ -31,6 +32,9 @@ export async function createTransaction( newBalance = newBalance.add(amount.toString()) const transaction = new Transaction() + if (id) { + transaction.id = id + } transaction.typeId = type transaction.memo = memo transaction.userId = user.id diff --git a/database/src/seeds/factory/transactionLink.ts b/database/src/seeds/factory/transactionLink.ts index 82155a908..956e33fb5 100644 --- a/database/src/seeds/factory/transactionLink.ts +++ b/database/src/seeds/factory/transactionLink.ts @@ -1,9 +1,10 @@ import { TransactionLinkInterface } from '../transactionLink/TransactionLinkInterface' -import { TransactionLink } from '../../entity' +import { TransactionLink, User } from '../../entity' import { Decimal } from 'decimal.js-light' import { findUserByIdentifier } from '../../queries' import { compoundInterest } from 'shared' import { randomBytes } from 'node:crypto' +import { AppDatabase } from '../../AppDatabase' export async function transactionLinkFactory( transactionLinkData: TransactionLinkInterface, @@ -19,6 +20,23 @@ export async function transactionLinkFactory( return createTransactionLink(transactionLinkData, userId) } +export async function transactionLinkFactoryBulk( + transactionLinks: TransactionLinkInterface[], + userCreationIndexedByEmail: Map +): Promise { + const dbTransactionLinks: TransactionLink[] = [] + for (const transactionLink of transactionLinks) { + const user = userCreationIndexedByEmail.get(transactionLink.email) + if (!user) { + throw new Error(`User ${transactionLink.email} not found`) + } + dbTransactionLinks.push(await createTransactionLink(transactionLink, user.id, false)) + } + const dataSource = AppDatabase.getInstance().getDataSource() + await dataSource.getRepository(TransactionLink).insert(dbTransactionLinks) + return dbTransactionLinks +} + export async function createTransactionLink(transactionLinkData: TransactionLinkInterface, userId: number, store: boolean = true): Promise { const holdAvailableAmount = compoundInterest(new Decimal(transactionLinkData.amount.toString()), CODE_VALID_DAYS_DURATION * 24 * 60 * 60) let createdAt = transactionLinkData.createdAt || new Date() diff --git a/database/src/seeds/factory/user.ts b/database/src/seeds/factory/user.ts index f4614587c..4a8a434d4 100644 --- a/database/src/seeds/factory/user.ts +++ b/database/src/seeds/factory/user.ts @@ -5,13 +5,55 @@ import { UserContactType, OptInType, PasswordEncryptionType } from 'shared' import { getHomeCommunity } from '../../queries/communities' import random from 'crypto-random-bigint' import { Community } from '../../entity' +import { AppDatabase } from '../..' export async function userFactory(user: UserInterface, homeCommunity?: Community | null): Promise { - let dbUserContact = new UserContact() - - dbUserContact.email = user.email ?? '' - dbUserContact.type = UserContactType.USER_CONTACT_EMAIL + // TODO: improve with cascade + let dbUser = await createUser(user, homeCommunity) + let dbUserContact = await createUserContact(user, dbUser.id) + dbUserContact = await dbUserContact.save() + dbUser.emailId = dbUserContact.id + dbUser.emailContact = dbUserContact + dbUser = await dbUser.save() + return dbUser +} + +// only use in non-parallel environment (seeding for example) +export async function userFactoryBulk(users: UserInterface[], homeCommunity?: Community | null): Promise { + const dbUsers: User[] = [] + const dbUserContacts: UserContact[] = [] + const lastUser = await User.findOne({ order: { id: 'DESC' }, select: ['id'], where: {} }) + const lastUserContact = await UserContact.findOne({ order: { id: 'DESC' }, select: ['id'], where: {} }) + let userId = lastUser ? lastUser.id + 1 : 1 + let emailId = lastUserContact ? lastUserContact.id + 1 : 1 + for(const user of users) { + const dbUser = await createUser(user, homeCommunity, false) + dbUser.id = userId + dbUser.emailId = emailId + + const dbUserContact = await createUserContact(user, userId, false) + dbUserContact.id = emailId + dbUserContact.userId = userId + dbUser.emailContact = dbUserContact + + dbUsers.push(dbUser) + dbUserContacts.push(dbUserContact) + + userId++ + emailId++ + } + const dataSource = AppDatabase.getInstance().getDataSource() + await dataSource.transaction(async transaction => { + await Promise.all([ + transaction.getRepository(User).insert(dbUsers), + transaction.getRepository(UserContact).insert(dbUserContacts) + ]) + }) + return dbUsers +} + +export async function createUser(user: UserInterface, homeCommunity?: Community | null, store: boolean = true): Promise { let dbUser = new User() dbUser.firstName = user.firstName ?? '' dbUser.lastName = user.lastName ?? '' @@ -25,9 +67,6 @@ export async function userFactory(user: UserInterface, homeCommunity?: Community dbUser.gradidoID = v4() if (user.emailChecked) { - dbUserContact.emailVerificationCode = random(64).toString() - dbUserContact.emailOptInTypeId = OptInType.EMAIL_OPT_IN_REGISTER - dbUserContact.emailChecked = true dbUser.password = random(64) dbUser.passwordEncryptionType = PasswordEncryptionType.GRADIDO_ID } @@ -38,12 +77,25 @@ export async function userFactory(user: UserInterface, homeCommunity?: Community dbUser.community = homeCommunity dbUser.communityUuid = homeCommunity.communityUuid! } - // TODO: improve with cascade - dbUser = await dbUser.save() - dbUserContact.userId = dbUser.id - dbUserContact = await dbUserContact.save() - dbUser.emailId = dbUserContact.id - dbUser.emailContact = dbUserContact - return dbUser.save() + return store ? dbUser.save() : dbUser +} + +export async function createUserContact(user: UserInterface, userId?: number, store: boolean = true): Promise { + let dbUserContact = new UserContact() + + dbUserContact.email = user.email ?? '' + dbUserContact.type = UserContactType.USER_CONTACT_EMAIL + + if (user.emailChecked) { + dbUserContact.emailVerificationCode = random(64).toString() + dbUserContact.emailOptInTypeId = OptInType.EMAIL_OPT_IN_REGISTER + dbUserContact.emailChecked = true + } + + if (userId) { + dbUserContact.userId = userId + } + + return store ? dbUserContact.save() : dbUserContact } \ No newline at end of file diff --git a/database/src/seeds/index.ts b/database/src/seeds/index.ts index c2a659e2f..5f085c5f8 100644 --- a/database/src/seeds/index.ts +++ b/database/src/seeds/index.ts @@ -1,102 +1,83 @@ import { AppDatabase } from '../AppDatabase' -import { clearDatabase } from '../../migration/clear' import { createCommunity } from './community' -import { userFactory } from './factory/user' +import { userFactoryBulk } from './factory/user' import { users } from './users' -import { datatype, internet, name } from 'faker' -import { creationFactory } from './factory/creation' +import { internet, name } from 'faker' +import { creationFactoryBulk } from './factory/creation' import { creations } from './creation' -import { transactionLinkFactory } from './factory/transactionLink' +import { transactionLinkFactoryBulk } from './factory/transactionLink' import { transactionLinks } from './transactionLink' import { contributionLinkFactory } from './factory/contributionLink' import { contributionLinks } from './contributionLink' import { User } from '../entity' -import { TransactionLink } from '../entity' -import { ContributionLink } from '../entity' +import { UserInterface } from './users/UserInterface' const RANDOM_USER_COUNT = 100 async function run() { - const now = new Date() - // clear database, use mysql2 directly, not AppDatabase - await clearDatabase() - + console.info('##seed## seeding started...') + const db = AppDatabase.getInstance() await db.init() + await clearDatabase() // seed home community - const homeCommunity = await createCommunity(false) - console.info('##seed## seeding home community successful...') + const homeCommunity = await createCommunity(false) + console.info(`##seed## seeding home community successful ...`) // seed standard users - // start creation of all users in parallel // put into map for later direct access - const userCreationIndexedByEmail = new Map>() - for (const user of users) { - userCreationIndexedByEmail.set(user.email!, userFactory(user, homeCommunity)) + const userCreationIndexedByEmail = new Map() + const defaultUsers = await userFactoryBulk(users, homeCommunity) + for (const dbUser of defaultUsers) { + userCreationIndexedByEmail.set(dbUser.emailContact.email, dbUser) } - const defaultUsersPromise = Promise.all(userCreationIndexedByEmail.values()).then(() => { - // log message after all users are created - console.info('##seed## seeding all standard users successful...') - }) + console.info(`##seed## seeding all standard users successful ...`) - // seed 100 random users - // start creation of all random users in parallel - const randomUsersCreation: Promise[] = [] + // seed 100 random users + const randomUsers = new Array(RANDOM_USER_COUNT) for (let i = 0; i < RANDOM_USER_COUNT; i++) { - randomUsersCreation.push(userFactory({ + randomUsers[i] = { firstName: name.firstName(), lastName: name.lastName(), email: internet.email(), - language: datatype.boolean() ? 'en' : 'de', - }, homeCommunity)) + language: Math.random() < 0.5 ? 'en' : 'de', + } } - const randomUsersPromise = Promise.all(randomUsersCreation).then(() => { - // log message after all random users are created - console.info(`##seed## seeding ${RANDOM_USER_COUNT} random users successful...`) - }) - - // create Contribution Links - // start creation of all contribution links in parallel - const contributionLinksPromises: Promise[] = [] - for (const contributionLink of contributionLinks) { - contributionLinksPromises.push(contributionLinkFactory(contributionLink)) - } - const contributionLinksPromise = Promise.all(contributionLinksPromises).then(() => { - // log message after all contribution links are created - console.info('##seed## seeding all contributionLinks successful...') - }) - - // create Transaction Links - // start creation of all transaction links in parallel - const transactionLinksPromises: Promise[] = [] - for (const transactionLink of transactionLinks) { - const user = await userCreationIndexedByEmail.get(transactionLink.email)! - transactionLinksPromises.push(transactionLinkFactory(transactionLink, user.id)) - } - const transactionLinksPromise = Promise.all(transactionLinksPromises).then(() => { - // log message after all transaction links are created - console.info('##seed## seeding all transactionLinks successful...') - }) + await userFactoryBulk(randomUsers, homeCommunity) + console.info(`##seed## seeding ${RANDOM_USER_COUNT} random users successful ...`) // create GDD serial, must be called one after another because seeding don't use semaphore - const moderatorUser = await userCreationIndexedByEmail.get('peter@lustig.de')! - for (const creation of creations) { - const user = await userCreationIndexedByEmail.get(creation.email)! - await creationFactory(creation, user, moderatorUser) - } + const moderatorUser = userCreationIndexedByEmail.get('peter@lustig.de')! + await creationFactoryBulk(creations, userCreationIndexedByEmail, moderatorUser) + console.info(`##seed## seeding all creations successful ...`) - // wait for all promises to be resolved - await Promise.all([ - defaultUsersPromise, - randomUsersPromise, - contributionLinksPromise, - transactionLinksPromise, - ]) + // create Contribution Links + for (const contributionLink of contributionLinks) { + await contributionLinkFactory(contributionLink) + } + console.info(`##seed## seeding all contributionLinks successful ...`) + + // create Transaction Links + await transactionLinkFactoryBulk(transactionLinks, userCreationIndexedByEmail) + console.info(`##seed## seeding all transactionLinks successful ...`) await db.destroy() - const timeDiffSeconds = (new Date().getTime() - now.getTime()) / 1000 - console.info(`##seed## seeding successful... after ${timeDiffSeconds} seconds`) + console.info(`##seed## seeding successful...`) +} + +async function clearDatabase() { + await AppDatabase.getInstance().getDataSource().transaction(async trx => { + await trx.query(`SET FOREIGN_KEY_CHECKS = 0`) + await trx.query(`TRUNCATE TABLE contributions`) + await trx.query(`TRUNCATE TABLE contribution_links`) + await trx.query(`TRUNCATE TABLE users`) + await trx.query(`TRUNCATE TABLE user_contacts`) + await trx.query(`TRUNCATE TABLE transactions`) + await trx.query(`TRUNCATE TABLE transaction_links`) + await trx.query(`TRUNCATE TABLE communities`) + await trx.query(`SET FOREIGN_KEY_CHECKS = 1`) + }) } run().catch((err) => {