speed up seeding

database/src/seeds/factory/creation.ts

@@ -1,9 +1,10 @@
-import { Contribution, User } from '../../entity'
+import { Contribution, Transaction, User } from '../../entity'
 import { Decimal } from 'decimal.js-light'
 import { CreationInterface } from '../creation/CreationInterface'
 import { ContributionType, ContributionStatus, TransactionTypeId } from '../../enum'
 import { findUserByIdentifier } from '../../queries'
 import { createTransaction } from './transaction'
+import { AppDatabase } from '../../AppDatabase'
 
 export function nMonthsBefore(date: Date, months = 1): string {
   return new Date(date.getFullYear(), date.getMonth() - months, 1).toISOString()
@@ -28,11 +29,42 @@ export async function creationFactory(
     if (!moderatorUser) {
       throw new Error('Moderator user not found')
     }
-    contribution = await confirmTransaction(contribution, moderatorUser)
+    await confirmTransaction(contribution, moderatorUser)
   }
   return contribution
 }
 
+export async function creationFactoryBulk(
+  creations: CreationInterface[],
+  userCreationIndexedByEmail: Map<string, User>,
+  moderatorUser: User,
+): Promise<Contribution[]> {
+  const lastTransaction = await Transaction.findOne({ order: { id: 'DESC' }, select: ['id'], where: {} })
+  let transactionId = lastTransaction ? lastTransaction.id + 1 : 1
+  const dbContributions: Contribution[] = []
+  const dbTransactions: Transaction[] = []
+
+  for (const creation of creations) {
+    const user = userCreationIndexedByEmail.get(creation.email)
+    if (!user) {
+      throw new Error(`User ${creation.email} not found`)
+    }
+    let contribution = await createContribution(creation, user, false)
+    if (creation.confirmed) {
+      const { contribution: _, transaction } = await confirmTransaction(contribution, moderatorUser, transactionId, false)
+      dbTransactions.push(transaction)
+      transactionId++
+    }
+    dbContributions.push(contribution)
+  }
+  const dataSource = AppDatabase.getInstance().getDataSource()
+  await dataSource.transaction(async (transaction) => {
+    await dataSource.getRepository(Contribution).insert(dbContributions)
+    await dataSource.getRepository(Transaction).insert(dbTransactions)    
+  })
+  return dbContributions
+}
+
 export async function createContribution(creation: CreationInterface, user: User, store: boolean = true): Promise<Contribution> {
   const contribution = new Contribution()
   contribution.user = user
@@ -47,7 +79,12 @@ export async function createContribution(creation: CreationInterface, user: User
   return store ? contribution.save() : contribution
 }
 
-export async function confirmTransaction(contribution: Contribution, moderatorUser: User, store: boolean = true): Promise<Contribution> {
+export async function confirmTransaction(
+  contribution: Contribution, 
+  moderatorUser: User, 
+  transactionId?: number, 
+  store: boolean = true
+): Promise<{ contribution: Contribution, transaction: Transaction }> {
   const now = new Date()
   const transaction = await createTransaction(
     contribution.amount,
@@ -57,7 +94,8 @@ export async function confirmTransaction(contribution: Contribution, moderatorUs
     TransactionTypeId.CREATION,
     now,
     contribution.contributionDate,
-    true,
+    transactionId,
+    store,
   )
   contribution.confirmedAt = now
   contribution.confirmedBy = moderatorUser.id
@@ -65,7 +103,12 @@ export async function confirmTransaction(contribution: Contribution, moderatorUs
   contribution.transaction = transaction
   contribution.contributionStatus = ContributionStatus.CONFIRMED
 
-  return store ? contribution.save() : contribution
+  if (store) {
+    await contribution.save()
+    await transaction.save()
+  }
+
+  return { contribution, transaction }
 }
     
 function getContributionDate(creation: CreationInterface): Date {

database/src/seeds/factory/transaction.ts

@@ -12,7 +12,8 @@ export async function createTransaction(
   linkedUser: User,
   type: TransactionTypeId,
   balanceDate: Date,
-  creationDate?: Date,  
+  creationDate?: Date,
+  id?: number,  
   store: boolean = true,
 ): Promise<Transaction> {
   
@@ -31,6 +32,9 @@ export async function createTransaction(
   newBalance = newBalance.add(amount.toString())
 
   const transaction = new Transaction()
+  if (id) {
+    transaction.id = id
+  }
   transaction.typeId = type
   transaction.memo = memo
   transaction.userId = user.id

database/src/seeds/factory/transactionLink.ts

@@ -1,9 +1,10 @@
 import { TransactionLinkInterface } from '../transactionLink/TransactionLinkInterface'
-import { TransactionLink } from '../../entity'
+import { TransactionLink, User } from '../../entity'
 import { Decimal } from 'decimal.js-light'
 import { findUserByIdentifier } from '../../queries'
 import { compoundInterest } from 'shared'
 import { randomBytes } from 'node:crypto'
+import { AppDatabase } from '../../AppDatabase'
 
 export async function transactionLinkFactory(
   transactionLinkData: TransactionLinkInterface,
@@ -19,6 +20,23 @@ export async function transactionLinkFactory(
   return createTransactionLink(transactionLinkData, userId)
 }
 
+export async function transactionLinkFactoryBulk(
+  transactionLinks: TransactionLinkInterface[], 
+  userCreationIndexedByEmail: Map<string, User>
+): Promise<TransactionLink[]> {
+  const dbTransactionLinks: TransactionLink[] = []
+  for (const transactionLink of transactionLinks) {
+    const user = userCreationIndexedByEmail.get(transactionLink.email)
+    if (!user) {
+      throw new Error(`User ${transactionLink.email} not found`)
+    }
+    dbTransactionLinks.push(await createTransactionLink(transactionLink, user.id, false))
+  }
+  const dataSource = AppDatabase.getInstance().getDataSource()
+  await dataSource.getRepository(TransactionLink).insert(dbTransactionLinks)
+  return dbTransactionLinks
+}
+
 export async function createTransactionLink(transactionLinkData: TransactionLinkInterface, userId: number, store: boolean = true): Promise<TransactionLink> {
   const holdAvailableAmount = compoundInterest(new Decimal(transactionLinkData.amount.toString()), CODE_VALID_DAYS_DURATION * 24 * 60 * 60)
   let createdAt = transactionLinkData.createdAt || new Date()

database/src/seeds/factory/user.ts

@@ -5,13 +5,55 @@ import { UserContactType, OptInType, PasswordEncryptionType } from 'shared'
 import { getHomeCommunity } from '../../queries/communities'
 import random from 'crypto-random-bigint'
 import { Community } from '../../entity'
+import { AppDatabase } from '../..'
 
 export async function userFactory(user: UserInterface, homeCommunity?: Community | null): Promise<User> {
-  let dbUserContact = new UserContact()
+  // TODO: improve with cascade 
-
+  let dbUser = await createUser(user, homeCommunity)
-  dbUserContact.email = user.email ?? ''
+  let dbUserContact = await createUserContact(user, dbUser.id)
-  dbUserContact.type = UserContactType.USER_CONTACT_EMAIL
+  dbUserContact = await dbUserContact.save()
+  dbUser.emailId = dbUserContact.id
+  dbUser.emailContact = dbUserContact
+  dbUser = await dbUser.save()
   
+  return dbUser
+}
+
+// only use in non-parallel environment (seeding for example)
+export async function userFactoryBulk(users: UserInterface[], homeCommunity?: Community | null): Promise<User[]> {
+  const dbUsers: User[] = []
+  const dbUserContacts: UserContact[] = []
+  const lastUser = await User.findOne({ order: { id: 'DESC' }, select: ['id'], where: {} })
+  const lastUserContact = await UserContact.findOne({ order: { id: 'DESC' }, select: ['id'], where: {} })
+  let userId = lastUser ? lastUser.id + 1 : 1
+  let emailId = lastUserContact ? lastUserContact.id + 1 : 1
+  for(const user of users) {
+    const dbUser = await createUser(user, homeCommunity, false)
+    dbUser.id = userId
+    dbUser.emailId = emailId
+
+    const dbUserContact = await createUserContact(user, userId, false)
+    dbUserContact.id = emailId
+    dbUserContact.userId = userId
+    dbUser.emailContact = dbUserContact
+
+    dbUsers.push(dbUser)
+    dbUserContacts.push(dbUserContact)
+    
+    userId++
+    emailId++
+  }
+  const dataSource = AppDatabase.getInstance().getDataSource()
+  await dataSource.transaction(async transaction => {
+    await Promise.all([
+      transaction.getRepository(User).insert(dbUsers),
+      transaction.getRepository(UserContact).insert(dbUserContacts)
+    ])
+  })
+  return dbUsers
+}
+
+export async function createUser(user: UserInterface, homeCommunity?: Community | null, store: boolean = true): Promise<User> {
   let dbUser = new User()
   dbUser.firstName = user.firstName ?? ''
   dbUser.lastName = user.lastName ?? ''
@@ -25,9 +67,6 @@ export async function userFactory(user: UserInterface, homeCommunity?: Community
   dbUser.gradidoID = v4()
 
   if (user.emailChecked) {
-    dbUserContact.emailVerificationCode = random(64).toString()
-    dbUserContact.emailOptInTypeId = OptInType.EMAIL_OPT_IN_REGISTER
-    dbUserContact.emailChecked = true
     dbUser.password = random(64)
     dbUser.passwordEncryptionType = PasswordEncryptionType.GRADIDO_ID
   }
@@ -38,12 +77,25 @@ export async function userFactory(user: UserInterface, homeCommunity?: Community
     dbUser.community = homeCommunity
     dbUser.communityUuid = homeCommunity.communityUuid!
   }
-  // TODO: improve with cascade 
-  dbUser = await dbUser.save()
-  dbUserContact.userId = dbUser.id
-  dbUserContact = await dbUserContact.save()
-  dbUser.emailId = dbUserContact.id
-  dbUser.emailContact = dbUserContact
 
-  return dbUser.save()
+  return store ? dbUser.save() : dbUser
+}
+
+export async function createUserContact(user: UserInterface, userId?: number, store: boolean = true): Promise<UserContact> {
+  let dbUserContact = new UserContact()
+
+  dbUserContact.email = user.email ?? ''
+  dbUserContact.type = UserContactType.USER_CONTACT_EMAIL
+
+  if (user.emailChecked) {
+    dbUserContact.emailVerificationCode = random(64).toString()
+    dbUserContact.emailOptInTypeId = OptInType.EMAIL_OPT_IN_REGISTER
+    dbUserContact.emailChecked = true
+  }
+
+  if (userId) {
+    dbUserContact.userId = userId
+  }
+
+  return store ? dbUserContact.save() : dbUserContact
 }

database/src/seeds/index.ts

@@ -1,102 +1,83 @@
 import { AppDatabase } from '../AppDatabase'
-import { clearDatabase } from '../../migration/clear'
 import { createCommunity } from './community'
-import { userFactory } from './factory/user'
+import { userFactoryBulk } from './factory/user'
 import { users } from './users'
-import { datatype, internet, name } from 'faker'
+import { internet, name } from 'faker'
-import { creationFactory } from './factory/creation'
+import { creationFactoryBulk } from './factory/creation'
 import { creations } from './creation'
-import { transactionLinkFactory } from './factory/transactionLink'
+import { transactionLinkFactoryBulk } from './factory/transactionLink'
 import { transactionLinks } from './transactionLink'
 import { contributionLinkFactory } from './factory/contributionLink'
 import { contributionLinks } from './contributionLink'
 import { User } from '../entity'
-import { TransactionLink } from '../entity'
+import { UserInterface } from './users/UserInterface'
-import { ContributionLink } from '../entity'
 
 const RANDOM_USER_COUNT = 100
 
 async function run() {
-  const now = new Date()
+  console.info('##seed## seeding started...')
-  // clear database, use mysql2 directly, not AppDatabase
+  
-  await clearDatabase()
-
   const db = AppDatabase.getInstance()
   await db.init()
+  await clearDatabase()
 
   // seed home community
-  const homeCommunity = await createCommunity(false)
+  const homeCommunity = await createCommunity(false)  
-  console.info('##seed## seeding home community successful...')
+  console.info(`##seed## seeding home community successful ...`)
 
   // seed standard users
-  // start creation of all users in parallel
   // put into map for later direct access
-  const userCreationIndexedByEmail = new Map<string, Promise<User>>()
+  const userCreationIndexedByEmail = new Map<string, User>()
-  for (const user of users) {
+  const defaultUsers = await userFactoryBulk(users, homeCommunity)
-    userCreationIndexedByEmail.set(user.email!, userFactory(user, homeCommunity))
+  for (const dbUser of defaultUsers) {
+    userCreationIndexedByEmail.set(dbUser.emailContact.email, dbUser)
   }
-  const defaultUsersPromise = Promise.all(userCreationIndexedByEmail.values()).then(() => {
+  console.info(`##seed## seeding all standard users successful ...`)  
-    // log message after all users are created
-    console.info('##seed## seeding all standard users successful...')
-  })
 
-  // seed 100 random users
+  // seed 100 random users  
-  // start creation of all random users in parallel
+  const randomUsers = new Array<UserInterface>(RANDOM_USER_COUNT)
-  const randomUsersCreation: Promise<User>[] = []
   for (let i = 0; i < RANDOM_USER_COUNT; i++) {
-    randomUsersCreation.push(userFactory({
+    randomUsers[i] = {
       firstName: name.firstName(),
       lastName: name.lastName(),
       email: internet.email(),
-      language: datatype.boolean() ? 'en' : 'de',
+      language: Math.random() < 0.5 ? 'en' : 'de',
-    }, homeCommunity))
+    }
   }
-  const randomUsersPromise = Promise.all(randomUsersCreation).then(() => {
+  await userFactoryBulk(randomUsers, homeCommunity)
-    // log message after all random users are created
+  console.info(`##seed## seeding ${RANDOM_USER_COUNT} random users successful ...`)
-    console.info(`##seed## seeding ${RANDOM_USER_COUNT} random users successful...`)
-  })
-
-  // create Contribution Links
-  // start creation of all contribution links in parallel
-  const contributionLinksPromises: Promise<ContributionLink>[] = []
-  for (const contributionLink of contributionLinks) {
-    contributionLinksPromises.push(contributionLinkFactory(contributionLink))
-  }
-  const contributionLinksPromise = Promise.all(contributionLinksPromises).then(() => {
-    // log message after all contribution links are created
-    console.info('##seed## seeding all contributionLinks successful...')
-  })
-
-  // create Transaction Links
-  // start creation of all transaction links in parallel
-  const transactionLinksPromises: Promise<TransactionLink>[] = []
-  for (const transactionLink of transactionLinks) {
-    const user = await userCreationIndexedByEmail.get(transactionLink.email)!
-    transactionLinksPromises.push(transactionLinkFactory(transactionLink, user.id))
-  }
-  const transactionLinksPromise = Promise.all(transactionLinksPromises).then(() => {
-    // log message after all transaction links are created
-    console.info('##seed## seeding all transactionLinks successful...')
-  })
 
   // create GDD serial, must be called one after another because seeding don't use semaphore
-  const moderatorUser = await userCreationIndexedByEmail.get('peter@lustig.de')!
+  const moderatorUser = userCreationIndexedByEmail.get('peter@lustig.de')!
-  for (const creation of creations) {
+  await creationFactoryBulk(creations, userCreationIndexedByEmail, moderatorUser)
-    const user = await userCreationIndexedByEmail.get(creation.email)!
+  console.info(`##seed## seeding all creations successful ...`)
-    await creationFactory(creation, user, moderatorUser)
-  }
 
-  // wait for all promises to be resolved
+  // create Contribution Links
-  await Promise.all([
+  for (const contributionLink of contributionLinks) {
-    defaultUsersPromise,
+    await contributionLinkFactory(contributionLink)
-    randomUsersPromise,
+  }
-    contributionLinksPromise,
+  console.info(`##seed## seeding all contributionLinks successful ...`)
-    transactionLinksPromise,
+
-  ])
+  // create Transaction Links
+  await transactionLinkFactoryBulk(transactionLinks, userCreationIndexedByEmail)
+  console.info(`##seed## seeding all transactionLinks successful ...`)
 
   await db.destroy()
-  const timeDiffSeconds = (new Date().getTime() - now.getTime()) / 1000
+  console.info(`##seed## seeding successful...`)
-  console.info(`##seed## seeding successful... after ${timeDiffSeconds} seconds`)
+}
+
+async function clearDatabase() {
+  await AppDatabase.getInstance().getDataSource().transaction(async trx => {
+    await trx.query(`SET FOREIGN_KEY_CHECKS = 0`)
+    await trx.query(`TRUNCATE TABLE contributions`)
+    await trx.query(`TRUNCATE TABLE contribution_links`)
+    await trx.query(`TRUNCATE TABLE users`)
+    await trx.query(`TRUNCATE TABLE user_contacts`)
+    await trx.query(`TRUNCATE TABLE transactions`)
+    await trx.query(`TRUNCATE TABLE transaction_links`)
+    await trx.query(`TRUNCATE TABLE communities`)
+    await trx.query(`SET FOREIGN_KEY_CHECKS = 1`)
+  })
 }
 
 run().catch((err) => {