refactor database folder structure, add AppDatabase for creating connection

bun.lock

@@ -15,7 +15,7 @@
     },
     "admin": {
       "name": "admin",
-      "version": "2.5.2",
+      "version": "2.6.0",
       "dependencies": {
         "@iconify/json": "^2.2.228",
         "@popperjs/core": "^2.11.8",
@@ -84,7 +84,7 @@
     },
     "backend": {
       "name": "backend",
-      "version": "2.5.2",
+      "version": "2.6.0",
       "dependencies": {
         "cross-env": "^7.0.3",
         "email-templates": "^10.0.1",
@@ -170,7 +170,7 @@
     },
     "database": {
       "name": "database",
-      "version": "2.5.2",
+      "version": "2.6.0",
       "dependencies": {
         "@types/uuid": "^8.3.4",
         "cross-env": "^7.0.3",
@@ -178,6 +178,7 @@
         "dotenv": "^10.0.0",
         "esbuild": "^0.25.2",
         "geojson": "^0.5.0",
+        "log4js": "^6.9.1",
         "mysql2": "^2.3.0",
         "reflect-metadata": "^0.1.13",
         "ts-mysql-migrate": "^1.0.2",
@@ -196,7 +197,7 @@
     },
     "dht-node": {
       "name": "dht-node",
-      "version": "2.5.2",
+      "version": "2.6.0",
       "dependencies": {
         "cross-env": "^7.0.3",
         "dht-rpc": "6.18.1",
@@ -227,7 +228,7 @@
     },
     "federation": {
       "name": "federation",
-      "version": "2.5.2",
+      "version": "2.6.0",
       "dependencies": {
         "cross-env": "^7.0.3",
         "sodium-native": "^3.4.1",
@@ -277,7 +278,7 @@
     },
     "frontend": {
       "name": "frontend",
-      "version": "2.5.2",
+      "version": "2.6.0",
       "dependencies": {
         "@morev/vue-transitions": "^3.0.2",
         "@types/leaflet": "^1.9.12",

database/esbuild.config.ts

@@ -1,9 +1,9 @@
 import { build } from 'esbuild'
 import fs from 'node:fs'
-import { latestDbVersion } from './src/config/detectLastDBVersion'
+import { latestDbVersion } from '@/detectLastDBVersion'
 
 build({
-  entryPoints: ['entity/index.ts'],
+  entryPoints: ['src/index.ts'],
   bundle: true,
   target: 'node18.20.7',
   platform: 'node',

database/migration/clear.ts

@@ -1,5 +1,5 @@
 import { Connection } from 'mysql2/promise'
-import { CONFIG } from './config'
+import { CONFIG } from '@/config'
 import { connectToDatabaseServer } from './prepare'
 
 export async function truncateTables(connection: Connection) {

database/migration/index.ts

@@ -0,0 +1,102 @@
+import { CONFIG } from '@/config'
+import { DatabaseState, getDatabaseState } from './prepare'
+
+import path from 'node:path'
+import { createPool } from 'mysql'
+import { Migration } from 'ts-mysql-migrate'
+import { clearDatabase } from './clear'
+import { latestDbVersion } from '@/detectLastDBVersion'
+
+const run = async (command: string) => {
+  if (command === 'clear') {
+    if (CONFIG.NODE_ENV === 'production') {
+      throw new Error('Clearing database in production is not allowed')
+    }
+    await clearDatabase()
+    return
+  }
+  // Database actions not supported by our migration library
+  // await createDatabase()
+  const state = await getDatabaseState()
+  if (state === DatabaseState.NOT_CONNECTED) {
+    throw new Error(
+      `Database not connected, is database server running?
+      host: ${CONFIG.DB_HOST}
+      port: ${CONFIG.DB_PORT}
+      user: ${CONFIG.DB_USER}
+      password: ${CONFIG.DB_PASSWORD.slice(-2)}
+      database: ${CONFIG.DB_DATABASE}`,
+    )
+  }
+  if (state === DatabaseState.HIGHER_VERSION) {
+    throw new Error('Database version is higher than required, please switch to the correct branch')
+  }
+  if (state === DatabaseState.SAME_VERSION) {
+    if (command === 'up') {
+      // biome-ignore lint/suspicious/noConsole: no logger present
+      console.log('Database is up to date')
+      return
+    }
+  }
+  // Initialize Migrations
+  const pool = createPool({
+    host: CONFIG.DB_HOST,
+    port: CONFIG.DB_PORT,
+    user: CONFIG.DB_USER,
+    password: CONFIG.DB_PASSWORD,
+    database: CONFIG.DB_DATABASE,
+  })
+  const migration = new Migration({
+    conn: pool,
+    tableName: CONFIG.MIGRATIONS_TABLE,
+    silent: true,
+    dir: path.join(__dirname, 'migrations'),
+  })
+  await migration.initialize()
+
+  // Execute command
+  switch (command) {
+    case 'up':
+      await migration.up() // use for upgrade script
+      break
+    case 'down':
+      await migration.down() // use for downgrade script
+      break
+    case 'reset':
+      if (CONFIG.NODE_ENV === 'production') {
+        throw new Error('Resetting database in production is not allowed')
+      }
+      await migration.reset()
+      break
+    default:
+      throw new Error(`Unsupported command ${command}`)
+  }
+  if (command === 'reset') {
+    // biome-ignore lint/suspicious/noConsole: no logger present
+    console.log('Database was reset')
+  } else {
+    const currentDbVersion = await migration.getLastVersion()
+    // biome-ignore lint/suspicious/noConsole: no logger present
+    console.log(`Database was ${command} migrated to version: ${currentDbVersion.fileName}`)
+    if (latestDbVersion === currentDbVersion.fileName.split('.')[0]) {
+      // biome-ignore lint/suspicious/noConsole: no logger present
+      console.log('Database is now up to date')
+    } else {
+      // biome-ignore lint/suspicious/noConsole: no logger present
+      console.log('The latest database version is: ', latestDbVersion)
+    }
+  }
+
+  // Terminate connections gracefully
+  pool.end()
+}
+
+run(process.argv[2])
+  .catch((err) => {
+    // biome-ignore lint/suspicious/noConsole: no logger present
+    console.log(err)
+    process.exit(1)
+  })
+  .then(() => {
+    process.exit()
+  })

database/migration/migrations/0012-login_user_backups_unify_wordlist.ts

@@ -11,15 +11,15 @@ import path from 'path'
 const TARGET_MNEMONIC_TYPE = 2
 const PHRASE_WORD_COUNT = 24
 const WORDS_MNEMONIC_0 = fs
-  .readFileSync(path.resolve(__dirname, '../src/config/mnemonic.uncompressed_buffer18112.txt'))
+  .readFileSync(path.resolve(__dirname, '../../src/config/mnemonic.uncompressed_buffer18112.txt'))
   .toString()
   .split(',')
 const WORDS_MNEMONIC_1 = fs
-  .readFileSync(path.resolve(__dirname, '../src/config/mnemonic.uncompressed_buffer18113.txt'))
+  .readFileSync(path.resolve(__dirname, '../../src/config/mnemonic.uncompressed_buffer18113.txt'))
   .toString()
   .split(',')
 const WORDS_MNEMONIC_2 = fs
-  .readFileSync(path.resolve(__dirname, '../src/config/mnemonic.uncompressed_buffer13116.txt'))
+  .readFileSync(path.resolve(__dirname, '../../src/config/mnemonic.uncompressed_buffer13116.txt'))
   .toString()
   .split(',')
 const WORDS_MNEMONIC = [WORDS_MNEMONIC_0, WORDS_MNEMONIC_1, WORDS_MNEMONIC_2]

database/migration/prepare.ts

@@ -1,7 +1,7 @@
 import { Connection, ResultSetHeader, RowDataPacket, createConnection } from 'mysql2/promise'
 
-import { CONFIG } from './config'
-import { latestDbVersion } from './config/detectLastDBVersion'
+import { CONFIG } from '@/config'
+import { latestDbVersion } from '@/detectLastDBVersion'
 
 export enum DatabaseState {
   NOT_CONNECTED = 'NOT_CONNECTED',

database/package.json

@@ -3,7 +3,7 @@
   "version": "2.6.0",
   "description": "Gradido Database Tool to execute database migrations",
   "main": "./build/index.js",
-  "types": "./entity/index.ts",
+  "types": "./src/index.ts",
   "exports": {
     ".": {
       "import": "./build/index.js",
@@ -19,13 +19,13 @@
     "typecheck": "tsc --noEmit",
     "lint": "biome check --error-on-warnings .",
     "lint:fix": "biome check --error-on-warnings . --write",
-    "clear": "cross-env TZ=UTC tsx src/index.ts clear",
-    "up": "cross-env TZ=UTC tsx src/index.ts up",
-    "down": "cross-env TZ=UTC tsx src/index.ts down",
-    "reset": "cross-env TZ=UTC tsx src/index.ts reset",
-    "up:backend_test": "cross-env TZ=UTC DB_DATABASE=gradido_test_backend tsx src/index.ts up",
-    "up:federation_test": "cross-env TZ=UTC DB_DATABASE=gradido_test_federation tsx src/index.ts up",
-    "up:dht_test": "cross-env TZ=UTC DB_DATABASE=gradido_test_dht tsx src/index.ts up"
+    "clear": "cross-env TZ=UTC tsx migration/index.ts clear",
+    "up": "cross-env TZ=UTC tsx migration/index.ts up",
+    "down": "cross-env TZ=UTC tsx migration/index.ts down",
+    "reset": "cross-env TZ=UTC tsx migration/index.ts reset",
+    "up:backend_test": "cross-env TZ=UTC DB_DATABASE=gradido_test_backend tsx migration/index.ts up",
+    "up:federation_test": "cross-env TZ=UTC DB_DATABASE=gradido_test_federation tsx migration/index.ts up",
+    "up:dht_test": "cross-env TZ=UTC DB_DATABASE=gradido_test_dht tsx migration/index.ts up"
   },
   "devDependencies": {
     "@biomejs/biome": "1.9.4",
@@ -41,6 +41,7 @@
     "dotenv": "^10.0.0",
     "esbuild": "^0.25.2",
     "geojson": "^0.5.0",
+    "log4js": "^6.9.1",
     "mysql2": "^2.3.0",
     "reflect-metadata": "^0.1.13",
     "ts-mysql-migrate": "^1.0.2",

database/src/AppDatabase.ts

@@ -0,0 +1,103 @@
+import { DataSource as DBDataSource, FileLogger } from 'typeorm'
+import { entities, Migration } from '@/entity'
+
+import { CONFIG } from '@/config'
+import { logger } from '@/logging'
+import { latestDbVersion } from '.'
+
+export class AppDatabase {
+  private static instance: AppDatabase
+  private connection: DBDataSource | undefined
+
+  /**
+   * The Singleton's constructor should always be private to prevent direct
+   * construction calls with the `new` operator.
+   */
+  private constructor() {}
+
+  /**
+   * The static method that controls the access to the singleton instance.
+   *
+   * This implementation let you subclass the Singleton class while keeping
+   * just one instance of each subclass around.
+   */
+  public static getInstance(): AppDatabase {
+    if (!AppDatabase.instance) {
+      AppDatabase.instance = new AppDatabase()
+    }
+    return AppDatabase.instance
+  }
+
+  public getDataSource(): DBDataSource {
+    if (!this.connection) {
+      throw new Error('Connection not initialized')
+    }
+    return this.connection
+  }
+
+  // create database connection, initialize with automatic retry and check for correct database version
+  public async init(): Promise<void> {
+    if (this.connection?.isInitialized) return
+    
+    // log sql query only of enable by .env, this produce so much data it should be only used when really needed
+    const logging: boolean = CONFIG.TYPEORM_LOGGING_ACTIVE
+    this.connection = new DBDataSource({
+      type: 'mysql',
+      legacySpatialSupport: false,
+      host: CONFIG.DB_HOST,
+      port: CONFIG.DB_PORT,
+      username: CONFIG.DB_USER,
+      password: CONFIG.DB_PASSWORD,
+      database: CONFIG.DB_DATABASE,
+      entities,
+      synchronize: false,
+      logging,
+      logger: logging ? new FileLogger('all', {
+        logPath: CONFIG.TYPEORM_LOGGING_RELATIVE_PATH,
+      }) : undefined,
+      extra: {
+        charset: 'utf8mb4_unicode_ci',
+      },
+    })
+    // retry connection on failure some times to allow database to catch up
+    for (let attempt = 1; attempt <= CONFIG.DB_CONNECT_RETRY_COUNT; attempt++) {
+      try {
+        await this.connection.initialize()
+        if(this.connection.isInitialized) {
+          logger.info(`Database connection established on attempt ${attempt}`)
+          break
+        }
+      } catch (error) {
+        logger.warn(`Attempt ${attempt} failed to connect to DB:`, error)
+        await new Promise((resolve) => setTimeout(resolve, CONFIG.DB_CONNECT_RETRY_DELAY_MS))
+      }
+    }
+    if (!this.connection?.isInitialized) {
+      throw new Error('Could not connect to database')
+    }
+    // check for correct database version
+    await this.checkDBVersion()
+  }
+
+  public async close(): Promise<void> {
+    await this.connection?.destroy()
+  }
+  // ######################################
+  // private methods
+  // ######################################
+  private async checkDBVersion(): Promise<void> {
+    const [dbVersion] = await Migration.find({ order: { version: 'DESC' }, take: 1 })
+    if(!dbVersion) {
+      throw new Error('Could not find database version')
+    }
+    if (!dbVersion.fileName.startsWith(latestDbVersion)) {
+      throw new Error(
+        `Wrong database version detected - the backend requires '${latestDbVersion}' but found '${
+          dbVersion.fileName
+        }`,
+      )
+    }    
+  }
+}
+
+export const getDataSource = () => AppDatabase.getInstance().getDataSource()

database/src/config/index.ts

@@ -8,6 +8,7 @@ const constants = {
     EXPECTED: 'v1.2022-03-18',
     CURRENT: '',
   },
+  LOG4JS_CATEGORY_NAME: 'database'
 }
 
 const database = {
@@ -22,6 +23,8 @@ const database = {
   DB_USER: process.env.DB_USER ?? 'root',
   DB_PASSWORD: process.env.DB_PASSWORD ?? '',
   DB_DATABASE: process.env.DB_DATABASE ?? 'gradido_community',
+  TYPEORM_LOGGING_RELATIVE_PATH: process.env.TYPEORM_LOGGING_RELATIVE_PATH ?? 'typeorm.database.log',
+  TYPEORM_LOGGING_ACTIVE: process.env.TYPEORM_LOGGING_ACTIVE === 'true' || false,
 }
 
 const migrations = {

database/src/detectLastDBVersion.ts

@@ -5,7 +5,7 @@ import path from 'node:path'
 const DB_VERSION_PATTERN = /^(\d{4}-[a-z0-9-_]+)/
 
 // Define the paths to check
-const migrationsDir = path.join(__dirname, '..', '..', 'migrations')
+const migrationsDir = path.join(__dirname, '..', 'migration', 'migrations')
 
 // Helper function to get the highest version number from the directory
 function getLatestDbVersion(dir: string): string {

database/src/entity/Community.ts

@@ -9,7 +9,7 @@ import {
   PrimaryGeneratedColumn,
   UpdateDateColumn,
 } from 'typeorm'
-import { GeometryTransformer } from '../src/typeorm/GeometryTransformer'
+import { GeometryTransformer } from './transformer/GeometryTransformer'
 import { FederatedCommunity } from './FederatedCommunity'
 import { User } from './User'