extend log4js config generator, add own layouts

bun.lock

@@ -161,6 +161,8 @@
       "dependencies": {
         "esbuild": "^0.25.2",
         "joi": "^17.13.3",
+        "source-map-support": "^0.5.21",
+        "yoctocolors-cjs": "^2.1.2",
       },
       "devDependencies": {
         "@biomejs/biome": "1.9.4",
@@ -182,6 +184,7 @@
         "log4js": "^6.9.1",
         "mysql2": "^2.3.0",
         "reflect-metadata": "^0.1.13",
+        "source-map-support": "^0.5.21",
         "ts-mysql-migrate": "^1.0.2",
         "tsx": "^4.19.4",
         "typeorm": "^0.3.22",
@@ -207,6 +210,9 @@
       "devDependencies": {
         "@biomejs/biome": "1.9.4",
         "@hyperswarm/dht": "6.5.1",
+        "@swc/cli": "^0.7.3",
+        "@swc/core": "^1.11.24",
+        "@swc/helpers": "^0.5.17",
         "@types/dotenv": "^8.2.3",
         "@types/jest": "27.5.1",
         "@types/joi": "^17.2.3",
@@ -219,7 +225,9 @@
         "jest": "27.5.1",
         "joi": "^17.13.3",
         "log4js": "^6.9.1",
+        "nodemon": "^2.0.7",
         "prettier": "^2.8.8",
+        "source-map-support": "^0.5.21",
         "ts-jest": "27.1.4",
         "tsx": "^4.19.4",
         "typeorm": "^0.3.22",
@@ -3286,6 +3294,8 @@
 
     "yocto-queue": ["yocto-queue@0.1.0", "", {}, "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="],
 
+    "yoctocolors-cjs": ["yoctocolors-cjs@2.1.2", "", {}, "sha512-cYVsTjKl8b+FrnidjibDWskAv7UKOfcwaVZdp/it9n1s9fU3IkgDbhdIRKCW4JDsAlECJY0ytoVPT3sK6kideA=="],
+
     "yup": ["yup@1.6.1", "", { "dependencies": { "property-expr": "^2.0.5", "tiny-case": "^1.0.3", "toposort": "^2.0.2", "type-fest": "^2.19.0" } }, "sha512-JED8pB50qbA4FOkDol0bYF/p60qSEDQqBD0/qeIrUCG1KbPBIQ776fCUNb9ldbPcSTxA69g/47XTo4TqWiuXOA=="],
 
     "zen-observable": ["zen-observable@0.8.15", "", {}, "sha512-PQ2PC7R9rslx84ndNBZB/Dkv8V8fZEpk83RLgXtYd0fwUgEjseMn1Dgajh2x6S8QbZAFa9p2qVCEuYZNgve0dQ=="],

config-schema/package.json

@@ -15,7 +15,7 @@
   "license": "Apache-2.0",
   "private": true,
   "scripts": {
-    "build": "esbuild src/index.ts --outdir=build --platform=node --target=node18.20.7 --bundle --packages=external",
+    "build": "esbuild src/index.ts --outdir=build --sourcemap --platform=node --target=node18.20.7 --bundle --packages=external",
     "build:bun": "bun build src/index.ts --outdir=build --target=bun --packages=external",
     "typecheck": "tsc --noEmit",
     "lint": "biome check --error-on-warnings .",
@@ -28,7 +28,9 @@
   },
   "dependencies": {
     "esbuild": "^0.25.2",
-    "joi": "^17.13.3"
+    "joi": "^17.13.3",
+    "source-map-support": "^0.5.21",
+    "yoctocolors-cjs": "^2.1.2"
   },
   "engines": {
     "node": ">=18"

config-schema/src/commonSchema.ts

@@ -131,6 +131,13 @@ export const LOG4JS_CONFIG = Joi.string()
   .default('log4js-config.json')
   .required()
 
+export const LOG_FILES_BASE_PATH = Joi.string()
+  .pattern(/^[a-zA-Z0-9-_\/\.]+$/)
+  .message('LOG_FILES_BASE_PATH must be a valid folder name, relative or absolute')
+  .description('log folder name for module log files')
+  .default('../logs/backend')
+  .optional()
+
 export const LOGIN_APP_SECRET = Joi.string()
   .pattern(/^[a-fA-F0-9]+$/)
   .message('need to be valid hex')

config-schema/src/index.ts

@@ -1,3 +1,5 @@
+import 'source-map-support/register'
 export * from './commonSchema'
 export { DatabaseConfigSchema } from './DatabaseConfigSchema'
 export { validate } from './validate'
+export { createLog4jsConfig, type Category, initLogger, defaultCategory } from './log4js-config'

config-schema/src/log4js-config/appenders.ts

@@ -0,0 +1,77 @@
+import type {
+  Appender,
+  DateFileAppender,
+  LogLevelFilterAppender,
+  StandardOutputAppender,
+} from 'log4js'
+import { CustomFileAppender } from './types'
+
+const fileAppenderTemplate = {
+  type: 'dateFile' as const,
+  pattern: 'yyyy-MM-dd',
+  compress: true,
+  keepFileExt: true,
+  fileNameSep: '_',
+  numBackups: 30,
+}
+
+const defaultAppenders = {
+  errorFile: {
+    type: 'dateFile' as const,
+    filename: 'errors.log',
+    pattern: 'yyyy-MM-dd',
+    layout: { type: 'coloredContext' as const, withStack: true },
+    compress: true,
+    keepFileExt: true,
+    fileNameSep: '_',
+    numBackups: 30,
+  } as DateFileAppender,
+  errors: {
+    type: 'logLevelFilter' as const,
+    level: 'error' as const,
+    appender: 'errorFile' as const,
+  } as LogLevelFilterAppender,
+  out: {
+    type: 'stdout' as const,
+    layout: { type: 'coloredContext' as const, withStack: 'error' },
+  } as StandardOutputAppender,
+}
+
+/**
+ * Creates the appender configuration for log4js.
+ *
+ * @param {CustomFileAppender[]} fileAppenders
+ *   the list of custom file appenders to add to the standard
+ *   appenders.
+ * @param {string} [basePath]
+ *   the base path for all log files.
+ * @param {boolean} [stacktraceOnStdout=false]
+ *   whether to show the stacktrace on the standard output
+ *   appender.
+ * @returns {Object<string, Appender>}
+ *   the appender configuration as a map
+ */
+export function createAppenderConfig(
+  fileAppenders: CustomFileAppender[],
+  basePath?: string,
+): { [name: string]: Appender } {
+  if (basePath) {
+    defaultAppenders.errorFile.filename = `${basePath}/errors`
+  }
+  const customAppender: { [name: string]: Appender } = { ...defaultAppenders }
+
+  fileAppenders.forEach((appender) => {
+    const filename = appender.filename ?? `${appender.name}.log`
+    const dateFile: DateFileAppender = {
+      ...fileAppenderTemplate,
+      filename: basePath ? `${basePath}/${filename}` : filename,
+    }
+    dateFile.layout = { 
+      type: 'coloredContext', 
+      withStack: appender.withStack,
+      withFile: appender.withFile,
+    }
+    customAppender[appender.name] = dateFile
+  })
+  return customAppender
+}

config-schema/src/log4js-config/coloredContext.ts

@@ -0,0 +1,76 @@
+import { LoggingEvent, Level } from 'log4js'
+import colors from 'yoctocolors-cjs'  
+import { LogLevel } from './types'
+
+export type coloredContextLayoutConfig = {
+  withStack?: LogLevel | boolean
+  withFile?: LogLevel | boolean
+}
+
+
+function colorize(str: string, level: Level): string {
+  switch(level.colour) {
+    case 'white': return colors.white(str)
+    case 'grey': return colors.gray(str)
+    case 'black': return colors.black(str)
+    case 'blue': return colors.blue(str)
+    case 'cyan': return colors.cyan(str)
+    case 'green': return colors.green(str)
+    case 'magenta': return colors.magenta(str)
+    case 'red': return colors.redBright(str)
+    case 'yellow': return colors.yellow(str)
+    default: return colors.gray(str)
+  }
+}
+
+// distinguish between objects with valid toString function (for examples classes derived from AbstractLoggingView) and other objects
+function composeDataString(data: (string | Object)[]): string {
+  return data.map((data) => {
+    // if it is a object and his toString function return only garbage
+    if (typeof data === 'object' && data.toString() === '[object Object]') {
+      return JSON.stringify(data)
+    }
+    return data.toString()
+  }).join(' ')
+}
+
+// automatic detect context objects and list them in logfmt style
+function composeContextString(data: Object): string {
+  return Object.entries(data).map(([key, value]) => {
+    return `${key}=${value} `
+  }).join(' ').trimEnd()
+}
+
+// check if option is enabled, either if option is them self a boolean or a valid log level and <= eventLogLevel
+function isEnabledByLogLevel(eventLogLevel: Level, targetLogLevel?: LogLevel | boolean): boolean {
+  if (!targetLogLevel) {
+    return false
+  }
+  if (typeof targetLogLevel === 'boolean') {
+    return targetLogLevel
+  }
+  return eventLogLevel.isGreaterThanOrEqualTo(targetLogLevel)
+}
+
+export function createColoredContextLayout(config: coloredContextLayoutConfig) {
+  return (logEvent: LoggingEvent) => {
+    const result: string[] = []
+    result.push(colorize(
+      `[${logEvent.startTime.toISOString()}] [${logEvent.level}] ${logEvent.categoryName} -`,
+      logEvent.level
+    ))
+    if (Object.keys(logEvent.context).length > 0) {
+      result.push(composeContextString(logEvent.context))
+    } 
+    result.push(composeDataString(logEvent.data))
+    const showCallstack = logEvent.callStack && isEnabledByLogLevel(logEvent.level, config.withStack)
+    if (!showCallstack && isEnabledByLogLevel(logEvent.level, config.withFile)) {
+      result.push(`\n    at ${logEvent.fileName}:${logEvent.lineNumber}`)
+    }
+    if (showCallstack) {
+      result.push(`\n${logEvent.callStack}`)
+    }
+    return result.join(' ')
+  } 
+}
+  

config-schema/src/log4js-config/index.ts

@@ -0,0 +1,79 @@
+import { readFileSync, writeFileSync } from 'node:fs'
+import { addLayout, Configuration, LoggingEvent, configure } from 'log4js'
+import { createAppenderConfig } from './appenders'
+import { Category, CustomFileAppender, LogLevel, defaultCategory } from './types'
+import { createColoredContextLayout } from './coloredContext'
+
+export { Category, LogLevel, defaultCategory }
+
+/**
+ * Creates the log4js configuration.
+ *
+ * @param {Category[]} categories - the categories to add to the configuration
+ * @param {string} [basePath] - the base path for log files
+ * @returns {Configuration} the log4js configuration
+ */
+
+addLayout("json", function() {
+  return function (logEvent: LoggingEvent) {
+    return JSON.stringify(logEvent)
+  }
+})
+
+addLayout("coloredContext", createColoredContextLayout)
+
+export function createLog4jsConfig(categories: Category[], basePath?: string): Configuration {
+  const customFileAppenders: CustomFileAppender[] = []
+  const result: Configuration = {
+    appenders: {},
+    categories: {},
+  }
+
+  categories.forEach((category: Category) => {
+    customFileAppenders.push({
+      name: category.name,
+      filename: category.filename,
+      withFile: true,
+      withStack: 'error',
+    })
+    // needed by log4js, show all error message accidentally without (proper) Category
+    result.categories['default'] = {
+      level: 'debug',
+      appenders: [
+        'out',
+        'errors',
+      ],
+      enableCallStack: true,
+    }
+    const appenders = [category.name, 'out']
+    if (category.additionalErrorsFile) {
+      appenders.push('errors')
+    }
+    result.categories[category.name] = {
+      level: category.level,
+      appenders,
+      enableCallStack: true,
+    }
+  })
+
+  result.appenders = createAppenderConfig(customFileAppenders, basePath)
+  return result
+}
+
+/**
+ * Initializes the logger.
+ *
+ * @param {Category[]} categories - the categories to add to the configuration
+ * @param {string} logFilesPath - the base path for log files
+ * @param {string} [log4jsConfigFileName] - the name of the log4js config file
+ */
+export function initLogger(categories: Category[], logFilesPath: string, log4jsConfigFileName: string = 'log4js-config.json'): void {
+   // if not log4js config file exists, create a default one
+   try {
+    configure(JSON.parse(readFileSync(log4jsConfigFileName, 'utf-8')))
+  } catch(_e) {
+    const options = createLog4jsConfig(categories, logFilesPath)
+    writeFileSync(log4jsConfigFileName, JSON.stringify(options, null, 2), {encoding: 'utf-8'})
+    configure(options)
+  }
+}

config-schema/src/log4js-config/types/Category.ts

@@ -0,0 +1,28 @@
+import { LogLevel } from './LogLevel'
+
+/**
+ * Configuration for a log4js category.
+ *
+ * @property {string} name - The name of the category.
+ * @property {string} [filename] - The filename for the category, use name if not set.
+ * @property {boolean} [stdout] - Whether to log to stdout.
+ * @property {boolean} [additionalErrorsFile] - Whether to log errors additional to the default error file.
+ * @property {LogLevel} level - The logging level.
+ */
+export type Category = {
+  name: string
+  filename?: string
+  stdout?: boolean
+  additionalErrorsFile?: boolean
+  level: LogLevel
+}
+
+export function defaultCategory(name: string, level: LogLevel): Category {
+  return {
+    name,
+    level,
+    stdout: true,
+    additionalErrorsFile: true,
+  }
+}
+  

config-schema/src/log4js-config/types/CustomFileAppender.ts

@@ -0,0 +1,31 @@
+import { LogLevel } from './LogLevel'
+/**
+ * use default dateFile Template for custom file appenders
+ *
+ * @example use name for key and filename, add .log to name
+ * ```
+ * const appenderConfig = createAppenderConfig([
+ *   { name: 'info' },
+ * ])
+ * ```
+ *
+ * @example if log file should contain the stacktrace
+ * ```
+ * const appenderConfig = createAppenderConfig([
+ *   { name: 'warn', filename: 'warn.log', withStack: true },
+ * ])
+ * ```
+ * 
+ * @example if log file should contain the stacktrace only from log level debug and higher
+ * ```
+ * const appenderConfig = createAppenderConfig([
+ *   { name: 'warn', filename: 'warn.log', withStack: 'debug' },
+ * ])
+ * ```
+ */
+export type CustomFileAppender = {
+  name: string
+  filename?: string
+  withStack?: LogLevel | boolean // with stack if boolean or from log level on or above
+  withFile?: LogLevel | boolean // with filename and line if boolean or from log level on or above
+}

config-schema/src/log4js-config/types/LogLevel.ts

@@ -0,0 +1,15 @@
+import { z } from 'zod'
+
+export const LOG_LEVEL = z.enum([
+  'all',
+  'mark',
+  'trace',
+  'debug',
+  'info',
+  'warn',
+  'error',
+  'fatal',
+  'off',
+])
+
+export type LogLevel = z.infer<typeof LOG_LEVEL>

config-schema/src/log4js-config/types/index.ts

@@ -0,0 +1,3 @@
+export * from './Category'
+export * from './CustomFileAppender'
+export * from './LogLevel'

yarn.lock

@@ -10594,7 +10594,7 @@ sort-keys@^1.0.0:
   resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.2.1.tgz#1ce5650fddd87abc099eda37dcff024c2667ae46"
   integrity sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==
 
-source-map-support@^0.5.6, source-map-support@~0.5.20:
+source-map-support@^0.5.21, source-map-support@^0.5.6, source-map-support@~0.5.20:
   version "0.5.21"
   resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f"
   integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==
@@ -12660,6 +12660,11 @@ yocto-queue@^0.1.0:
   resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b"
   integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==
 
+yoctocolors-cjs@^2.1.2:
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/yoctocolors-cjs/-/yoctocolors-cjs-2.1.2.tgz#f4b905a840a37506813a7acaa28febe97767a242"
+  integrity sha512-cYVsTjKl8b+FrnidjibDWskAv7UKOfcwaVZdp/it9n1s9fU3IkgDbhdIRKCW4JDsAlECJY0ytoVPT3sK6kideA==
+
 yup@^1.6.1:
   version "1.6.1"
   resolved "https://registry.yarnpkg.com/yup/-/yup-1.6.1.tgz#8defcff9daaf9feac178029c0e13b616563ada4b"