diff --git a/.dockerignore b/.dockerignore index 01c8b1b48..8a354191d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,5 @@ **/node_modules **/build -**/coverage \ No newline at end of file +**/coverage +.git +**/.turbo \ No newline at end of file diff --git a/.github/workflows/test_admin_interface.yml b/.github/workflows/test_admin_interface.yml index a2d1fe5ef..b9d62a5ad 100644 --- a/.github/workflows/test_admin_interface.yml +++ b/.github/workflows/test_admin_interface.yml @@ -56,12 +56,14 @@ jobs: uses: oven-sh/setup-bun@v2 - name: install dependencies - run: bun install --frozen-lockfile + run: | + bun install --filter admin --frozen-lockfile + bun install --global turbo@^2 - name: Admin Interface | Unit tests id: test run: | - bun turbo admin#test admin#lint + turbo admin#test admin#lint echo "success=$([ $? -eq 0 ] && echo true || echo false)" >> $GITHUB_OUTPUT lint: diff --git a/.github/workflows/test_backend.yml b/.github/workflows/test_backend.yml index 5eaaf58d1..f177668f8 100644 --- a/.github/workflows/test_backend.yml +++ b/.github/workflows/test_backend.yml @@ -56,7 +56,7 @@ jobs: uses: oven-sh/setup-bun@v2 - name: install dependencies - run: bun install --frozen-lockfile + run: bun install --filter backend --frozen-lockfile - name: Wait for MariaDB to be ready run: until nc -z localhost 3306; do echo waiting for db; sleep 1; done; @@ -82,10 +82,12 @@ jobs: uses: oven-sh/setup-bun@v2 - name: install dependencies - run: bun install --frozen-lockfile + run: | + bun install --filter backend --frozen-lockfile + bun install --global turbo@^2 - name: Backend | Typecheck - run: bun turbo backend#typecheck backend#build + run: turbo backend#typecheck backend#build locales: if: needs.files-changed.outputs.backend == 'true' diff --git a/.github/workflows/test_config.yml b/.github/workflows/test_config.yml index 8a681ab4c..5ba0832c6 100644 --- a/.github/workflows/test_config.yml +++ b/.github/workflows/test_config.yml @@ -29,15 +29,12 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - - name: Install turbo - run: yarn global add turbo@^2 - - - name: Prune config with turbos help - run: turbo prune config-schema + - name: install bun + uses: oven-sh/setup-bun@v2 - name: install dependencies - run: cd out && yarn install --frozen-lockfile --production=false + run: bun install --filter config-schema --frozen-lockfile - name: typecheck - run: cd out && turbo typecheck + run: cd config-schema && yarn typecheck diff --git a/.github/workflows/test_database.yml b/.github/workflows/test_database.yml index 4b1c4886c..5a448e434 100644 --- a/.github/workflows/test_database.yml +++ b/.github/workflows/test_database.yml @@ -45,23 +45,22 @@ jobs: - name: Database | docker-compose run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach mariadb - - name: Install turbo - run: yarn global add turbo@^2 - - - name: Prune database with turbos help - run: turbo prune database + - name: install bun + uses: oven-sh/setup-bun@v2 - name: install dependencies - run: cd out && yarn install --frozen-lockfile --production=false + run: | + bun install --filter database --frozen-lockfile + bun install --global turbo@^2 - name: Wait for MariaDB to be ready run: docker run --rm --network gradido_internal-net busybox sh -c 'until nc -z mariadb 3306; do echo waiting for db; sleep 1; done;' - name: Database | up - run: cd out && turbo up + run: turbo up - name: Database | reset - run: cd out && turbo reset + run: turbo reset lint: if: needs.files-changed.outputs.database == 'true' @@ -72,14 +71,13 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - - name: Install turbo - run: yarn global add turbo@^2 - - - name: Prune database with turbos help - run: turbo prune database + - name: install bun + uses: oven-sh/setup-bun@v2 - name: install dependencies - run: cd out && yarn install --frozen-lockfile --production=false + run: | + bun install --filter database --frozen-lockfile + bun install --global turbo@^2 - - name: Database - run: cd out && turbo build typecheck \ No newline at end of file + - name: Database | build & typecheck + run: turbo database#build database#typecheck \ No newline at end of file diff --git a/.github/workflows/test_dht_node.yml b/.github/workflows/test_dht_node.yml index 95d0d5af4..8968609eb 100644 --- a/.github/workflows/test_dht_node.yml +++ b/.github/workflows/test_dht_node.yml @@ -46,23 +46,22 @@ jobs: node-version: '18.20.7' - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: docker-compose mariadb run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach --no-deps mariadb - - name: Install turbo - run: yarn global add turbo@^2 - - - name: Prune dht-node with turbos help - run: turbo prune dht-node + - name: install bun + uses: oven-sh/setup-bun@v2 - name: install dependencies - run: cd out && yarn install --frozen-lockfile --production=false + run: | + bun install --filter dht-node --frozen-lockfile + bun install --global turbo@^2 - name: Wait for MariaDB to be ready run: until nc -z localhost 3306; do echo waiting for db; sleep 1; done; - name: run unit test & build & typecheck - run: cd out && turbo dht-node#test dht-node#build dht-node#typecheck + run: turbo dht-node#test dht-node#build dht-node#typecheck diff --git a/.github/workflows/test_federation.yml b/.github/workflows/test_federation.yml index 25f8eaab5..d139b9d27 100644 --- a/.github/workflows/test_federation.yml +++ b/.github/workflows/test_federation.yml @@ -51,18 +51,17 @@ jobs: - name: docker-compose mariadb run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach --no-deps mariadb - - name: Install turbo - run: yarn global add turbo@^2 - - - name: Prune federation with turbos help - run: turbo prune federation + - name: install bun + uses: oven-sh/setup-bun@v2 - name: install dependencies - run: cd out && yarn install --frozen-lockfile --production=false + run: | + bun install --filter federation --frozen-lockfile + bun install --global turbo@^2 - name: Wait for MariaDB to be ready run: until nc -z localhost 3306; do echo waiting for db; sleep 1; done; - name: Federation | Unit tests id: test - run: cd out && turbo federation#test federation#build federation#typecheck \ No newline at end of file + run: turbo federation#test federation#build federation#typecheck \ No newline at end of file diff --git a/.github/workflows/test_frontend.yml b/.github/workflows/test_frontend.yml index 4853ec0d7..2f8931c5d 100644 --- a/.github/workflows/test_frontend.yml +++ b/.github/workflows/test_frontend.yml @@ -54,10 +54,10 @@ jobs: uses: oven-sh/setup-bun@v2 - name: install dependencies - run: bun install --frozen-lockfile + run: bun install --filter frontend --frozen-lockfile - name: Frontend | Unit tests - run: bun turbo frontend#test + run: cd frontend && yarn test lint: if: needs.files-changed.outputs.config == 'true' || needs.files-changed.outputs.frontend == 'true' @@ -79,12 +79,14 @@ jobs: uses: oven-sh/setup-bun@v2 - name: install dependencies - run: bun install --frozen-lockfile + run: | + bun install --filter frontend --frozen-lockfile + bun install --global turbo@^2 - name: Frontend | Lint id: lint run: | - bun turbo frontend#lint + turbo frontend#lint echo "success=$([ $? -eq 0 ] && echo true || echo false)" >> $GITHUB_OUTPUT stylelint: diff --git a/Dockerfile b/Dockerfile index 5d714b472..dda955621 100644 --- a/Dockerfile +++ b/Dockerfile @@ -147,7 +147,7 @@ COPY --chown=app:app --from=build /app/backend ./backend COPY --chown=app:app --from=build /app/frontend ./frontend COPY --chown=app:app --from=build /app/admin ./admin COPY --chown=app:app --from=build /app/database ./database -COPY --chown=app:app --from=build /app/config ./config +COPY --chown=app:app --from=build /app/config-schema ./config-schema COPY --chown=app:app --from=build /app/federation ./federation COPY --chown=app:app --from=build /app/dht-node ./dht-node @@ -158,4 +158,39 @@ EXPOSE ${FRONTEND_MODULE_PORT} EXPOSE ${ADMIN_MODULE_PORT} # Command to start -CMD ["turbo", "start", "--env-mode=loose"] \ No newline at end of file +CMD ["turbo", "start", "--env-mode=loose"] + +################################################################################## +# FINAL PRODUCTION IMAGE ######################################################### +################################################################################## +FROM node:18.20.7-alpine3.21 as production-slim + +ENV TURBO_CACHE_DIR=/tmp/turbo +ENV DOCKER_WORKDIR="/app" +ENV NODE_ENV="production" +ENV DB_HOST=mariadb +WORKDIR ${DOCKER_WORKDIR} + +# Ports exposen +EXPOSE ${BACKEND_PORT} +EXPOSE ${FEDERATION_PORT} +EXPOSE ${FRONTEND_MODULE_PORT} +EXPOSE ${ADMIN_MODULE_PORT} + +# Copy only the build artifacts from the previous build stage +COPY --chown=app:app --from=build /app/backend/build ./backend/build +COPY --chown=app:app --from=build /app/backend/locales ./backend/locales +COPY --chown=app:app --from=build /app/backend/log4js-config.json ./backend/log4js-config.json + +COPY --chown=app:app --from=build /app/dht-node/build ./dht-node/build +COPY --chown=app:app --from=build /app/dht-node/log4js-config.json ./dht-node/log4js-config.json + +COPY --chown=app:app --from=build /app/federation/build ./federation/build +COPY --chown=app:app --from=build /app/federation/log4js-config.json ./federation/log4js-config.json + +COPY --chown=app:app --from=build /app/frontend/build ./frontend +COPY --chown=app:app --from=build /app/admin/build ./admin + +RUN yarn global add udx-native@1.5.3 sodium-native@4.0.0 + +CMD ["turbo", "start", "--env-mode=loose"] diff --git a/admin/Dockerfile b/admin/Dockerfile index 8f0a9b8ea..0fa88c333 100644 --- a/admin/Dockerfile +++ b/admin/Dockerfile @@ -17,7 +17,8 @@ ENV BUILD_COMMIT=${BUILD_COMMIT} ARG BUILD_COMMIT_SHORT ENV BUILD_COMMIT_SHORT=${BUILD_COMMIT_SHORT} ## SET NODE_ENV -ARG NODE_ENV="production" +ARG NODE_ENV=production +ENV NODE_ENV=${NODE_ENV} ## App relevant Envs ENV PORT="8080" ## Timezone @@ -40,9 +41,6 @@ LABEL maintainer="support@ogradido.net" ## install: git #RUN apk --no-cache add git # RUN bun add --global yarn@1.22.20 -RUN yarn global add bun -# Add bun's global bin directory to PATH -ENV PATH="/root/.bun/bin:${PATH}" # Settings ## Expose Container Port @@ -52,57 +50,49 @@ EXPOSE ${PORT} RUN mkdir -p ${DOCKER_WORKDIR} WORKDIR ${DOCKER_WORKDIR} +################################################################################## +# BUN ############################################################################ +################################################################################## +FROM base as bun-base + +RUN apk update && apk add --no-cache curl tar bash +RUN curl -fsSL https://bun.sh/install | bash +# Add bun's global bin directory to PATH +ENV PATH="/root/.bun/bin:${PATH}" + ################################################################################## # Development #################################################################### ################################################################################## -FROM base AS development +FROM bun-base AS development # used for getting git commit hash direct from .git RUN apk update && apk add --no-cache git # Run command -CMD /bin/sh -c "bun install --frozen-lockfile && bun turbo admin#dev --env-mode=loose" +CMD /bin/sh -c "bun install --filter admin --no-cache --frozen-lockfile \ + && bun install --global --no-cache --no-save turbo@^2 \ + && turbo admin#dev --env-mode=loose" + ################################################################################## -# Base with turbo ################################################################ +# Build ########################################################################## ################################################################################## -FROM base as turbo-base - -RUN apk update && apk add --no-cache libc6-compat \ - && bun add --global turbo@^2 \ - && rm -rf /tmp/* ~/.cache node_modules/.cache - -################################################################################## -# BUILDER (create partly monorepo only with data needed by admin) ################ -################################################################################## -FROM turbo-base as builder +FROM bun-base as build COPY --chown=app:app . . -RUN turbo prune admin --docker +RUN bun install --filter admin --no-cache --frozen-lockfile \ +&& bun install --global turbo@^2 -################################################################################## -# INSTALLER (create production image) ############################################ -################################################################################## -FROM turbo-base AS installer - -# First install the dependencies (as they change less often) -COPY --chown=app:app --from=builder /app/out/json/ . -RUN yarn install --frozen-lockfile --production=false \ - && rm -rf /tmp/* ~/.cache node_modules/.cache \ - && yarn cache clean - -# Build the project -COPY --chown=app:app --from=builder /app/out/full/ . # TODO: get the actually git commit hash into docker -ENV BUILD_COMMIT='deadbeefdeadbeefdeadbeefdeadbeefdeadbeef' -ENV BUILD_COMMIT_SHORT='deadbeef' -ENV BUILD_VERSION='broken' -RUN turbo build --env-mode=loose +ENV BUILD_COMMIT=deadbeefdeadbeefdeadbeefdeadbeefdeadbeef +ENV BUILD_COMMIT_SHORT=deadbeef +ENV BUILD_VERSION=broken +RUN turbo admin#build --env-mode=loose ################################################################################## # TEST ########################################################################### ################################################################################## -FROM installer as test +FROM build as test # Run command CMD /bin/sh -c "turbo admin#test --env-mode=loose" @@ -112,11 +102,9 @@ CMD /bin/sh -c "turbo admin#test --env-mode=loose" ################################################################################## FROM lipanski/docker-static-website:latest as production -EXPOSE 8080 - # tiny static webserver # https://lipanski.com/posts/smallest-docker-image-static-website # copy builded frontend files -COPY --from=installer /app/admin/build/ ./admin/ +COPY --from=build /app/admin/build/ ./admin/ diff --git a/backend/Dockerfile b/backend/Dockerfile index ff2ce19b4..6d126fa25 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -13,7 +13,7 @@ ENV BUILD_VERSION="0.0.0.0" ## We cannot do `$(git rev-parse --short HEAD)` here so we default to 0000000 ENV BUILD_COMMIT="0000000" ## SET NODE_ENV -ENV NODE_ENV="production" +ENV NODE_ENV=production ## App relevant Envs ENV PORT="4000" ## Timezone @@ -35,9 +35,7 @@ LABEL maintainer="support@gradido.net" # Install Additional Software ## install: git #RUN apk --no-cache add git -RUN yarn global add bun -# Add bun's global bin directory to PATH -ENV PATH="/root/.bun/bin:${PATH}" + # Settings ## Expose Container Port @@ -47,77 +45,62 @@ EXPOSE ${PORT} RUN mkdir -p ${DOCKER_WORKDIR} WORKDIR ${DOCKER_WORKDIR} +################################################################################## +# BUN ############################################################################ +################################################################################## +FROM base as bun-base + +RUN apk update && apk add --no-cache curl tar bash +RUN curl -fsSL https://bun.sh/install | bash +# Add bun's global bin directory to PATH +ENV PATH="/root/.bun/bin:${PATH}" + ################################################################################## # Development #################################################################### ################################################################################## -FROM base AS development - +FROM bun-base AS development # Run command -CMD /bin/sh -c "bun install --frozen-lockfile && bun turbo backend#dev --env-mode=loose" +CMD /bin/sh -c "bun install --filter backend --no-cache --frozen-lockfile \ + && bun install --global --no-cache --no-save turbo@^2 \ + && turbo backend#dev --env-mode=loose" ################################################################################## -# Base with turbo ################################################################ +# Build ########################################################################## ################################################################################## -FROM base as turbo-base - -RUN apk update && apk add --no-cache libc6-compat \ - && bun add --global turbo@^2 \ - && rm -rf /tmp/* ~/.cache node_modules/.cache - -################################################################################## -# BUILDER (create partly monorepo only with data needed by backend) ############## -################################################################################## -FROM turbo-base as builder +FROM bun-base as build COPY --chown=app:app . . -RUN turbo prune backend --docker - - -################################################################################## -# INSTALLER (create production image) ############################################ -################################################################################## -FROM turbo-base AS installer - -# First install the dependencies (as they change less often) -COPY --chown=app:app --from=builder /app/out/json/ . -RUN yarn install --frozen-lockfile --production=false \ - && rm -rf /tmp/* ~/.cache node_modules/.cache \ - && yarn cache clean - -# Build the project -COPY --chown=app:app --from=builder /app/out/full/ . -RUN turbo build +RUN bun install --filter backend --no-cache --frozen-lockfile \ + && bun install --global --no-cache --no-save turbo@^2 \ + && turbo backend#build backend#typecheck --env-mode=loose ################################################################################## # TEST ########################################################################### ################################################################################## -FROM installer as test +FROM build as test ENV DB_HOST=mariadb # Run command CMD /bin/sh -c "turbo backend#test --env-mode=loose" - ################################################################################## # PRODUCTION (Does contain only "binary"- and static-files to reduce image size) # ################################################################################## FROM base as production # Copy "binary"-files from build image -COPY --chown=app:app --from=installer ${DOCKER_WORKDIR}/backend/build/index.js ./index.js +COPY --chown=app:app --from=build ${DOCKER_WORKDIR}/backend/build/index.js ./index.js RUN yarn add sodium-native@4.0.0 email-templates@10.0.1 \ && rm -rf /tmp/* ~/.cache node_modules/.cache \ && yarn cache clean # Copy log4js-config.json to provide log configuration -COPY --chown=app:app --from=installer ${DOCKER_WORKDIR}/backend/log4js-config.json ./log4js-config.json +COPY --chown=app:app --from=build ${DOCKER_WORKDIR}/backend/log4js-config.json ./log4js-config.json # Copy locales -COPY --chown=app:app --from=installer ${DOCKER_WORKDIR}/backend/locales ./locales - -ENV NODE_ENV=production +COPY --chown=app:app --from=build ${DOCKER_WORKDIR}/backend/locales ./locales # Run command -CMD ["node", "index.js"] \ No newline at end of file +CMD ["node", "index.js"] diff --git a/backend/esbuild.config.ts b/backend/esbuild.config.ts index 73d0a0c12..9b545dc4d 100644 --- a/backend/esbuild.config.ts +++ b/backend/esbuild.config.ts @@ -2,7 +2,7 @@ import { esbuildDecorators } from '@anatine/esbuild-decorators' import { build } from 'esbuild' build({ - entryPoints: ['src/index.ts'], + entryPoints: ['src/index.ts', 'src/password/EncryptionWorker.ts'], outdir: 'build', platform: 'node', target: 'node18.20.7', @@ -11,5 +11,6 @@ build({ // legalComments: 'inline', external: ['sodium-native', 'email-templates'], plugins: [esbuildDecorators()], - minify: true, + minify: false, + sourcemap: true, }) diff --git a/backend/src/config/index.ts b/backend/src/config/index.ts index ab07e7fa2..10688a236 100644 --- a/backend/src/config/index.ts +++ b/backend/src/config/index.ts @@ -76,12 +76,12 @@ const community = { const loginServer = { LOGIN_APP_SECRET: process.env.LOGIN_APP_SECRET ?? '21ffbbc616fe', LOGIN_SERVER_KEY: process.env.LOGIN_SERVER_KEY ?? 'a51ef8ac7ef1abf162fb7a65261acd7a', - USE_CRYPTO_WORKER: process.env.USE_CRYPTO_WORKER ?? false, + USE_CRYPTO_WORKER: process.env.USE_CRYPTO_WORKER === 'true', } const email = { - EMAIL: process.env.EMAIL === 'true' || false, - EMAIL_TEST_MODUS: process.env.EMAIL_TEST_MODUS === 'true' || false, + EMAIL: process.env.EMAIL === 'true', + EMAIL_TEST_MODUS: process.env.EMAIL_TEST_MODUS === 'true', EMAIL_TEST_RECEIVER: process.env.EMAIL_TEST_RECEIVER ?? 'stage1@gradido.net', EMAIL_USERNAME: process.env.EMAIL_USERNAME ?? '', EMAIL_SENDER: process.env.EMAIL_SENDER ?? 'info@gradido.net', diff --git a/backend/src/password/EncryptionWorker.ts b/backend/src/password/EncryptionWorker.ts index 34f33226f..391883883 100644 --- a/backend/src/password/EncryptionWorker.ts +++ b/backend/src/password/EncryptionWorker.ts @@ -1,7 +1,5 @@ import { worker } from 'workerpool' -import { CONFIG } from '@/config' - import { crypto_box_SEEDBYTES, crypto_hash_sha512_BYTES, @@ -15,7 +13,7 @@ import { crypto_shorthash_BYTES, } from 'sodium-native' -export const SecretKeyCryptographyCreateKey = ( +export const SecretKeyCryptographyCreateKeyFunc = ( salt: string, password: string, configLoginAppSecret: Buffer, @@ -46,8 +44,8 @@ export const SecretKeyCryptographyCreateKey = ( return encryptionKeyHash.readBigUInt64LE() } -if (CONFIG.USE_CRYPTO_WORKER === true && typeof process.send === 'function') { +if (process.env.USE_CRYPTO_WORKER === 'true' && typeof process.send === 'function') { worker({ - SecretKeyCryptographyCreateKey, + SecretKeyCryptographyCreateKeyFunc, }) } diff --git a/backend/src/password/EncryptorUtils.ts b/backend/src/password/EncryptorUtils.ts index 45f15e6dd..53e217078 100644 --- a/backend/src/password/EncryptorUtils.ts +++ b/backend/src/password/EncryptorUtils.ts @@ -1,5 +1,5 @@ -import { cpus } from 'os' -import path from 'path' +import { cpus } from 'node:os' +import path from 'node:path' import { User } from 'database' import { Pool, pool } from 'workerpool' @@ -12,7 +12,7 @@ import { backendLogger as logger } from '@/server/logger' import { crypto_shorthash_KEYBYTES } from 'sodium-native' -import { SecretKeyCryptographyCreateKey as SecretKeyCryptographyCreateKeySync } from './EncryptionWorker' +import { SecretKeyCryptographyCreateKeyFunc } from './EncryptionWorker' const configLoginAppSecret = Buffer.from(CONFIG.LOGIN_APP_SECRET, 'hex') const configLoginServerKey = Buffer.from(CONFIG.LOGIN_SERVER_KEY, 'hex') @@ -21,12 +21,8 @@ let encryptionWorkerPool: Pool | undefined if (CONFIG.USE_CRYPTO_WORKER === true) { encryptionWorkerPool = pool( - path.join(__dirname, '..', '..', 'build', 'src', 'password', '/EncryptionWorker.js'), - { - // TODO: put maxQueueSize into config - maxQueueSize: 30 * cpus().length, - }, - ) + path.join(__dirname, '..', 'build', 'password', 'EncryptionWorker.js'), + { maxQueueSize: 30 * cpus().length }) } // We will reuse this for changePassword @@ -52,22 +48,20 @@ export const SecretKeyCryptographyCreateKey = async ( crypto_shorthash_KEYBYTES, ) } - let result: Promise + let result: bigint if (encryptionWorkerPool) { - result = (await encryptionWorkerPool.exec('SecretKeyCryptographyCreateKey', [ + result = await encryptionWorkerPool.exec('SecretKeyCryptographyCreateKeyFunc', [ salt, password, configLoginAppSecret, configLoginServerKey, - ])) as Promise + ]) } else { - result = Promise.resolve( - SecretKeyCryptographyCreateKeySync( - salt, - password, - configLoginAppSecret, - configLoginServerKey, - ), + result = SecretKeyCryptographyCreateKeyFunc( + salt, + password, + configLoginAppSecret, + configLoginServerKey, ) } return result @@ -75,6 +69,7 @@ export const SecretKeyCryptographyCreateKey = async ( // pool is throwing this error // throw new Error('Max queue size of ' + this.maxQueueSize + ' reached'); // will be shown in frontend to user + console.log(JSON.stringify(e, null, 2)) throw new LogError('Server is full, please try again in 10 minutes.', e) } } diff --git a/bun.lock b/bun.lock index 632c5fe78..9e1319a73 100644 --- a/bun.lock +++ b/bun.lock @@ -247,13 +247,16 @@ "log4js": "^6.7.1", "reflect-metadata": "^0.1.13", "sodium-native": "^3.3.0", - "tsx": "^4.19.4", + "tsconfig-paths": "^4.1.1", "type-graphql": "^1.1.1", "typeorm": "^0.3.16", "uuid": "8.3.2", }, "devDependencies": { "@biomejs/biome": "1.9.4", + "@swc/cli": "^0.7.3", + "@swc/core": "^1.11.24", + "@swc/helpers": "^0.5.17", "@types/express": "4.17.21", "@types/jest": "27.0.2", "@types/lodash.clonedeep": "^4.5.6", diff --git a/database/Dockerfile b/database/Dockerfile index 311bd3443..4d226f068 100644 --- a/database/Dockerfile +++ b/database/Dockerfile @@ -14,6 +14,8 @@ ENV BUILD_VERSION="0.0.0.0" ENV BUILD_COMMIT="0000000" ## SET NODE_ENV ENV NODE_ENV="production" +## Timezone +ENV TZ=UTC # Labels LABEL org.label-schema.build-date="${BUILD_DATE}" diff --git a/dht-node/Dockerfile b/dht-node/Dockerfile index a1096b9c7..a219600b3 100644 --- a/dht-node/Dockerfile +++ b/dht-node/Dockerfile @@ -1,7 +1,8 @@ ################################################################################## # BASE ########################################################################### ################################################################################## -FROM node:18.20.7-alpine3.21 as base +FROM node:18.20.7-bookworm-slim as base +#FROM node:18.20.7-alpine3.21 as base #FROM ubuntu:latest as base # ENVs (available in production aswell, can be overwritten by commandline or env file) @@ -17,6 +18,8 @@ ENV BUILD_COMMIT="0000000" ENV NODE_ENV="production" ## App relevant Envs #ENV PORT="5000" +## Timezone +ENV TZ=UTC # Labels LABEL org.label-schema.build-date="${BUILD_DATE}" @@ -46,43 +49,42 @@ LABEL maintainer="support@gradido.net" RUN mkdir -p ${DOCKER_WORKDIR} WORKDIR ${DOCKER_WORKDIR} -################################################################################## -# Base with turbo ################################################################ -################################################################################## -FROM base as turbo-base -RUN apk update && apk add --no-cache libc6-compat \ - && yarn global add turbo@^2 \ - && rm -rf /tmp/* ~/.cache node_modules/.cache \ - && yarn cache clean +################################################################################## +# BUN ############################################################################ +################################################################################## +FROM base as bun-base + +RUN apt update && apt install -y --no-install-recommends ca-certificates curl bash unzip +#RUN apk update && apk add --no-cache curl tar bash +RUN curl -fsSL https://bun.sh/install | bash +# Add bun's global bin directory to PATH +ENV PATH="/root/.bun/bin:${PATH}" + +################################################################################## +# Development #################################################################### +################################################################################## +FROM bun-base AS development + +# Run command +CMD /bin/sh -c "bun install --filter dht-node --no-cache --frozen-lockfile \ + && bun install --global --no-cache --no-save turbo@^2 \ + && turbo dht-node#dev --env-mode=loose" ################################################################################## # BUILDER (create partly monorepo only with data needed by dht-node) ############# ################################################################################## -FROM turbo-base as builder +FROM bun-base as build COPY --chown=app:app . . -RUN turbo prune dht-node --docker +RUN bun install --filter dht-node --no-cache --frozen-lockfile \ + && bun install --global --no-cache --no-save turbo@^2 \ + && turbo dht-node#build dht-node#typecheck --env-mode=loose -################################################################################## -# INSTALLER (create production image) ############################################ -################################################################################## -FROM turbo-base AS installer - -# First install the dependencies (as they change less often) -COPY --chown=app:app --from=builder /app/out/json/ . -RUN yarn install --frozen-lockfile --production=false \ - && rm -rf /tmp/* ~/.cache node_modules/.cache \ - && yarn cache clean - -# Build the project -COPY --chown=app:app --from=builder /app/out/full/ . -RUN turbo build typecheck - ################################################################################## # TEST ########################################################################### ################################################################################## -FROM installer as test +FROM build as test ENV DB_HOST=mariadb # Run command @@ -94,16 +96,19 @@ CMD /bin/sh -c "turbo dht-node#test --env-mode=loose" FROM base as production # Copy "binary"-files from build image -COPY --chown=app:app --from=installer ${DOCKER_WORKDIR}/dht-node/build/index.js ./index.js +COPY --chown=app:app --from=build ${DOCKER_WORKDIR}/dht-node/build/index.js ./index.js +COPY --chown=app:app --from=build ${DOCKER_WORKDIR}/scripts/clean-prebuilds.sh ./clean-prebuilds.sh +#COPY --chown=app:app --from=build ${DOCKER_WORKDIR}/dht-node/package.json ./package.json # We also install the native node_modules which cannot be bundled # TODO: find a elegant way to use the right versions from yarn.lock -RUN yarn add udx-native@1.5.3 sodium-native@4.0.0 \ +RUN yarn add @hyperswarm/dht@^6.5.1 \ && rm -rf /tmp/* ~/.cache node_modules/.cache \ - && yarn cache clean + && yarn cache clean \ + && ./clean-prebuilds.sh # Copy log4js-config.json to provide log configuration -COPY --chown=app:app --from=installer ${DOCKER_WORKDIR}/dht-node/log4js-config.json ./log4js-config.json - +COPY --chown=app:app --from=build ${DOCKER_WORKDIR}/dht-node/log4js-config.json ./log4js-config.json +ENV DB_HOST=mariadb # Run command -CMD ["TZ=UTC", "node", "index.js"] \ No newline at end of file +CMD ["node", "index.js"] \ No newline at end of file diff --git a/dht-node/esbuild.config.ts b/dht-node/esbuild.config.ts index ae8fb1ade..8e0c6583a 100644 --- a/dht-node/esbuild.config.ts +++ b/dht-node/esbuild.config.ts @@ -10,4 +10,5 @@ build({ // legalComments: 'inline', external: ['udx-native', 'sodium-native'], minify: true, + sourcemap: false, }) diff --git a/dht-node/package.json b/dht-node/package.json index d327efe66..7e604ee73 100644 --- a/dht-node/package.json +++ b/dht-node/package.json @@ -22,10 +22,8 @@ "cross-env": "^7.0.3", "database": "*", "dotenv": "10.0.0", - "esbuild": "^0.25.3", "joi": "^17.13.3", "log4js": "^6.9.1", - "tsx": "^4.19.4", "typeorm": "^0.3.22", "uuid": "^8.3.2" }, @@ -36,9 +34,11 @@ "@types/joi": "^17.2.3", "@types/node": "^17.0.45", "@types/uuid": "^8.3.4", + "esbuild": "^0.25.3", "jest": "27.5.1", "prettier": "^2.8.8", "ts-jest": "27.1.4", + "tsx": "^4.19.4", "typescript": "^4.9.5" }, "engines": { diff --git a/docker-compose.override.yml b/docker-compose.override.yml index e3a3dafe8..4cb2ec4b7 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -14,7 +14,7 @@ services: volumes: # This makes sure the docker container has its own node modules. # Therefore it is possible to have a different node version on the host machine - - node_modules:/app/node_modules + - node_modules_frontend:/app/node_modules - turbo_cache:/tmp/turbo # bind the local folder to the docker to allow live reload - .:/app @@ -33,7 +33,7 @@ services: volumes: # This makes sure the docker container has its own node modules. # Therefore it is possible to have a different node version on the host machine - - node_modules:/app/node_modules + - node_modules_admin:/app/node_modules - turbo_cache:/tmp/turbo # bind the local folder to the docker to allow live reload - .:/app @@ -54,7 +54,7 @@ services: volumes: # This makes sure the docker container has its own node modules. # Therefore it is possible to have a different node version on the host machine - - node_modules:/app/node_modules + - node_modules_backend:/app/node_modules - turbo_cache:/tmp/turbo # bind the local folder to the docker to allow live reload - .:/app @@ -71,11 +71,11 @@ services: - external-net - internal-net environment: - - NODE_ENV="development" + - NODE_ENV=development volumes: # This makes sure the docker container has its own node modules. # Therefore it is possible to have a different node version on the host machine - - node_modules:/app/node_modules + - node_modules_dht:/app/node_modules - turbo_cache:/tmp/turbo # bind the local folder to the docker to allow live reload - .:/app @@ -113,11 +113,11 @@ services: - external-net - internal-net environment: - - NODE_ENV="development" + - NODE_ENV=development volumes: # This makes sure the docker container has its own node modules. # Therefore it is possible to have a different node version on the host machine - - node_modules:/app/node_modules + - node_modules_federation:/app/node_modules - turbo_cache:/tmp/turbo # bind the local folder to the docker to allow live reload - .:/app @@ -135,11 +135,11 @@ services: build: target: up environment: - - NODE_ENV="development" + - NODE_ENV=development volumes: # This makes sure the docker container has its own node modules. # Therefore it is possible to have a different node version on the host machine - - node_modules:/app/node_modules + - node_modules_database:/app/node_modules - turbo_cache:/tmp/turbo # bind the local folder to the docker to allow live reload - ./database:/app @@ -212,7 +212,14 @@ services: volumes: node_modules: + node_modules_dht: + node_modules_admin: + node_modules_frontend: + node_modules_backend: + node_modules_federation: + node_modules_database: dlt_connector_modules: dlt-database_node_modules: dlt-database_build: - turbo_cache: \ No newline at end of file + turbo_cache: + turbo_cache_dht: \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index bcee473ef..6e3b80786 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,7 +23,7 @@ services: - external-net - internal-net ports: - - 3000:${FRONTEND_MODULE_PORT:-3000} + - ${FRONTEND_MODULE_PORT:-3000}:3000 environment: # Envs used in Dockerfile # - DOCKER_WORKDIR="/app" @@ -54,7 +54,7 @@ services: - external-net - internal-net ports: - - 3000:${ADMIN_MODULE_PORT:-8080} + - ${ADMIN_MODULE_PORT:-8080}:3000 environment: - BUILD_COMMIT=${BUILD_COMMIT} - BUILD_COMMIT_SHORT=${BUILD_COMMIT_SHORT} @@ -100,7 +100,7 @@ services: networks: - internal-net ports: - - 4000:${BACKEND_PORT:-4000} + - ${BACKEND_PORT:-4000}:4000 depends_on: - mariadb environment: @@ -145,7 +145,7 @@ services: - BUILD_DATE - BUILD_VERSION - BUILD_COMMIT - - NODE_ENV="production" + - NODE_ENV=production - DB_HOST=mariadb # Application only envs #env_file: @@ -203,7 +203,7 @@ services: - internal-net - external-net ports: - - 5010:${FEDERATION_PORT:-5010} + - ${FEDERATION_PORT:-5010}:5010 depends_on: - mariadb environment: @@ -212,7 +212,7 @@ services: - BUILD_DATE - BUILD_VERSION - BUILD_COMMIT - - NODE_ENV="production" + - NODE_ENV=production - DB_HOST=mariadb # Application only envs #env_file: @@ -231,6 +231,8 @@ services: context: . dockerfile: ./database/Dockerfile target: up + profiles: + - migration depends_on: - mariadb networks: @@ -242,7 +244,7 @@ services: - BUILD_DATE - BUILD_VERSION - BUILD_COMMIT - - NODE_ENV="production" + - NODE_ENV=production - DB_HOST=mariadb # Application only envs #env_file: diff --git a/federation/Dockerfile b/federation/Dockerfile index 35fc6af98..fcdad2136 100644 --- a/federation/Dockerfile +++ b/federation/Dockerfile @@ -13,10 +13,12 @@ ENV BUILD_VERSION="0.0.0.0" ## We cannot do `$(git rev-parse --short HEAD)` here so we default to 0000000 ENV BUILD_COMMIT="0000000" ## SET NODE_ENV -ENV NODE_ENV="production" +ENV NODE_ENV=production ## App relevant Envs ENV PORT="5010" # ENV PORT="${env.FEDERATION_PORT}" +## Timezone +ENV TZ=UTC # Labels LABEL org.label-schema.build-date="${BUILD_DATE}" @@ -43,44 +45,42 @@ EXPOSE ${PORT} RUN mkdir -p ${DOCKER_WORKDIR} WORKDIR ${DOCKER_WORKDIR} -################################################################################## -# Base with turbo ################################################################ -################################################################################## -FROM base as turbo-base - -RUN apk update && apk add --no-cache libc6-compat \ - && yarn global add turbo@^2 \ - && rm -rf /tmp/* ~/.cache node_modules/.cache \ - && yarn cache clean ################################################################################## -# BUILDER (create partly monorepo only with data needed by federation) ########### +# BUN ############################################################################ ################################################################################## -FROM turbo-base as builder +FROM base as bun-base + +RUN apk update && apk add --no-cache curl tar bash +RUN curl -fsSL https://bun.sh/install | bash +# Add bun's global bin directory to PATH +ENV PATH="/root/.bun/bin:${PATH}" + +################################################################################## +# Development #################################################################### +################################################################################## +FROM bun-base AS development + +# Run command +CMD /bin/sh -c "bun install --filter federation --no-cache --frozen-lockfile \ + && bun install --global --no-cache --no-save turbo@^2 \ + && turbo federation#dev --env-mode=loose" + + +################################################################################## +# Build ########################################################################## +################################################################################## +FROM bun-base as build COPY --chown=app:app . . -RUN turbo prune federation --docker - - -################################################################################## -# INSTALLER (create production image) ############################################ -################################################################################## -FROM turbo-base AS installer - -# First install the dependencies (as they change less often) -COPY --chown=app:app --from=builder /app/out/json/ . -RUN yarn install --frozen-lockfile --production=false \ - && rm -rf /tmp/* ~/.cache node_modules/.cache \ - && yarn cache clean - -# Build the project -COPY --chown=app:app --from=builder /app/out/full/ . -RUN turbo build typecheck +RUN bun install --filter federation --no-cache --frozen-lockfile \ + && bun install --global --no-cache --no-save turbo@^2 \ + && turbo federation#build federation#typecheck --env-mode=loose ################################################################################## # TEST ########################################################################### ################################################################################## -FROM installer as test +FROM build as test ENV DB_HOST=mariadb @@ -93,7 +93,7 @@ CMD /bin/sh -c "turbo federation#test --env-mode=loose" FROM base as production # Copy "binary"-files from build image -COPY --chown=app:app --from=installer ${DOCKER_WORKDIR}/federation/build/index.js ./index.js +COPY --chown=app:app --from=build ${DOCKER_WORKDIR}/federation/build/index.js ./index.js # We also install the native node_modules which cannot be bundled # TODO: find a elegant way to use the right versions from yarn.lock @@ -102,7 +102,7 @@ RUN yarn add sodium-native@4.0.0 \ && yarn cache clean # Copy log4js-config.json to provide log configuration -COPY --chown=app:app --from=installer ${DOCKER_WORKDIR}/federation/log4js-config.json ./log4js-config.json +COPY --chown=app:app --from=build ${DOCKER_WORKDIR}/federation/log4js-config.json ./log4js-config.json # Run command -CMD ["TZ=UTC", "node", "index.js"] \ No newline at end of file +CMD ["node", "index.js"] \ No newline at end of file diff --git a/federation/package.json b/federation/package.json index b57e48263..02e832a83 100644 --- a/federation/package.json +++ b/federation/package.json @@ -9,10 +9,10 @@ "private": false, "scripts": { - "build": "tsx ./esbuild.config.ts", + "build": "ts-node ./esbuild.config.ts", "start": "cross-env TZ=UTC NODE_ENV=production node build/index.js", "start:bun": "cross-env TZ=UTC NODE_ENV=production bun build/index.js", - "dev": "cross-env TZ=UTC tsx watch src/index.ts", + "dev": "cross-env TZ=UTC nodemon -w src --ext ts,json,css -r tsconfig-paths/register src/index.ts", "dev:bun": "cross-env TZ=UTC bun --hot src/index.ts", "typecheck": "tsc --noEmit", "test": "cross-env TZ=UTC NODE_ENV=development DB_DATABASE=gradido_test_federation jest --runInBand --forceExit --detectOpenHandles", @@ -41,13 +41,16 @@ "log4js": "^6.7.1", "reflect-metadata": "^0.1.13", "sodium-native": "^3.3.0", - "tsx": "^4.19.4", + "tsconfig-paths": "^4.1.1", "type-graphql": "^1.1.1", "typeorm": "^0.3.16", "uuid": "8.3.2" }, "devDependencies": { "@biomejs/biome": "1.9.4", + "@swc/cli": "^0.7.3", + "@swc/core": "^1.11.24", + "@swc/helpers": "^0.5.17", "@types/express": "4.17.21", "@types/jest": "27.0.2", "@types/lodash.clonedeep": "^4.5.6", @@ -57,6 +60,7 @@ "apollo-server-testing": "2.25.2", "graphql-tag": "^2.12.6", "jest": "27.2.4", + "nodemon": "^2.0.7", "prettier": "^3.5.3", "ts-jest": "27.0.5", "typescript": "^4.9.5" diff --git a/federation/tsconfig.json b/federation/tsconfig.json index f72262d8e..2d0e18ed8 100644 --- a/federation/tsconfig.json +++ b/federation/tsconfig.json @@ -84,4 +84,7 @@ "skipLibCheck": true, /* Skip type checking of declaration files. */ "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ }, + "ts-node": { + "swc": true + } } diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 9482d8e88..3bf6af01c 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -16,7 +16,8 @@ ENV BUILD_COMMIT=${BUILD_COMMIT} ARG BUILD_COMMIT_SHORT ENV BUILD_COMMIT_SHORT=${BUILD_COMMIT_SHORT} ## SET NODE_ENV -ENV NODE_ENV="production" +ARG NODE_ENV=production +ENV NODE_ENV=${NODE_ENV} ## App relevant Envs ENV PORT="3000" ## Timezone @@ -38,7 +39,6 @@ LABEL maintainer="support@ogradido.net" # Install Additional Software ## install: node-gyp dependencies #RUN apk --no-cache add g++ make python3 -RUN yarn global add bun # Add bun's global bin directory to PATH ENV PATH="/root/.bun/bin:${PATH}" @@ -50,59 +50,49 @@ EXPOSE ${PORT} RUN mkdir -p ${DOCKER_WORKDIR} WORKDIR ${DOCKER_WORKDIR} +################################################################################## +# BUN ############################################################################ +################################################################################## +FROM base as bun-base + +RUN apk update && apk add --no-cache curl tar bash +RUN curl -fsSL https://bun.sh/install | bash +# Add bun's global bin directory to PATH +ENV PATH="/root/.bun/bin:${PATH}" + ################################################################################## # Development #################################################################### ################################################################################## -FROM base AS development +FROM bun-base AS development # used for getting git commit hash direct from .git RUN apk update && apk add --no-cache git # Run command -CMD /bin/sh -c "bun install --frozen-lockfile && bun turbo frontend#dev --env-mode=loose" - - -################################################################################## -# Base with turbo ################################################################ -################################################################################## -FROM base as turbo-base - -RUN apk update && apk add --no-cache libc6-compat \ - && bun add --global turbo@^2 \ - && rm -rf /tmp/* ~/.cache node_modules/.cache - -################################################################################## -# BUILDER (create partly monorepo only with data needed by frontend) ############# -################################################################################## -FROM turbo-base as builder - -COPY --chown=app:app . . -RUN turbo prune frontend --docker +CMD /bin/sh -c "bun install --filter frontend --no-cache --frozen-lockfile \ + && bun install --global --no-cache --no-save turbo@^2 \ + && turbo frontend#dev --env-mode=loose" ################################################################################## # INSTALLER (create production image) ############################################ ################################################################################## -FROM turbo-base AS installer +FROM bun-base AS build + +COPY --chown=app:app . . +RUN bun install --filter frontend --no-cache --frozen-lockfile \ + && bun install --global turbo@^2 -# First install the dependencies (as they change less often) -COPY --chown=app:app --from=builder /app/out/json/ . -RUN yarn install --frozen-lockfile --production=false \ - && rm -rf /tmp/* ~/.cache node_modules/.cache \ - && yarn cache clean - -# Build the project -COPY --chown=app:app --from=builder /app/out/full/ . # TODO: get the actually git commit hash into docker ENV BUILD_COMMIT='deadbeefdeadbeefdeadbeefdeadbeefdeadbeef' ENV BUILD_COMMIT_SHORT='deadbeef' ENV BUILD_VERSION='broken' -RUN turbo build --env-mode=loose +RUN turbo frontend#build --env-mode=loose ################################################################################## # TEST ########################################################################### ################################################################################## -FROM installer as test +FROM build as test # Run command CMD /bin/sh -c "turbo frontend#test --env-mode=loose" @@ -117,5 +107,5 @@ FROM lipanski/docker-static-website:latest as production # https://lipanski.com/posts/smallest-docker-image-static-website # copy builded frontend files -COPY --from=installer /app/frontend/build/ . +COPY --from=build /app/frontend/build/ . diff --git a/frontend/turbo.json b/frontend/turbo.json index 13d3bc9f8..771b65a25 100644 --- a/frontend/turbo.json +++ b/frontend/turbo.json @@ -12,7 +12,10 @@ "persistent": true }, "build": { - "dependsOn": ["compile-scss"] + "dependsOn": ["compile-scss", "^build"], + "outputs": ["build/**"], + "cache": true } } } + \ No newline at end of file diff --git a/scripts/clean-prebuilds.sh b/scripts/clean-prebuilds.sh new file mode 100755 index 000000000..d1afd92f0 --- /dev/null +++ b/scripts/clean-prebuilds.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +# Determine the current platform +ARCH=$(uname -m) +PLATFORM="" + +case "$ARCH" in + x86_64) + PLATFORM="linux-x64" + ;; + aarch64 | arm64) + PLATFORM="linux-arm64" + ;; + *) + echo "Sorry, your architecture is not on the guest list: $ARCH" + exit 1 + ;; +esac + +echo "Platform detected: $PLATFORM" + +# Find all prebuilds folders +find node_modules -type d -name prebuilds | while read prebuild_dir; do + echo "Processing: $prebuild_dir" + + for subdir in "$prebuild_dir"/*; do + [ -d "$subdir" ] || continue + foldername=$(basename "$subdir") + if [ "$foldername" != "$PLATFORM" ]; then + echo " ➜ Deleting: $subdir (you won't miss it)" + rm -rf "$subdir" + else + echo " ✓ Keeping: $subdir (it's a keeper)" + fi + done +done