From 13bb5636fb5da025a68b4f53aabe97dee17ea8bf Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Mon, 28 Apr 2025 18:17:52 +0200 Subject: [PATCH] update dht workflow and docker --- .github/workflows/test_dht_node.yml | 84 +++++++++++-------------- dht-node/Dockerfile | 98 ++++++++++++++--------------- 2 files changed, 84 insertions(+), 98 deletions(-) diff --git a/.github/workflows/test_dht_node.yml b/.github/workflows/test_dht_node.yml index 4cb50a093..03448ec29 100644 --- a/.github/workflows/test_dht_node.yml +++ b/.github/workflows/test_dht_node.yml @@ -7,13 +7,14 @@ jobs: name: Detect File Changes - DHT Node runs-on: ubuntu-latest outputs: + config: ${{ steps.changes.outputs.config }} database: ${{ steps.changes.outputs.database }} dht_node: ${{ steps.changes.outputs.dht_node }} docker-compose: ${{ steps.changes.outputs.docker-compose }} steps: - uses: actions/checkout@v3.3.0 - - name: Check for frontend file changes + - name: Check for dht-node, config-schema, database, docker-compose file changes uses: dorny/paths-filter@v2.11.1 id: changes with: @@ -23,69 +24,58 @@ jobs: build: name: Docker Build Test - DHT Node - if: needs.files-changed.outputs.dht_node == 'true' + if: needs.files-changed.outputs.config == 'true' || needs.files-changed.outputs.database == 'true' || needs.files-changed.outputs.dht_node == 'true' || needs.files-changed.outputs.docker-compose == 'true' needs: files-changed runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v3 - - name: Build 'test' image - run: | - docker build --target test -t "gradido/dht-node:test" -f dht-node/Dockerfile . - docker save "gradido/dht-node:test" > /tmp/dht-node.tar - - - name: Upload Artifact - uses: actions/upload-artifact@v4 - with: - name: docker-dht-node-test - path: /tmp/dht-node.tar - - lint: - name: Lint - DHT Node - if: needs.files-changed.outputs.dht_node == 'true' - needs: files-changed - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Lint - run: cd database && yarn && cd ../config && yarn install && cd ../dht-node && yarn && yarn run lint + - name: Build 'test' image (typecheck & build) + run: docker build --target test -t "gradido/dht-node:test" -f dht-node/Dockerfile . unit_test: name: Unit Tests - DHT Node - if: needs.files-changed.outputs.database == 'true' || needs.files-changed.outputs.dht_node == 'true' || needs.files-changed.outputs.docker-compose == 'true' || needs.files-changed.outputs.mariadb == 'true' - needs: [files-changed, build] + if: needs.files-changed.outputs.config == 'true' || needs.files-changed.outputs.database == 'true' || needs.files-changed.outputs.dht_node == 'true' || needs.files-changed.outputs.docker-compose == 'true' + needs: files-changed runs-on: ubuntu-latest + outputs: + test-success: ${{ steps.test.outputs.success }} steps: + - name: Set Node.js version + uses: actions/setup-node@v4 + with: + node-version: '18.20.7' + - name: Checkout code uses: actions/checkout@v3 - - name: Download Docker Image - uses: actions/download-artifact@v4 - with: - name: docker-dht-node-test - path: /tmp - - - name: Load Docker Image - run: docker load < /tmp/dht-node.tar - - name: docker-compose mariadb run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach --no-deps mariadb - - name: Sleep for 30 seconds - run: sleep 30s - shell: bash + - name: Install turbo + run: yarn global add turbo@^2 - - name: docker-compose database - run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach --no-deps database + - name: Prune dht-node with turbos help + run: turbo prune dht-node - - name: Sleep for 30 seconds - run: sleep 30s - shell: bash + - name: install dependencies + run: cd out && yarn install --frozen-lockfile --production=false - - name: Unit tests - run: cd database && yarn && yarn build && cd ../config && yarn install && cd ../dht-node && yarn && yarn test - #- name: Unit tests - # run: docker run --env NODE_ENV=test --env DB_HOST=mariadb --network gradido_internal-net --rm gradido/dht-node:test yarn run test + - name: Wait for MariaDB to be ready + run: docker run --rm --network gradido_internal-net busybox sh -c 'until nc -z mariadb 3306; do echo waiting for db; sleep 1; done;' + + - name: run unit test & lint & build + id: test + run: | + cd out && turbo dht-node#lint dht-node#test + echo "success=$([ $? -eq 0 ] && echo true || echo false)" >> $GITHUB_OUTPUT + + lint: + name: Lint - DHT Node + if: needs.files-changed.outputs.config == 'true' || needs.files-changed.outputs.database == 'true' || needs.files-changed.outputs.dht_node == 'true' || needs.files-changed.outputs.docker-compose == 'true' + needs: [files-changed, unit_test] + runs-on: ubuntu-latest + steps: + - name: Check result from previous step + run: if [ "${{ needs.unit_test.outputs.test-success }}" != "true" ]; then exit 1; fi diff --git a/dht-node/Dockerfile b/dht-node/Dockerfile index dab401762..d4565ff2b 100644 --- a/dht-node/Dockerfile +++ b/dht-node/Dockerfile @@ -46,13 +46,43 @@ LABEL maintainer="support@gradido.net" RUN mkdir -p ${DOCKER_WORKDIR} WORKDIR ${DOCKER_WORKDIR} -RUN mkdir -p /database -RUN mkdir -p /config +################################################################################## +# Base with turbo ################################################################ +################################################################################## +FROM base as turbo-base + +RUN apk update && apk add --no-cache libc6-compat \ + && yarn global add turbo@^2 \ + && rm -rf /tmp/* ~/.cache node_modules/.cache \ + && yarn cache clean + +################################################################################## +# BUILDER (create partly monorepo only with data needed by dht-node) ############# +################################################################################## +FROM turbo-base as builder + +COPY --chown=app:app . . +RUN turbo prune dht-node --docker + +################################################################################## +# INSTALLER (create production image) ############################################## +################################################################################## +FROM turbo-base AS installer + +# First install the dependencies (as they change less often) +COPY --chown=app:app --from=builder /app/out/json/ . +RUN yarn install --frozen-lockfile --production=false \ + && rm -rf /tmp/* ~/.cache node_modules/.cache \ + && yarn cache clean + +# Build the project +COPY --chown=app:app --from=builder /app/out/full/ . +RUN turbo build ################################################################################## # DEVELOPMENT (Connected to the local environment, to reload on demand) ########## ################################################################################## -FROM base as development +FROM installer as development # We don't need to copy or build anything since we gonna bind to the # local filesystem which will need a rebuild anyway @@ -60,42 +90,17 @@ FROM base as development # Run command # (for development we need to execute yarn install since the # node_modules are on another volume and need updating) -CMD /bin/sh -c "cd /database && yarn install && yarn build && cd /config && yarn install && cd /app && yarn install && yarn run dev" - -################################################################################## -# BUILD (Does contain all files and is therefore bloated) ######################## -################################################################################## -FROM base as build - -# Copy everything from dht-node -COPY ./dht-node/ ./ -# Copy everything from database -COPY ./database/ ../database/ -# Copy everything from config -COPY ./config/ ../config/ - -# yarn install and build config -RUN cd ../config && yarn install --production=false --frozen-lockfile --non-interactive && yarn run build - -# yarn install dht-node -RUN yarn install --production=false --frozen-lockfile --non-interactive - -# yarn install database -RUN cd ../database && yarn install --production=false --frozen-lockfile --non-interactive - -# yarn build -RUN yarn run build - -# yarn build database -RUN cd ../database && yarn run build +CMD /bin/sh -c "turbo dev --env-mode=loose" + ################################################################################## # TEST ########################################################################### ################################################################################## -FROM build as test +FROM installer as test +ENV DB_HOST=mariadb # Run command -CMD /bin/sh -c "yarn run start" +CMD /bin/sh -c "turbo dht-node#test --env-mode=loose" ################################################################################## # PRODUCTION (Does contain only "binary"- and static-files to reduce image size) # @@ -103,25 +108,16 @@ CMD /bin/sh -c "yarn run start" FROM base as production # Copy "binary"-files from build image -COPY --from=build ${DOCKER_WORKDIR}/build ./build -COPY --from=build ${DOCKER_WORKDIR}/../database/build ../database/build -COPY --from=build ${DOCKER_WORKDIR}/../config/build ../config/build -# We also copy the node_modules express and serve-static for the run script -COPY --from=build ${DOCKER_WORKDIR}/node_modules ./node_modules -COPY --from=build ${DOCKER_WORKDIR}/../database/node_modules ../database/node_modules -COPY --from=build ${DOCKER_WORKDIR}/../config/node_modules ../config/node_modules +COPY --chown=app:app --from=installer ${DOCKER_WORKDIR}/dht-node/build/index.js ./index.js + +# We also install the native node_modules which cannot be bundled +# TODO: find a elegant way to use the right versions from yarn.lock +RUN yarn add udx-native@1.5.3 sodium-native@4.0.0 \ + && rm -rf /tmp/* ~/.cache node_modules/.cache \ + && yarn cache clean -# Copy static files -# COPY --from=build ${DOCKER_WORKDIR}/public ./public -# Copy package.json for script definitions (lock file should not be needed) -COPY --from=build ${DOCKER_WORKDIR}/package.json ./package.json -# Copy tsconfig.json to provide alias path definitions -COPY --from=build ${DOCKER_WORKDIR}/tsconfig.json ./tsconfig.json # Copy log4js-config.json to provide log configuration -COPY --from=build ${DOCKER_WORKDIR}/log4js-config.json ./log4js-config.json - -# Copy run scripts run/ -# COPY --from=build ${DOCKER_WORKDIR}/run ./run +COPY --chown=app:app --from=installer ${DOCKER_WORKDIR}/dht-node/log4js-config.json ./log4js-config.json # Run command -CMD /bin/sh -c "yarn run start" \ No newline at end of file +CMD ["node", "index.js"] \ No newline at end of file