Merge pull request #3593 from gradido/setup_improve

feat(workflow): update .env.dist in deploy and make install script more idempotent

admin/.env.template

@@ -1,5 +1,3 @@
-CONFIG_VERSION=$ADMIN_CONFIG_VERSION
-
 COMMUNITY_HOST=$COMMUNITY_HOST
 URL_PROTOCOL=$URL_PROTOCOL
 WALLET_AUTH_PATH=$WALLET_AUTH_PATH

backend/.env.template

@@ -1,5 +1,3 @@
-# must match the CONFIG_VERSION.EXPECTED definition in scr/config/index.ts
-
 # Server
 JWT_SECRET=$JWT_SECRET
 JWT_EXPIRES_IN=$JWT_EXPIRES_IN

database/.env.template

@@ -1,5 +1,3 @@
-CONFIG_VERSION=$DATABASE_CONFIG_VERSION
-
 DB_HOST=127.0.0.1
 DB_PORT=3306
 DB_USER=$DB_USER

database/migration/index.ts

@@ -25,7 +25,7 @@ const run = async (command: string) => {
       host: ${CONFIG.DB_HOST}
       port: ${CONFIG.DB_PORT}
       user: ${CONFIG.DB_USER}
-      password: ${CONFIG.DB_PASSWORD.slice(-2)}
+      password: last 2 character: ${CONFIG.DB_PASSWORD.slice(-2)}
       database: ${CONFIG.DB_DATABASE}`,
     )
   }

deployment/bare_metal/.env.dist

@@ -1,7 +1,13 @@
 # Need to adjust!
+# Will be seen on login and from other communities if dht and federation modules are active
 COMMUNITY_NAME="Your community name"
+# Description should have at least 10 characters
 COMMUNITY_DESCRIPTION="Short Description from your Community."
+# your domain name, without protocol (without https:// or http:// )
+# domain name should be configured in your dns records to point to this server
+# hetzner_cloud/install.sh will be acquire a SSL-certificate via letsencrypt for this domain
 COMMUNITY_HOST=gddhost.tld
+# used in E-Mails and some error message, should be the email address of your own support (team)
 COMMUNITY_SUPPORT_MAIL=support@supportmail.com
 
 # setup email account for sending gradido system messages to users
@@ -12,10 +18,44 @@ EMAIL_PASSWORD=1234
 EMAIL_SMTP_HOST=smtp.lustig.de
 EMAIL_SMTP_PORT=587
 
-BACKEND_PORT=4000
+# Federation, Settings for transactions with other communities
-
 # if set to true allow sending gradidos to another communities
 FEDERATION_XCOM_SENDCOINS_ENABLED=false
+# if set to true, allow redeeming gradido link from another community
+CROSS_TX_REDEEM_LINK_ACTIVE=false
+
+# if you set the value of FEDERATION_DHT_TOPIC, the DHT hyperswarm will start to announce and listen 
+# on an hash created from this topic
+# need for discover other communities
+# get the correct topic from Gradido Academy, GRADIDO_HUB is our test topic
+FEDERATION_DHT_TOPIC=GRADIDO_HUB
+
+# Advanced Server Setup
+
+# SCSS Parsing 
+# grass speed up frontend building, but needs some time for the first setup because it is a rust program which need to be compiled
+USE_GRASS=false
+
+# Logging
+LOG_LEVEL=info
+GRADIDO_LOG_PATH=/home/gradido/gradido/deployment/bare_metal/log
+TYPEORM_LOGGING_RELATIVE_PATH=../deployment/bare_metal/log/typeorm.backend.log
+
+# Need adjustments for test system
+# protocol for community host, on production usually https, on local dev usually http
+URL_PROTOCOL=https
+
+# only for test server
+DEPLOY_SEED_DATA=false
+# test email
+# if true all email will be send to EMAIL_TEST_RECEIVER instead of email address of user
+EMAIL_TEST_MODUS=false
+EMAIL_TEST_RECEIVER=test_team@gradido.net
+
+# webhook for auto update on github repository changes
+WEBHOOK_GITHUB_BRANCH=master
+
+# Business Logic
 
 # how many minutes email verification code is valid
 # also used for password reset code
@@ -23,36 +63,8 @@ EMAIL_CODE_VALID_TIME=1440
 # how many minutes user must wait before he can request the email verification code again
 # also used for password reset code
 EMAIL_CODE_REQUEST_TIME=10
-
+# login expire time
-# Need to adjust by updates
+JWT_EXPIRES_IN=10m
-# config versions
-DATABASE_CONFIG_VERSION=v1.2022-03-18
-BACKEND_CONFIG_VERSION=v23.2024-04-04
-FRONTEND_CONFIG_VERSION=v6.2024-02-27
-ADMIN_CONFIG_VERSION=v2.2024-01-04
-FEDERATION_CONFIG_VERSION=v2.2023-08-24
-FEDERATION_DHT_CONFIG_VERSION=v4.2024-01-17
-FEDERATION_DHT_TOPIC=GRADIDO_HUB
-
-# Need adjustments for test system
-URL_PROTOCOL=https
-# start script
-# only for test server
-DEPLOY_SEED_DATA=false
-# test email
-# if true all email will be send to EMAIL_TEST_RECEIVER instead of email address of user
-EMAIL_TEST_MODUS=false
-EMAIL_TEST_RECEIVER=test_team@gradido.net
-USE_CRYPTO_WORKER=true
-
-# Logging
-LOG_LEVEL=info
-GRADIDO_LOG_PATH=/home/gradido/gradido/deployment/bare_metal/log
-TYPEORM_LOGGING_RELATIVE_PATH=../deployment/bare_metal/log/typeorm.backend.log
-
-# webhook
-WEBHOOK_GITHUB_SECRET=secret
-WEBHOOK_GITHUB_BRANCH=master
 
 # frontend and admin paths, usually don't need changes
 # used in nginx config and for links in emails
@@ -66,44 +78,19 @@ EMAIL_LINK_SETPASSWORD_PATH=/reset-password/
 EMAIL_LINK_FORGOTPASSWORD_PATH=/forgot-password
 EMAIL_LINK_OVERVIEW_PATH=/overview
 ADMIN_AUTH_PATH=/admin/authenticate?token=
+# need to change when frontend is hosted on a different domain as the backend
+WALLET_URL=$URL_PROTOCOL://$COMMUNITY_HOST
 GRAPHQL_PATH=/graphql
 
-# login expire time
-JWT_SECRET=secret123
-JWT_EXPIRES_IN=10m
-
 # Federation
-# if you set the value of FEDERATION_DHT_TOPIC, the DHT hyperswarm will start to announce and listen 
-# on an hash created from this topic
 # FEDERATION_DHT_TOPIC=GRADIDO_HUB
-# FEDERATION_DHT_SEED=64ebcb0e3ad547848fef4197c6e2332f
+# 
 # the api port is the baseport, which will be added with the api-version, e.g. 1_0 = 5010
 FEDERATION_COMMUNITY_API_PORT=5000
-FEDERATION_VALIDATE_COMMUNITY_TIMER=60000
 
 # comma separated list of api-versions, which cause starting several federation modules
 FEDERATION_COMMUNITY_APIS=1_0
 
-# externe gradido services (more added in future)
-GDT_ACTIVE=false
-
-AUTO_POLL_INTERVAL=30000
-
-# DLT-Connector (still in develop)
-DLT_ACTIVE=false
-DLT_CONNECTOR_PORT=6010
-DLT_NODE_SERVER_PORT=8340
-DLT_NODE_SERVER_URL=$URL_PROTOCOL://$COMMUNITY_HOST/dlt
-DLT_GRADIDO_NODE_SERVER_HOME_FOLDER=/home/gradido/.gradido
-
-# used for combining a newsletter on klicktipp with this gradido community
-# if used, user will be subscribed on register and can unsubscribe in his account
-KLICKTIPP=false
-KLICKTIPP_USER=gradido_test
-KLICKTIPP_PASSWORD=secret321
-KLICKTIPP_APIKEY_DE=SomeFakeKeyDE
-KLICKTIPP_APIKEY_EN=SomeFakeKeyEN
-
 # Meta data in frontend pages, important when shared via facebook or twitter or for search engines
 META_TITLE_DE="Gradido – Dein Dankbarkeitskonto"
 META_TITLE_EN="Gradido - Your gratitude account"
@@ -113,6 +100,35 @@ META_KEYWORDS_DE="Grundeinkommen, Währung, Dankbarkeit, Schenk-Ökonomie, Natü
 META_KEYWORDS_EN="Basic Income, Currency, Gratitude, Gift Economy, Natural Economy of Life, Economy, Ecology, Potential Development, Giving and Thanking, Cycle of Life, Monetary System"
 META_AUTHOR="Bernd Hückstädt - Gradido-Akademie"
 
+# externe gradido services 
+
+GDT_ACTIVE=false
+GDT_API_URL=https://gdt.gradido.net
+
+# DLT-Connector (still in develop)
+# verify transaction additional via gradido blockchain
+DLT_ACTIVE=false
+DLT_CONNECTOR_PORT=6010
+DLT_NODE_SERVER_PORT=8340
+DLT_GRADIDO_NODE_SERVER_HOME_FOLDER=/home/gradido/.gradido
+
+# HIERO used from dlt-connector
+HIERO_HEDERA_NETWORK=testnet
+# https://docs.hedera.com/hedera/networks/testnet/testnet-access
+HIERO_OPERATOR_ID=
+HIERO_OPERATOR_KEY=
+
+# for inspector from outside of server
+DLT_NODE_SERVER_URL=$URL_PROTOCOL://$COMMUNITY_HOST/dlt
+
+# used for combining a newsletter on klicktipp with this gradido community
+# if used, user will be subscribed on register and can unsubscribe in his account
+KLICKTIPP=false
+KLICKTIPP_USER=gradido_test
+KLICKTIPP_PASSWORD=secret321
+KLICKTIPP_APIKEY_DE=SomeFakeKeyDE
+KLICKTIPP_APIKEY_EN=SomeFakeKeyEN
+
 # update page shown while updating gradido
 # page will be fed with status changes
 NGINX_UPDATE_PAGE_ROOT=/home/gradido/gradido/deployment/bare_metal/nginx/update-page
@@ -124,24 +140,82 @@ NGINX_SSL_DHPARAM=/etc/letsencrypt/ssl-dhparams.pem
 NGINX_SSL_INCLUDE=/etc/letsencrypt/options-ssl-nginx.conf
 NGINX_REWRITE_LEGACY_URLS=false
 
-# LEGACY
+# Services
-WEBHOOK_ELOPAGE_SECRET=secret
+BACKEND_PORT=4000
 
 # GMS
+# Speak with Gradido Academy to get access to GMS
 GMS_ACTIVE=false
-# Coordinates of Illuminz test instance
+GMS_API_URL=https://gms-stage.gradido.net/gms
-#GMS_API_URL=http://54.176.169.179:3071
+GMS_DASHBOARD_URL=https://gms-stage.gradido.net/
-GMS_API_URL=http://localhost:4044/
+GMS_USER_SEARCH_FRONTEND_ROUTE=user-search
-GMS_DASHBOARD_URL=http://localhost:8080/
+# set your own or placeholder webhook_secret will be replaced by start.sh
-GMS_WEBHOOK_SECRET=secret
+GMS_WEBHOOK_SECRET=webhook_secret
 GMS_CREATE_USER_THROW_ERRORS=false
 
 # HUMHUB
+# Host your own humhub Server
 HUMHUB_ACTIVE=false
-HUMHUB_API_URL=https://community.gradido.net
+HUMHUB_API_URL=https://your-humhub-domain.tld
-HUMHUB_JWT_KEY=
+# Humhub jwt key, setup together with humhub
+# set your own or placeholder jwt_secret will be replaced by start.sh
+HUMHUB_JWT_KEY=jwt_secret
 
 # OPENAI
+# you need a paid openai account to use this feature
 OPENAI_ACTIVE=false
-OPENAI_API_KEY=''
+# assistant id for your openai assistant
-OPENAI_ASSISTANT_ID=asst_MF5cchZr7wY7rNXayuWvZFsM
+OPENAI_ASSISTANT_ID=
+# your api key form openai
+OPENAI_API_KEY=
+
+# Performance Settings
+
+# in milliseconds, how often inspector and frontend ask on some views for new data
+AUTO_POLL_INTERVAL=60000
+# set to true if password encryption should take place in another thread, which don't block the main loop, true is recommended
+USE_CRYPTO_WORKER=true
+# in milliseconds, how often the other communities should be checked if still online and reachable
+FEDERATION_VALIDATE_COMMUNITY_TIMER=60000
+
+# Security Settings, placeholder (jwt_secret, webhook_secret, binary8_secret, binary16_secret, binary32_secret) 
+# will be replaced by start.sh
+
+JWT_SECRET=jwt_secret
+# secret for webhook, must be set here and in github.com webhook
+WEBHOOK_GITHUB_SECRET=webhook_secret
+# basic for key pair generation for federation, maximal 64 characters 
+FEDERATION_DHT_SEED=binary32_secret
+WEBHOOK_ELOPAGE_SECRET=webhook_secret
+# keys for dlt
+GRADIDO_BLOCKCHAIN_CRYPTO_APP_SECRET=binary8_secret
+GRADIDO_BLOCKCHAIN_SERVER_CRYPTO_KEY=binary16_secret
+HOME_COMMUNITY_SEED=binary32_secret
+
+# Server Settings needed if modules are hosted on different servers (non-default setup)
+
+# GUI-Module Configs (with nginx-routing always running on COMMUNITY_HOST)
+ADMIN_MODULE_PROTOCOL=$URL_PROTOCOL
+ADMIN_MODULE_HOST=$COMMUNITY_HOST
+ADMIN_MODULE_PORT=8080
+
+FRONTEND_MODULE_PROTOCOL=$URL_PROTOCOL
+FRONTEND_MODULE_HOST=$COMMUNITY_HOST
+FRONTEND_MODULE_PORT=3000
+
+# Non-GUI-Module Configs (with nginx-routing always running on localhost in case of on bare metal)
+BACKEND_MODULE_PROTOCOL=http
+BACKEND_MODULE_HOST=localhost
+BACKEND_PORT=4000
+
+DHT_MODULE_PROTOCOL=http
+DHT_MODULE_HOST=localhost
+DHT_MODULE_PORT=5000
+
+DLT_MODULE_PROTOCOL=http
+DLT_MODULE_HOST=localhost
+DLT_MODULE_PORT=6010
+
+FEDERATION_MODULE_PROTOCOL=http
+FEDERATION_MODULE_HOST=localhost
+FEDERATION_MODULE_PORT=5010

deployment/bare_metal/Dockerfile

@@ -40,6 +40,8 @@ ENV BRANCH_NAME=$BRANCH_NAME
 RUN git clone https://github.com/gradido/gradido.git --branch $BRANCH_NAME 
 RUN cp /app/gradido/deployment/bare_metal/.env.dist /app/gradido/deployment/bare_metal/.env
 RUN sed -i 's/^URL_PROTOCOL=https$/URL_PROTOCOL=http/' /app/gradido/deployment/bare_metal/.env
+RUN sed -i 's/^COMMUNITY_HOST=gddhost.tld$/COMMUNITY_HOST=localhost/' /app/gradido/deployment/bare_metal/.env
+
 
 # setup nginx
 WORKDIR /app/gradido/deployment/bare_metal/nginx

deployment/bare_metal/install-missing-deps.sh

@@ -1,10 +1,11 @@
 #!/bin/bash
 
+# source profile so PATH/NVM/BUN werden gesetzt (safe for non-login)
+if [ -f /home/gradido/.profile ]; then
+  . /home/gradido/.profile
+fi
 # Ensure required tools are installed
 
-# make sure correct node version is installed
-export NVM_DIR="$HOME/.nvm"
-[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
 if ! command -v nvm > /dev/null
 then
     echo "'nvm' is missing, will be installed now!"
@@ -14,7 +15,7 @@ install_nvm() {
     nvm install 
     nvm use 
     nvm alias default 
-    npm i -g yarn pm2
+    npm i -g pm2 turbo
     pm2 startup
 }
 nvm use || install_nvm
@@ -52,23 +53,24 @@ fi
 if ! command -v turbo > /dev/null
 then
     echo "'turbo' is missing, will be installed now!"
-    bun install --global turbo
+    npm i -g turbo
 fi
 
 # rust and grass 
-if ! command -v cargo > /dev/null
+if [ "$USE_GRASS" = true ]; then
-then
+    if ! command -v cargo > /dev/null
-    echo "'cargo' is missing, will be installed now!"
+    then
-    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+        echo "'cargo' is missing, will be installed now!"
-    export CARGO_HOME="$HOME/.cargo"
+        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-    export PATH="$CARGO_HOME/bin:$PATH"
+        export CARGO_HOME="$HOME/.cargo"
+        export PATH="$CARGO_HOME/bin:$PATH"
+    fi
+    if ! command -v grass > /dev/null
+    then
+        echo "'grass' is missing, will be installed now!"
+        cargo install grass
+    fi
 fi
-if ! command -v grass > /dev/null
-then
-    echo "'grass' is missing, will be installed now!"
-    cargo install grass
-fi
-
 # redis
 if ! command -v  redis-cli --version > /dev/null
 then 

deployment/bare_metal/install.sh

@@ -1,149 +0,0 @@
-#!/bin/bash
-
-# This install script requires the minimum requirements already installed.
-# How to do this is described in detail in [setup.md](./setup.md)
-
-# Find current directory & configure paths
-## For manualy use in terminal
-## set -o allexport
-## SCRIPT_DIR=$(pwd)
-## PROJECT_ROOT=$SCRIPT_DIR/../..
-## set +o allexport
-# Use here in script
-set -o allexport
-SCRIPT_PATH=$(realpath $0)
-SCRIPT_DIR=$(dirname $SCRIPT_PATH)
-PROJECT_ROOT=$SCRIPT_DIR/../..
-set +o allexport
-
-# Load .env or .env.dist if not present
-# NOTE: all config values will be in process.env when starting
-# the services and will therefore take precedence over the .env
-if [ -f "$SCRIPT_DIR/.env" ]; then
-    set -o allexport
-    source $SCRIPT_DIR/.env
-    set +o allexport
-else
-    set -o allexport
-    source $SCRIPT_DIR/.env.dist
-    set +o allexport
-fi
-
-# Configure git
-git config pull.ff only
-
-# Install mariadb
-sudo apt-get install -y mariadb-server
-sudo mysql_secure_installation
-# Enter current password for root (enter for none): enter
-# Switch to unix_socket authentication [Y/n] Y
-# Change the root password? [Y/n] n
-# Remove anonymous users? [Y/n] Y
-# Disallow root login remotely? [Y/n] Y
-# Remove test database and access to it? [Y/n] Y
-# Reload privilege tables now? [Y/n] Y
-
-# Install nginx
-sudo apt-get install -y nginx
-sudo rm /etc/nginx/sites-enabled/default
-sudo ln -s /home/gradido/gradido/deployment/bare_metal/nginx/sites-available/gradido.conf /etc/nginx/sites-available
-# sudo ln -s /etc/nginx/sites-available/gradido.conf /etc/nginx/sites-enabled
-sudo ln -s /home/gradido/gradido/deployment/bare_metal/nginx/sites-available/update-page.conf /etc/nginx/sites-available
-sudo ln -s /home/gradido/gradido/deployment/bare_metal/nginx/common /etc/nginx/
-sudo rmdir /etc/nginx/conf.d
-sudo ln -s /home/gradido/gradido/deployment/bare_metal/nginx/conf.d /etc/nginx/
-
-# Allow nginx configuration and restart for gradido
-#TODO generate file
-sudo nano /etc/sudoers.d/gradido
-> gradido ALL=(ALL) NOPASSWD: /etc/init.d/nginx start,/etc/init.d/nginx stop,/etc/init.d/nginx restart
-sudo chmod a+rw /etc/nginx/sites-enabled
-
-# Install node 16.x
-sudo apt-get install -y curl
-curl -sL https://deb.nodesource.com/setup_16.x | sudo -E bash -
-sudo apt-get install -y nodejs
-sudo apt-get install -y build-essential
-
-# Install yarn
-sudo apt-get install -y gnupg
-curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
-echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
-sudo apt-get update
-sudo apt-get install -y yarn
-
-# Install pm2
-sudo yarn global add pm2
-pm2 startup
-> execute command output in shell
-
-# Install certbot
-sudo apt-get install -y certbot
-sudo apt-get install -y python3-certbot-nginx
-sudo certbot
-> Enter email address (used for urgent renewal and security notices) > e.g. support@supportmail.com
-> Please read the Terms of Service at > Y
-> Would you be willing, once your first certificate is successfully issued, to > N
-> No names were found in your configuration files. Please enter in your domain > stage1.gradido.net
-# Note: this will throw an error regarding not beeing able to identify the nginx corresponding
-#       config but produce the required certificate - thats perfectly fine this way
-# Troubleshoot: to manually renew a certificate with running nginx use the following command:
-#               (this might be required once to properly have things setup for the cron to autorenew)
-# sudo certbot --nginx -d example.com -d www.example.com
-# Troubleshoot: to check ut if things working you can use
-# sudo certbot renew --dry-run
-
-# Install logrotate
-sudo apt-get install -y logrotate
-envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $SCRIPT_DIR/logrotate/gradido.conf.template > $SCRIPT_DIR/logrotate/gradido.conf
-sudo cp $SCRIPT_DIR/logrotate/gradido.conf.template /etc/logrotate.d/gradido.conf
-sudo chown root:root /etc/logrotate.d/gradido.conf
-
-# Install mysql autobackup
-sudo apt-get install -y automysqlbackup
-
-# Webhooks (optional) (for development)
-sudo apt install -y webhook
-# TODO generate
-# put hook into github
-# TODO adjust secret
-# TODO adjust branch if needed
-# https://stage1.gradido.net/hooks/github
-envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $SCRIPT_DIR/webhook/hooks.json.template > ~/hooks.json
-
-webhook -hooks ~/hooks.json &
-# or for debugging
-# webhook -hooks ~/hooks.json -verbose
-
-# create db user
-export DB_USER=gradido
-export DB_PASSWORD=$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo);
-sudo mysql <<EOFMYSQL
-    CREATE USER '$DB_USER'@'localhost' IDENTIFIED BY '$DB_PASSWORD';
-    GRANT ALL PRIVILEGES ON *.* TO '$DB_USER'@'localhost';
-    FLUSH PRIVILEGES;
-EOFMYSQL
-
-# Configure database
-envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $PROJECT_ROOT/database/.env.template > $PROJECT_ROOT/database/.env
-
-# Configure backend
-export JWT_SECRET=$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo);
-envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $PROJECT_ROOT/backend/.env.template > $PROJECT_ROOT/backend/.env
-
-# Configure frontend
-envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $PROJECT_ROOT/frontend/.env.template > $PROJECT_ROOT/frontend/.env
-
-# Configure admin
-envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $PROJECT_ROOT/admin/.env.template > $PROJECT_ROOT/admin/.env
-
-# Configure dht-node
-envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $PROJECT_ROOT/dht-node/.env.template > $PROJECT_ROOT/dht-node/.env
-
-# create cronjob to delete yarn output in /tmp
-# crontab -e
-# hourly job: 0 * * * * find /tmp -name "yarn--*" -cmin +60 -exec rm -r {} \; > /dev/null
-# daily job:  0 4 * * * find /tmp -name "yarn--*" -ctime +1 -exec rm -r {} \; > /dev/null
-# Start gradido
-# Note: on first startup some errors will occur - nothing serious
-./start.sh

deployment/bare_metal/start.sh

@@ -2,6 +2,11 @@
 # stop if something fails
 set -euo pipefail
 
+# source profile so PATH/NVM/BUN werden gesetzt (safe for non-login)
+if [ -f /home/gradido/.profile ]; then
+  . /home/gradido/.profile
+fi
+
 # check for parameter
 FAST_MODE=false
 POSITIONAL_ARGS=()
@@ -184,7 +189,7 @@ cd $PROJECT_ROOT
 # TODO: this overfetches alot, but ensures we can use start.sh with tags
 git fetch --all
 git checkout $BRANCH_NAME
-git pull
+git pull origin $BRANCH_NAME
 git submodule update --init --recursive
 export BUILD_COMMIT="$(git rev-parse HEAD)"
 
@@ -299,7 +304,7 @@ done
 
 # Install all node_modules
 log_step 'Installing node_modules'
-bun install
+bun install --frozen-lockfile
 
 # build all modules
 log_step 'build all modules'
@@ -309,12 +314,12 @@ turbo build --env-mode=loose --concurrency=$(nproc)
 if [ "$DLT_ACTIVE" = true ]; then
   log_step 'build inspector'
   cd $PROJECT_ROOT/inspector
-  bun install
+  bun install --frozen-lockfile
   bun run build
 
   log_step 'build dlt-connector'
   cd $PROJECT_ROOT/dlt-connector
-  bun install
+  bun install --frozen-lockfile
   bun run build
   
   cd $PROJECT_ROOT

deployment/hetzner_cloud/README.md

@@ -1,6 +1,15 @@
 # Migration
 [Migration from 2.2.0 to 2.2.1](migration/2_2_0-2_2_1/README.md)
 
+# Key Pair
+It is recommended to create a new ssh key pair for your gradido server.
+You can create it with this command:
+```bash
+ssh-keygen -t ed25519 -C "your_email@example.com"
+```
+
+**Reason**: We recommend `ed25519` because it provides strong security with smaller key sizes, faster performance, and resistance to known attacks, making it more secure and efficient than traditional RSA keys.
+
 # Setup on Hetzner Cloud Server
 Suggested OS:
 Debian 12
@@ -15,7 +24,7 @@ ssh_authorized_keys:
   - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAkLGbzbG7KIGfkssKJBkc/0EVAzQ/8vjvVHzNdxhK8J yourname
 ```
 
-I made a (german) video to show it to you:
+I made a (german) video to show it to you (video is older, cloudConfig.yaml differ):
 
 [![Video](https://img.youtube.com/vi/fORK3Bt3lPw/hqdefault.jpg)](https://www.youtube.com/watch?v=fORK3Bt3lPw)
 
@@ -23,14 +32,7 @@ I made a (german) video to show it to you:
 ### setup your domain pointing on server ip address 
 ### login to your new server as root
 ```bash
-ssh -i /path/to/privKey root@gddhost.tld
+ssh -i ~/.ssh/id_ed25519 root@gddhost.tld
-```
-
-### Change default shell
-
-```bash
-chsh -s /bin/bash
-chsh -s /bin/bash gradido
 ```
 
 ### Set password for user `gradido`
@@ -51,16 +53,15 @@ su gradido
 If you logout from the server you can test authentication:
 
 ```bash
-$ ssh -i /path/to/privKey gradido@gddhost.tld
+$ ssh -i ~/.ssh/id_ed25519 gradido@gddhost.tld
 # This should log you in and allow you to use sudo commands, which will require the user's password
 ```
 
 ### Disable password root login via ssh
 
 ```bash
-sudo cp /etc/ssh/sshd_config /etc/ssh/sshd_config.org
+sudo sed -i -e '/^\(#\|\)PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config.d/ssh-hardening.conf
-sudo sed -i -e '/^\(#\|\)PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
+sudo sed -i '$a AllowUsers gradido' /etc/ssh/sshd_config.d/ssh-hardening.conf
-sudo sed -i '$a AllowUsers gradido' /etc/ssh/sshd_config
 sudo /etc/init.d/ssh restart
 ```
 
@@ -69,16 +70,17 @@ sudo /etc/init.d/ssh restart
 ```bash
 $ ssh gradido@gddhost.tld
 # Will result in in either a passphrase request for your key or the message 'Permission denied (publickey)'
-$ ssh -i /path/to/privKey root@gddhost.tld
+$ ssh -i ~/.ssh/id_ed25519 root@gddhost.tld
 # Will result in 'Permission denied (publickey)'
-$ ssh -i /path/to/privKey gradido@gddhost.tld
+$ ssh -i ~/.ssh/id_ed25519 gradido@gddhost.tld
 # Will succeed after entering the correct keys passphrase (if any)
 ```
 
 ### Install `Gradido` code
+`latest` is a tag pointing on last stable release
 ```bash
 cd ~
-git clone https://github.com/gradido/gradido.git
+git clone https://github.com/gradido/gradido.git --branch latest --depth 1
 ```
 
 ### Adjust the values in `.env`
@@ -99,21 +101,35 @@ All your following installations in `install.sh` will fail!*
 cd ~/gradido/deployment/bare_metal
 cp .env.dist .env
 nano .env
-
-# adjust values accordingly
 ```
 
-### Run `install.sh` with branch name
+For a minimal setup you need at least to change this values: 
+```env
+COMMUNITY_NAME="Your community name"
+COMMUNITY_DESCRIPTION="Short Description from your Community."
+# your domain name, without protocol (without https:// or http:// )
+# domain name should be configured in your dns records to point to this server
+# hetzner_cloud/install.sh will be acquire a SSL-certificate via letsencrypt for this domain
+COMMUNITY_HOST=gddhost.tld
+
+# setup email account for sending gradido system messages to users
+EMAIL_USERNAME=peter@lustig.de
+EMAIL_SENDER=peter@lustig.de
+EMAIL_PASSWORD=1234
+EMAIL_SMTP_HOST=smtp.lustig.de
+```
+
+### Run `install.sh` with branch or tag name
 ***!!! Attention !!!***
 Don't use this script if you have custom config in /etc/nginx/conf.d, because this script
 will remove it and ln ../bare_metal/nginx/conf.d
 
 ```bash
 cd ~/gradido/deployment/hetzner_cloud
-sudo ./install.sh release-2_2_0
+sudo ./install.sh latest
 ```
 
-I made a (german) video to show it to you:
+I made a (german) video to show it to you (video is older, output will differ):
 
 [![Video](https://img.youtube.com/vi/9h-55Si6bMk/hqdefault.jpg)](https://www.youtube.com/watch?v=9h-55Si6bMk)
 
@@ -133,3 +149,16 @@ sudo mysql -D gradido_community -e "insert into user_roles(user_id, role) values
 I made a (german) video to show it to you:
 
 [![Video](https://img.youtube.com/vi/xVQ5t4MnLrE/hqdefault.jpg)](https://www.youtube.com/watch?v=xVQ5t4MnLrE)
+
+### Troubleshooting
+
+If after some tests this error occur, right after `Requesting a certificate for your-domain.tld`, try again another day. Letsencrypt is rate limited:
+
+```bash
+An unexpected error occurred:
+AttributeError: can't set attribute
+```
+
+### But it isn't working
+
+If it isn't working you can write us: [support@gradido.net](mailto:support@gradido.net)

deployment/hetzner_cloud/cloudConfig.yaml

@@ -14,6 +14,7 @@ packages:
   - git
   - mariadb-server
   - nginx
+  - redis
   - curl
   - build-essential
   - gnupg
@@ -22,6 +23,7 @@ packages:
   - logrotate
   - automysqlbackup
   - expect
+  - unzip
 package_update: true
 package_upgrade: true
 write_files:

deployment/hetzner_cloud/install.sh

@@ -1,7 +1,28 @@
 #!/bin/bash
+# stop if something fails
+set -euo pipefail
+
+log_error() {
+    local message="$1"
+    echo -e "\e[31m$message\e[0m" # red in console
+}
+
+# called always on error, log error really visible with ascii art in red on console and html
+# stop script execution
+onError() {
+  local exit_code=$?
+  log_error "Command failed!"
+  log_error " /\\_/\\ Line: $(caller 0)"
+  log_error "( x.x )  Exit Code: $exit_code"
+  log_error " >   <   Offending command: '$BASH_COMMAND'"
+  log_error ""
+  exit 1  
+}
+trap onError ERR
+
 # check for parameter
 if [ -z "$1" ]; then
-    echo "Usage: Please provide a branch name as the first argument."
+    log_error "Usage: Please provide a branch name as the first argument."
     exit 1
 fi
 
@@ -19,25 +40,88 @@ LOCAL_SCRIPT_DIR=$(dirname $LOCAL_SCRIPT_PATH)
 PROJECT_ROOT=$SCRIPT_DIR/..
 set +o allexport
 
+# Replace placeholder secrets in .env
+echo 'Replace placeholder secrets in .env'
+# Load .env or .env.dist if not present
+# NOTE: all config values will be in process.env when starting
+# the services and will therefore take precedence over the .env
+if [ -f "$SCRIPT_PATH/.env" ]; then
+    ENV_FILE="$SCRIPT_PATH/.env"
+
+    # --- Secret Generators -------------------------------------------------------
+
+    gen_jwt_secret() {
+        # 32 Character, URL-safe: A-Z a-z 0-9 _ -
+        tr -dc 'A-Za-z0-9_-' < /dev/urandom | head -c 32 2>/dev/null || true
+    }
+
+    gen_webhook_secret() {
+        # URL-safe, longer security (40 chars)
+        tr -dc 'A-Za-z0-9_-' < /dev/urandom | head -c 40 2>/dev/null || true
+    }
+
+    gen_binary_secret() {
+        local bytes="$1"
+        # Hex -> 2 chars pro byte
+        openssl rand -hex "$bytes" 2>/dev/null || true
+    }
+
+    # --- Mapping of Placeholder -> Function --------------------------------------
+
+    generate_secret_for() {
+    case "$1" in
+        jwt_secret)      gen_jwt_secret      ;;
+        webhook_secret)  gen_webhook_secret  ;;
+        binary8_secret)  gen_binary_secret 8 ;;
+        binary16_secret) gen_binary_secret 16;;
+        binary32_secret) gen_binary_secret 32;;
+        *) 
+            echo "Unknown Placeholder: $1" >&2 
+            exit 1
+            ;;
+    esac
+    }
+
+    # --- Placeholder List --------------------------------------------------------
+
+    placeholders=(
+    "jwt_secret"
+    "webhook_secret"
+    "binary8_secret"
+    "binary16_secret"
+    "binary32_secret"
+    )
+
+    # --- Processing in .env -------------------------------------------------
+
+    TMP_FILE="${ENV_FILE}.tmp"
+    cp "$ENV_FILE" "$TMP_FILE"
+
+    for ph in "${placeholders[@]}"; do
+        # Iterate over all lines containing the placeholder
+        while grep -q "$ph" "$TMP_FILE"; do
+            new_value=$(generate_secret_for "$ph")
+            # Replace only the first occurrence per line
+            sed -i "0,/$ph/s//$new_value/" "$TMP_FILE"
+        done
+    done
+
+    # Write back
+    mv "$TMP_FILE" "$ENV_FILE"
+    chown gradido:gradido "$ENV_FILE"
+fi
+
 # If install.sh will be called more than once
-# We have to load the backend .env to get DB_USERNAME, DB_PASSWORD AND JWT_SECRET
+# We have to load the backend .env to get DB_USERNAME and DB_PASSWORD 
-# and the dht-node .env to get FEDERATION_DHT_SEED
 export_var(){
   export $1=$(grep -v '^#' $PROJECT_ROOT/backend/.env | grep -e "$1" | sed -e 's/.*=//')
-  export $1=$(grep -v '^#' $PROJECT_ROOT/dht-node/.env | grep -e "$1" | sed -e 's/.*=//')
 }
 
 if [ -f "$PROJECT_ROOT/backend/.env" ]; then
     export_var 'DB_USER'
     export_var 'DB_PASSWORD'
-    export_var 'JWT_SECRET'
 fi
 
-if [ -f "$PROJECT_ROOT/dht-node/.env" ]; then
-    export_var 'FEDERATION_DHT_SEED'
-fi
-
-
 # Load .env or .env.dist if not present
 # NOTE: all config values will be in process.env when starting
 # the services and will therefore take precedence over the .env
@@ -97,15 +181,17 @@ systemctl restart fail2ban
 rm /etc/nginx/sites-enabled/default
 envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $SCRIPT_PATH/nginx/sites-available/gradido.conf.template > $SCRIPT_PATH/nginx/sites-available/gradido.conf
 envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $SCRIPT_PATH/nginx/sites-available/update-page.conf.template > $SCRIPT_PATH/nginx/sites-available/update-page.conf
-mkdir $SCRIPT_PATH/nginx/sites-enabled
+mkdir -p $SCRIPT_PATH/nginx/sites-enabled
-ln -s $SCRIPT_PATH/nginx/sites-available/update-page.conf $SCRIPT_PATH/nginx/sites-enabled/default
+ln -sf $SCRIPT_PATH/nginx/sites-available/update-page.conf $SCRIPT_PATH/nginx/sites-enabled/default
-ln -s $SCRIPT_PATH/nginx/sites-enabled/default /etc/nginx/sites-enabled
+ln -sf $SCRIPT_PATH/nginx/sites-enabled/default /etc/nginx/sites-enabled
-ln -s $SCRIPT_PATH/nginx/common /etc/nginx/
+ln -sf $SCRIPT_PATH/nginx/common /etc/nginx/
-rmdir /etc/nginx/conf.d
+if [ -e /etc/nginx/conf.d ] && [ ! -L /etc/nginx/conf.d ]; then
-ln -s $SCRIPT_PATH/nginx/conf.d /etc/nginx/
+    rm -rf /etc/nginx/conf.d
+    ln -s $SCRIPT_PATH/nginx/conf.d /etc/nginx/
+fi
 
 # Make nginx restart automatic
-mkdir /etc/systemd/system/nginx.service.d
+mkdir -p /etc/systemd/system/nginx.service.d
 # Define the content to be put into the override.conf file
 CONFIG_CONTENT="[Unit]
 StartLimitIntervalSec=500
@@ -124,11 +210,17 @@ sudo systemctl daemon-reload
 # setup https with certbot
 certbot certonly --nginx --non-interactive --agree-tos --domains $COMMUNITY_HOST --email $COMMUNITY_SUPPORT_MAIL
 
+export NVM_DIR="/home/gradido/.nvm"
+BUN_VERSION_FILE="$PROJECT_ROOT/.bun-version"
+if [ ! -f "$BUN_VERSION_FILE" ]; then
+    echo ".bun-version file not found at: $BUN_VERSION_FILE"
+    exit 1
+fi
+export BUN_VERSION="$(cat "$BUN_VERSION_FILE" | tr -d '[:space:]')"
+export BUN_INSTALL="/home/gradido/.bun"
+
 # run as gradido user (until EOF)
-sudo -u gradido bash <<'EOF'
+sudo -u gradido bash <<EOF
-    export NVM_DIR="/home/gradido/.nvm"
-    NODE_VERSION="v18.20.7"
-    export NVM_DIR
     # Install nvm if it doesn't exist
     if [ ! -d "$NVM_DIR" ]; then
         curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
@@ -137,15 +229,26 @@ sudo -u gradido bash <<'EOF'
     [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
 
     # Install Node if not already installed
-    if ! nvm ls $NODE_VERSION >/dev/null 2>&1; then
+    if ! nvm ls >/dev/null 2>&1; then
-        nvm install $NODE_VERSION
+        nvm install
     fi
-    # Install yarn and pm2
+    # Install pm2 and turbo
-    npm i -g yarn pm2 
+    npm i -g pm2 turbo
-    # start pm2
+
-    pm2 startup
+    echo "'bun' v$BUN_VERSION will be installed now!"
+    curl -fsSL https://bun.com/install | bash -s "bun-v${BUN_VERSION}"
 EOF
 
+# Load bun
+export BUN_INSTALL="/home/gradido/.bun"
+export PATH="$BUN_INSTALL/bin:$PATH"
+
+# Load nvm 
+export NVM_DIR="/home/gradido/.nvm"
+[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
+# start pm2
+pm2 startup
+
 # Install logrotate
 envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $SCRIPT_PATH/logrotate/gradido.conf.template > $SCRIPT_PATH/logrotate/gradido.conf
 cp $SCRIPT_PATH/logrotate/gradido.conf /etc/logrotate.d/gradido.conf
@@ -153,15 +256,15 @@ cp $SCRIPT_PATH/logrotate/gradido.conf /etc/logrotate.d/gradido.conf
 # create db user
 export DB_USER=gradido
 # create a new password only if it not already exist
-if [ -z "${DB_PASSWORD}" ]; then
+: "${DB_PASSWORD:=$(tr -dc '_A-Za-z0-9' < /dev/urandom | head -c 32)}"
-    export DB_PASSWORD=$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c 32; echo);
-fi
 
 # Check if DB_PASSWORD is still empty, then exit with an error
 if [ -z "${DB_PASSWORD}" ]; then
     echo "Error: Failed to generate DB_PASSWORD."
     exit 1
 fi
+export DB_PASSWORD
+
 mysql <<EOFMYSQL
     CREATE USER IF NOT EXISTS '$DB_USER'@'localhost' IDENTIFIED BY '$DB_PASSWORD';
     GRANT ALL PRIVILEGES ON *.* TO '$DB_USER'@'localhost';
@@ -195,5 +298,4 @@ chown -R gradido:gradido $PROJECT_ROOT
 sudo -u gradido crontab < $LOCAL_SCRIPT_DIR/crontabs.txt
 
 # Start gradido
-# Note: on first startup some errors will occur - nothing serious
 sudo -u gradido $SCRIPT_PATH/start.sh $1

dht-node/.env.template

@@ -1,6 +1,3 @@
-# must match the CONFIG_VERSION.EXPECTED definition in scr/config/index.ts
-CONFIG_VERSION=$FEDERATION_DHT_CONFIG_VERSION
-
 # Database
 DB_HOST=127.0.0.1
 DB_PORT=3306
@@ -14,7 +11,6 @@ COMMUNITY_NAME=$COMMUNITY_NAME
 COMMUNITY_DESCRIPTION=$COMMUNITY_DESCRIPTION
 
 # Federation
-FEDERATION_DHT_CONFIG_VERSION=$FEDERATION_DHT_CONFIG_VERSION
 # if you set the value of FEDERATION_DHT_TOPIC, the DHT hyperswarm will start to announce and listen 
 # on an hash created from this topic
 FEDERATION_DHT_TOPIC=$FEDERATION_DHT_TOPIC

dlt-connector/src/config/schema.ts

@@ -37,7 +37,7 @@ export const configSchema = v.object({
   GRADIDO_BLOCKCHAIN_SERVER_CRYPTO_KEY: hex16Schema,
   HOME_COMMUNITY_SEED: v.pipe(
     hexSchema,
-    v.length(64, 'expect seed length minimum 64 characters (32 Bytes)'),
+    v.length(64, 'expect seed length 64 characters (32 Bytes)'),
     v.transform<string, MemoryBlock>((input: string) => MemoryBlock.fromHex(input)),
   ),
   HIERO_HEDERA_NETWORK: v.optional(

federation/.env.template

@@ -1,6 +1,3 @@
-# must match the CONFIG_VERSION.EXPECTED definition in scr/config/index.ts
-CONFIG_VERSION=$FEDERATION_CONFIG_VERSION
-
 LOG_LEVEL=$LOG_LEVEL
 # this is set fix to false, because it is important for 'production' environments. only set to true if a graphql-playground should be in use
 GRAPHIQL=false

frontend/.env.template

@@ -1,5 +1,3 @@
-CONFIG_VERSION=$FRONTEND_CONFIG_VERSION
-
 # Endpoints
 GRAPHQL_PATH=$GRAPHQL_PATH
 ADMIN_AUTH_PATH=$ADMIN_AUTH_PATH

package.json

@@ -21,7 +21,8 @@
   ],
   "scripts": {
     "release": "bumpp -r",
-    "version": "auto-changelog -p --commit-limit 0 && git add CHANGELOG.md",
+    "version": "auto-changelog -p --commit-limit 0 && git add CHANGELOG.md && git commit -m \"update changelog\" && git push",
+    "postversion": "git push origin :refs/tags/latest && git tag -f latest && git push origin latest",
     "installAll": "bun run install",
     "docker": "cross-env BUILD_COMMIT=$(git rev-parse HEAD) docker compose -f docker-compose.yml up",
     "docker:rebuild": "cross-env BUILD_COMMIT=$(git rev-parse HEAD) docker compose -f docker-compose.yml build",