make install.sh more idempotent, replace placeholder with random values in .env

deployment/bare_metal/.env.dist

@@ -1,7 +1,12 @@
 # Need to adjust!
+# Will be seen on login and from other communities if dht and federation modules are active
 COMMUNITY_NAME="Your community name"
 COMMUNITY_DESCRIPTION="Short Description from your Community."
+# your domain name, without protocol (without https:// or http:// )
+# domain name should be configured in your dns records to point to this server
+# hetzner_cloud/install.sh will be acquire a SSL-certificate via letsencrypt for this domain
 COMMUNITY_HOST=gddhost.tld
+# used in E-Mails and some error message, should be the email address of your own support (team)
 COMMUNITY_SUPPORT_MAIL=support@supportmail.com
 
 # setup email account for sending gradido system messages to users
@@ -12,10 +17,40 @@ EMAIL_PASSWORD=1234
 EMAIL_SMTP_HOST=smtp.lustig.de
 EMAIL_SMTP_PORT=587
 
-BACKEND_PORT=4000
+# Federation, Settings for transactions with other communities
-
 # if set to true allow sending gradidos to another communities
 FEDERATION_XCOM_SENDCOINS_ENABLED=false
+# if set to true, allow redeeming gradido link from another community
+CROSS_TX_REDEEM_LINK_ACTIVE=false
+
+# if you set the value of FEDERATION_DHT_TOPIC, the DHT hyperswarm will start to announce and listen 
+# on an hash created from this topic
+# need for discover other communities
+# get the correct topic from Gradido Academy, GRADIDO_HUB is our test topic
+FEDERATION_DHT_TOPIC=GRADIDO_HUB
+
+# Advanced Server Setup
+
+# Logging
+LOG_LEVEL=info
+GRADIDO_LOG_PATH=/home/gradido/gradido/deployment/bare_metal/log
+TYPEORM_LOGGING_RELATIVE_PATH=../deployment/bare_metal/log/typeorm.backend.log
+
+# Need adjustments for test system
+# protocol for community host, on production usually https, on local dev usually http
+URL_PROTOCOL=https
+
+# only for test server
+DEPLOY_SEED_DATA=false
+# test email
+# if true all email will be send to EMAIL_TEST_RECEIVER instead of email address of user
+EMAIL_TEST_MODUS=false
+EMAIL_TEST_RECEIVER=test_team@gddhost.tld
+
+# webhook for auto update on github repository changes
+WEBHOOK_GITHUB_BRANCH=master
+
+# Business Logic
 
 # how many minutes email verification code is valid
 # also used for password reset code
@@ -23,36 +58,8 @@ EMAIL_CODE_VALID_TIME=1440
 # how many minutes user must wait before he can request the email verification code again
 # also used for password reset code
 EMAIL_CODE_REQUEST_TIME=10
-
+# login expire time
-# Need to adjust by updates
+JWT_EXPIRES_IN=10m
-# config versions
-DATABASE_CONFIG_VERSION=v1.2022-03-18
-BACKEND_CONFIG_VERSION=v23.2024-04-04
-FRONTEND_CONFIG_VERSION=v6.2024-02-27
-ADMIN_CONFIG_VERSION=v2.2024-01-04
-FEDERATION_CONFIG_VERSION=v2.2023-08-24
-FEDERATION_DHT_CONFIG_VERSION=v4.2024-01-17
-FEDERATION_DHT_TOPIC=GRADIDO_HUB
-
-# Need adjustments for test system
-URL_PROTOCOL=https
-# start script
-# only for test server
-DEPLOY_SEED_DATA=false
-# test email
-# if true all email will be send to EMAIL_TEST_RECEIVER instead of email address of user
-EMAIL_TEST_MODUS=false
-EMAIL_TEST_RECEIVER=test_team@gradido.net
-USE_CRYPTO_WORKER=true
-
-# Logging
-LOG_LEVEL=info
-GRADIDO_LOG_PATH=/home/gradido/gradido/deployment/bare_metal/log
-TYPEORM_LOGGING_RELATIVE_PATH=../deployment/bare_metal/log/typeorm.backend.log
-
-# webhook
-WEBHOOK_GITHUB_SECRET=secret
-WEBHOOK_GITHUB_BRANCH=master
 
 # frontend and admin paths, usually don't need changes
 # used in nginx config and for links in emails
@@ -66,44 +73,19 @@ EMAIL_LINK_SETPASSWORD_PATH=/reset-password/
 EMAIL_LINK_FORGOTPASSWORD_PATH=/forgot-password
 EMAIL_LINK_OVERVIEW_PATH=/overview
 ADMIN_AUTH_PATH=/admin/authenticate?token=
-GRAPHQL_PATH=/graphql
+# need to change when frontend is hosted on a different domain as the backend
-
+WALLET_URL=$URL_PROTOCOL://$COMMUNITY_HOST
-# login expire time
+GRAPHQL_URI=$URL_PROTOCOL://$COMMUNITY_HOST/graphql
-JWT_SECRET=secret123
-JWT_EXPIRES_IN=10m
 
 # Federation
-# if you set the value of FEDERATION_DHT_TOPIC, the DHT hyperswarm will start to announce and listen 
-# on an hash created from this topic
 # FEDERATION_DHT_TOPIC=GRADIDO_HUB
-# FEDERATION_DHT_SEED=64ebcb0e3ad547848fef4197c6e2332f
+# 
 # the api port is the baseport, which will be added with the api-version, e.g. 1_0 = 5010
 FEDERATION_COMMUNITY_API_PORT=5000
-FEDERATION_VALIDATE_COMMUNITY_TIMER=60000
 
 # comma separated list of api-versions, which cause starting several federation modules
 FEDERATION_COMMUNITY_APIS=1_0
 
-# externe gradido services (more added in future)
-GDT_ACTIVE=false
-
-AUTO_POLL_INTERVAL=30000
-
-# DLT-Connector (still in develop)
-DLT_ACTIVE=false
-DLT_CONNECTOR_PORT=6010
-DLT_NODE_SERVER_PORT=8340
-DLT_NODE_SERVER_URL=$URL_PROTOCOL://$COMMUNITY_HOST/dlt
-DLT_GRADIDO_NODE_SERVER_HOME_FOLDER=/home/gradido/.gradido
-
-# used for combining a newsletter on klicktipp with this gradido community
-# if used, user will be subscribed on register and can unsubscribe in his account
-KLICKTIPP=false
-KLICKTIPP_USER=gradido_test
-KLICKTIPP_PASSWORD=secret321
-KLICKTIPP_APIKEY_DE=SomeFakeKeyDE
-KLICKTIPP_APIKEY_EN=SomeFakeKeyEN
-
 # Meta data in frontend pages, important when shared via facebook or twitter or for search engines
 META_TITLE_DE="Gradido – Dein Dankbarkeitskonto"
 META_TITLE_EN="Gradido - Your gratitude account"
@@ -113,6 +95,34 @@ META_KEYWORDS_DE="Grundeinkommen, Währung, Dankbarkeit, Schenk-Ökonomie, Natü
 META_KEYWORDS_EN="Basic Income, Currency, Gratitude, Gift Economy, Natural Economy of Life, Economy, Ecology, Potential Development, Giving and Thanking, Cycle of Life, Monetary System"
 META_AUTHOR="Bernd Hückstädt - Gradido-Akademie"
 
+# externe gradido services 
+
+GDT_ACTIVE=false
+
+# DLT-Connector (still in develop)
+# verify transaction additional via gradido blockchain
+DLT_ACTIVE=false
+DLT_CONNECTOR_PORT=6010
+DLT_NODE_SERVER_PORT=8340
+DLT_GRADIDO_NODE_SERVER_HOME_FOLDER=/home/gradido/.gradido
+
+# HIERO used from dlt-connector
+HIERO_HEDERA_NETWORK=testnet
+# https://docs.hedera.com/hedera/networks/testnet/testnet-access
+HIERO_OPERATOR_ID=
+HIERO_OPERATOR_KEY=
+
+# for inspector from outside of server
+DLT_NODE_SERVER_URL=$URL_PROTOCOL://$COMMUNITY_HOST/dlt
+
+# used for combining a newsletter on klicktipp with this gradido community
+# if used, user will be subscribed on register and can unsubscribe in his account
+KLICKTIPP=false
+KLICKTIPP_USER=gradido_test
+KLICKTIPP_PASSWORD=secret321
+KLICKTIPP_APIKEY_DE=SomeFakeKeyDE
+KLICKTIPP_APIKEY_EN=SomeFakeKeyEN
+
 # update page shown while updating gradido
 # page will be fed with status changes
 NGINX_UPDATE_PAGE_ROOT=/home/gradido/gradido/deployment/bare_metal/nginx/update-page
@@ -124,24 +134,82 @@ NGINX_SSL_DHPARAM=/etc/letsencrypt/ssl-dhparams.pem
 NGINX_SSL_INCLUDE=/etc/letsencrypt/options-ssl-nginx.conf
 NGINX_REWRITE_LEGACY_URLS=false
 
-# LEGACY
+# Services
-WEBHOOK_ELOPAGE_SECRET=secret
+BACKEND_PORT=4000
 
 # GMS
+# Speak with Gradido Academy to get access to GMS
 GMS_ACTIVE=false
-# Coordinates of Illuminz test instance
+GMS_API_URL=https://gms-stage.gradido.net/gms
-#GMS_API_URL=http://54.176.169.179:3071
+GMS_DASHBOARD_URL=https://gms-stage.gradido.net/
-GMS_API_URL=http://localhost:4044/
+GMS_USER_SEARCH_FRONTEND_ROUTE=user-search
-GMS_DASHBOARD_URL=http://localhost:8080/
+# set your own or placeholder webhook_secret will be replaced by start.sh
-GMS_WEBHOOK_SECRET=secret
+GMS_WEBHOOK_SECRET=webhook_secret
 GMS_CREATE_USER_THROW_ERRORS=false
 
 # HUMHUB
+# Host your own humhub Server
 HUMHUB_ACTIVE=false
-HUMHUB_API_URL=https://community.gradido.net
+HUMHUB_API_URL=https://your-humhub-domain.tld
-HUMHUB_JWT_KEY=
+# Humhub jwt key, setup together with humhub
+# set your own or placeholder jwt_secret will be replaced by start.sh
+HUMHUB_JWT_KEY=jwt_secret
 
 # OPENAI
+# you need a paid openai account to use this feature
 OPENAI_ACTIVE=false
-OPENAI_API_KEY=''
+# assistant id for your openai assistant
-OPENAI_ASSISTANT_ID=asst_MF5cchZr7wY7rNXayuWvZFsM
+OPENAI_ASSISTANT_ID=
+# your api key form openai
+OPENAI_API_KEY=
+
+# Performance Settings
+
+# in milliseconds, how often inspector and frontend ask on some views for new data
+AUTO_POLL_INTERVAL=60000
+# set to true if password encryption should take place in another thread, which don't block the main loop, true is recommended
+USE_CRYPTO_WORKER=true
+# in milliseconds, how often the other communities should be checked if still online and reachable
+FEDERATION_VALIDATE_COMMUNITY_TIMER=60000
+
+# Security Settings, placeholder (jwt_secret, webhook_secret, binary8_secret, binary16_secret, binary32_secret) 
+# will be replaced by start.sh
+
+JWT_SECRET=jwt_secret
+# secret for webhook, must be set here and in github.com webhook
+WEBHOOK_GITHUB_SECRET=webhook_secret
+# basic for key pair generation for federation, maximal 64 characters 
+FEDERATION_DHT_SEED=binary32_secret
+WEBHOOK_ELOPAGE_SECRET=webhook_secret
+# keys for dlt
+GRADIDO_BLOCKCHAIN_CRYPTO_APP_SECRET=binary8_secret
+GRADIDO_BLOCKCHAIN_SERVER_CRYPTO_KEY=binary16_secret
+HOME_COMMUNITY_SEED=binary32_secret
+
+# Server Settings needed if modules are hosted on different servers (non-default setup)
+
+# GUI-Module Configs (with nginx-routing always running on COMMUNITY_HOST)
+ADMIN_MODULE_PROTOCOL=$URL_PROTOCOL
+ADMIN_MODULE_HOST=$COMMUNITY_HOST
+ADMIN_MODULE_PORT=8080
+
+FRONTEND_MODULE_PROTOCOL=$URL_PROTOCOL
+FRONTEND_MODULE_HOST=$COMMUNITY_HOST
+FRONTEND_MODULE_PORT=3000
+
+# Non-GUI-Module Configs (with nginx-routing always running on localhost in case of on bare metal)
+BACKEND_MODULE_PROTOCOL=http
+BACKEND_MODULE_HOST=localhost
+BACKEND_PORT=4000
+
+DHT_MODULE_PROTOCOL=http
+DHT_MODULE_HOST=localhost
+DHT_MODULE_PORT=5000
+
+DLT_MODULE_PROTOCOL=http
+DLT_MODULE_HOST=localhost
+DLT_MODULE_PORT=6010
+
+FEDERATION_MODULE_PROTOCOL=http
+FEDERATION_MODULE_HOST=localhost
+FEDERATION_MODULE_PORT=5010

deployment/bare_metal/start.sh

@@ -299,7 +299,7 @@ done
 
 # Install all node_modules
 log_step 'Installing node_modules'
-bun install
+bun install --frozen-lockfile
 
 # build all modules
 log_step 'build all modules'
@@ -309,12 +309,12 @@ turbo build --env-mode=loose --concurrency=$(nproc)
 if [ "$DLT_ACTIVE" = true ]; then
   log_step 'build inspector'
   cd $PROJECT_ROOT/inspector
-  bun install
+  bun install --frozen-lockfile
   bun run build
 
   log_step 'build dlt-connector'
   cd $PROJECT_ROOT/dlt-connector
-  bun install
+  bun install --frozen-lockfile
   bun run build
   
   cd $PROJECT_ROOT

deployment/hetzner_cloud/install.sh

@@ -1,7 +1,28 @@
 #!/bin/bash
+# stop if something fails
+set -euo pipefail
+
+log_error() {
+    local message="$1"
+    echo -e "\e[31m$message\e[0m" >&3 # red in console
+}
+
+# called always on error, log error really visible with ascii art in red on console and html
+# stop script execution
+onError() {
+  local exit_code=$?
+  log_error "Command failed!"
+  log_error " /\\_/\\ Line: $(caller 0)"
+  log_error "( x.x )  Exit Code: $exit_code"
+  log_error " >   <   Offending command: '$BASH_COMMAND'"
+  log_error ""
+  exit 1  
+}
+trap onError ERR
+
 # check for parameter
 if [ -z "$1" ]; then
-    echo "Usage: Please provide a branch name as the first argument."
+    log_error "Usage: Please provide a branch name as the first argument."
     exit 1
 fi
 
@@ -19,25 +40,96 @@ LOCAL_SCRIPT_DIR=$(dirname $LOCAL_SCRIPT_PATH)
 PROJECT_ROOT=$SCRIPT_DIR/..
 set +o allexport
 
+# Replace placeholder secrets in .env
+echo 'Replace placeholder secrets in .env'
+# Load .env or .env.dist if not present
+# NOTE: all config values will be in process.env when starting
+# the services and will therefore take precedence over the .env
+if [ -f "$SCRIPT_PATH/.env" ]; then
+    ENV_FILE = $SCRIPT_PATH/.env    
+
+    # --- Secret Generators -------------------------------------------------------
+
+    gen_jwt_secret() {
+    # 32 Character, URL-safe: A-Z a-z 0-9 _ -
+    tr -dc 'A-Za-z0-9_-' < /dev/urandom | head -c 32
+    }
+
+    gen_webhook_secret() {
+    # URL-safe, longer security (40 chars)
+    tr -dc 'A-Za-z0-9_-' < /dev/urandom | head -c 40
+    }
+
+    gen_binary_secret() {
+    local bytes="$1"
+    # Hex -> 2 chars pro byte
+    openssl rand -hex "$bytes"
+    }
+
+    # --- Mapping of Placeholder -> Function --------------------------------------
+
+    generate_secret_for() {
+    case "$1" in
+        jwt_secret)
+        gen_jwt_secret
+        ;;
+        webhook_secret)
+        gen_webhook_secret
+        ;;
+        binary8_secret)
+        gen_binary_secret 8
+        ;;
+        binary16_secret)
+        gen_binary_secret 16
+        ;;
+        binary32_secret)
+        gen_binary_secret 32
+        ;;
+        *)
+        echo "Unknown Placeholder: $1" >&2
+        exit 1
+        ;;
+    esac
+    }
+
+    # --- Placeholder List --------------------------------------------------------
+
+    placeholders=(
+    "jwt_secret"
+    "webhook_secret"
+    "binary8_secret"
+    "binary16_secret"
+    "binary32_secret"
+    )
+
+    # --- Processing in .env -------------------------------------------------
+
+    TMP_FILE="${ENV_FILE}.tmp"
+    cp "$ENV_FILE" "$TMP_FILE"
+
+    for ph in "${placeholders[@]}"; do
+    # Secret generate
+    new_value="$(generate_secret_for "$ph")"
+
+    # Only replace lines that do NOT start with #
+    sed -i "/^[[:space:]]*#/! s/$ph/$new_value/g" "$TMP_FILE"
+    done
+
+    # Write back
+    mv "$TMP_FILE" "$ENV_FILE"
+fi
+
 # If install.sh will be called more than once
-# We have to load the backend .env to get DB_USERNAME, DB_PASSWORD AND JWT_SECRET
+# We have to load the backend .env to get DB_USERNAME and DB_PASSWORD 
-# and the dht-node .env to get FEDERATION_DHT_SEED
 export_var(){
   export $1=$(grep -v '^#' $PROJECT_ROOT/backend/.env | grep -e "$1" | sed -e 's/.*=//')
-  export $1=$(grep -v '^#' $PROJECT_ROOT/dht-node/.env | grep -e "$1" | sed -e 's/.*=//')
 }
 
 if [ -f "$PROJECT_ROOT/backend/.env" ]; then
     export_var 'DB_USER'
     export_var 'DB_PASSWORD'
-    export_var 'JWT_SECRET'
 fi
 
-if [ -f "$PROJECT_ROOT/dht-node/.env" ]; then
-    export_var 'FEDERATION_DHT_SEED'
-fi
-
-
 # Load .env or .env.dist if not present
 # NOTE: all config values will be in process.env when starting
 # the services and will therefore take precedence over the .env
@@ -97,15 +189,17 @@ systemctl restart fail2ban
 rm /etc/nginx/sites-enabled/default
 envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $SCRIPT_PATH/nginx/sites-available/gradido.conf.template > $SCRIPT_PATH/nginx/sites-available/gradido.conf
 envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $SCRIPT_PATH/nginx/sites-available/update-page.conf.template > $SCRIPT_PATH/nginx/sites-available/update-page.conf
-mkdir $SCRIPT_PATH/nginx/sites-enabled
+mkdir -p $SCRIPT_PATH/nginx/sites-enabled
-ln -s $SCRIPT_PATH/nginx/sites-available/update-page.conf $SCRIPT_PATH/nginx/sites-enabled/default
+ln -sf $SCRIPT_PATH/nginx/sites-available/update-page.conf $SCRIPT_PATH/nginx/sites-enabled/default
-ln -s $SCRIPT_PATH/nginx/sites-enabled/default /etc/nginx/sites-enabled
+ln -sf $SCRIPT_PATH/nginx/sites-enabled/default /etc/nginx/sites-enabled
-ln -s $SCRIPT_PATH/nginx/common /etc/nginx/
+ln -sf $SCRIPT_PATH/nginx/common /etc/nginx/
-rmdir /etc/nginx/conf.d
+if [ -e /etc/nginx/conf.d ] && [ ! -L /etc/nginx/conf.d ]; then
-ln -s $SCRIPT_PATH/nginx/conf.d /etc/nginx/
+    rm -rf /etc/nginx/conf.d
+    ln -s $SCRIPT_PATH/nginx/conf.d /etc/nginx/
+fi
 
 # Make nginx restart automatic
-mkdir /etc/systemd/system/nginx.service.d
+mkdir -p /etc/systemd/system/nginx.service.d
 # Define the content to be put into the override.conf file
 CONFIG_CONTENT="[Unit]
 StartLimitIntervalSec=500

dlt-connector/src/config/schema.ts

@@ -37,7 +37,7 @@ export const configSchema = v.object({
   GRADIDO_BLOCKCHAIN_SERVER_CRYPTO_KEY: hex16Schema,
   HOME_COMMUNITY_SEED: v.pipe(
     hexSchema,
-    v.length(64, 'expect seed length minimum 64 characters (32 Bytes)'),
+    v.length(64, 'expect seed length 64 characters (32 Bytes)'),
     v.transform<string, MemoryBlock>((input: string) => MemoryBlock.fromHex(input)),
   ),
   HIERO_HEDERA_NETWORK: v.optional(