Merge remote-tracking branch

'origin/2730-devops-add-federation-modul-to-deployment-scripts' into
2630-feature-federation-implement-a-graphql-endpoint-for-request-getpublickey

database/migrations/0060-update_communities_table.ts

@@ -23,6 +23,7 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis
 
 export async function downgrade(queryFn: (query: string, values?: any[]) => Promise<Array<any>>) {
   // write downgrade logic as parameter of queryFn
+  await queryFn('DELETE FROM `communities` WHERE `last_announced_at` IS NULL;')
   await queryFn(
     'ALTER TABLE `communities` MODIFY COLUMN `last_announced_at` datetime(3) NOT NULL AFTER `end_point`;',
   )

deployment/bare_metal/.env.dist

@@ -65,7 +65,9 @@ FEDERATION_COMMUNITY_URL=http://stage1.gradido.net
 # the api port is the baseport, which will be added with the api-version, e.g. 1_0 = 5010
 FEDERATION_COMMUNITY_API_PORT=5000
 
-
+FEDERATION_CONFIG_VERSION=v1.2023-01-09
+# comma separated list of api-versions, which cause starting several federation moduls
+FEDERATION_COMMUNITY_APIS=1_0,1_1,2_0
 
 # database
 DATABASE_CONFIG_VERSION=v1.2022-03-18

deployment/bare_metal/nginx/sites-available/gradido-federation.conf.template

@@ -0,0 +1,15 @@
+
+    location /api/$FEDERATION_APIVERSION {
+        proxy_http_version 1.1;
+        proxy_set_header   Upgrade $http_upgrade;
+        proxy_set_header   Connection 'upgrade';
+        proxy_set_header   X-Forwarded-For $remote_addr;
+        proxy_set_header   X-Real-IP  $remote_addr;
+        proxy_set_header   Host $host;
+        
+        proxy_pass         http://127.0.0.1:$FEDERATION_PORT;
+        proxy_redirect     off;
+
+        access_log $GRADIDO_LOG_PATH/nginx-access.federation-$FEDERATION_PORT.log gradido_log;
+        error_log $GRADIDO_LOG_PATH/nginx-error.federation-$FEDERATION_PORT.log warn;
+    }

deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template

@@ -113,50 +113,7 @@ server {
     }
     
     # Federation
-    location /api/1_0 {
+    $FEDERATION_NGINX_CONF
-        proxy_http_version 1.1;
-        proxy_set_header   Upgrade $http_upgrade;
-        proxy_set_header   Connection 'upgrade';
-        proxy_set_header   X-Forwarded-For $remote_addr;
-        proxy_set_header   X-Real-IP  $remote_addr;
-        proxy_set_header   Host $host;
-        
-        proxy_pass         http://127.0.0.1:5010;
-        proxy_redirect     off;
-
-        access_log $GRADIDO_LOG_PATH/nginx-access.federation-5010.log gradido_log;
-        error_log $GRADIDO_LOG_PATH/nginx-error.federation-5010.log warn;
-    }
-
-    location /api/1_1 {
-        proxy_http_version 1.1;
-        proxy_set_header   Upgrade $http_upgrade;
-        proxy_set_header   Connection 'upgrade';
-        proxy_set_header   X-Forwarded-For $remote_addr;
-        proxy_set_header   X-Real-IP  $remote_addr;
-        proxy_set_header   Host $host;
-        
-        proxy_pass         http://127.0.0.1:5011;
-        proxy_redirect     off;
-
-        access_log $GRADIDO_LOG_PATH/nginx-access.federation-5011.log gradido_log;
-        error_log $GRADIDO_LOG_PATH/nginx-error.federation-5011.log warn;
-    }
-
-    location /api/2_0 {
-        proxy_http_version 1.1;
-        proxy_set_header   Upgrade $http_upgrade;
-        proxy_set_header   Connection 'upgrade';
-        proxy_set_header   X-Forwarded-For $remote_addr;
-        proxy_set_header   X-Real-IP  $remote_addr;
-        proxy_set_header   Host $host;
-        
-        proxy_pass         http://127.0.0.1:5020;
-        proxy_redirect     off;
-
-        access_log $GRADIDO_LOG_PATH/nginx-access.federation-5020.log gradido_log;
-        error_log $GRADIDO_LOG_PATH/nginx-error.federation-5020.log warn;
-    }
 
     # TODO this could be a performance optimization
     #location /vue {

deployment/bare_metal/nginx/sites-available/gradido.conf.template

@@ -99,50 +99,7 @@ server {
     }
 
     # Federation
-    location /api/1_0 {
+    $FEDERATION_NGINX_CONF
-        proxy_http_version 1.1;
-        proxy_set_header   Upgrade $http_upgrade;
-        proxy_set_header   Connection 'upgrade';
-        proxy_set_header   X-Forwarded-For $remote_addr;
-        proxy_set_header   X-Real-IP  $remote_addr;
-        proxy_set_header   Host $host;
-        
-        proxy_pass         http://127.0.0.1:5010;
-        proxy_redirect     off;
-
-        access_log $GRADIDO_LOG_PATH/nginx-access.federation-5010.log gradido_log;
-        error_log $GRADIDO_LOG_PATH/nginx-error.federation-5010.log warn;
-    }
-
-    location /api/1_1 {
-        proxy_http_version 1.1;
-        proxy_set_header   Upgrade $http_upgrade;
-        proxy_set_header   Connection 'upgrade';
-        proxy_set_header   X-Forwarded-For $remote_addr;
-        proxy_set_header   X-Real-IP  $remote_addr;
-        proxy_set_header   Host $host;
-        
-        proxy_pass         http://127.0.0.1:5011;
-        proxy_redirect     off;
-
-        access_log $GRADIDO_LOG_PATH/nginx-access.federation-5011.log gradido_log;
-        error_log $GRADIDO_LOG_PATH/nginx-error.federation-5011.log warn;
-    }
-
-    location /api/2_0 {
-        proxy_http_version 1.1;
-        proxy_set_header   Upgrade $http_upgrade;
-        proxy_set_header   Connection 'upgrade';
-        proxy_set_header   X-Forwarded-For $remote_addr;
-        proxy_set_header   X-Real-IP  $remote_addr;
-        proxy_set_header   Host $host;
-        
-        proxy_pass         http://127.0.0.1:5020;
-        proxy_redirect     off;
-
-        access_log $GRADIDO_LOG_PATH/nginx-access.federation-5020.log gradido_log;
-        error_log $GRADIDO_LOG_PATH/nginx-error.federation-5020.log warn;
-    }
 
     # TODO this could be a performance optimization
     #location /vue {

deployment/bare_metal/start.sh

@@ -59,8 +59,9 @@ ln -s /etc/nginx/sites-available/update-page.conf /etc/nginx/sites-enabled/
 sudo /etc/init.d/nginx restart
 
 # stop all services
-echo 'Stopping all Gradido services' >> $UPDATE_HTML
+echo 'Stopping and Delete all Gradido services' >> $UPDATE_HTML
-pm2 stop all
+pm2 delete all
+pm2 save
 
 # git
 BRANCH=${1:-master}
@@ -73,12 +74,41 @@ git pull
 export BUILD_COMMIT="$(git rev-parse HEAD)"
 
 # Generate gradido.conf from template
+# *** 1st prepare for each apiversion the federation conf for nginx from federation-template
+# *** set FEDERATION_PORT from FEDERATION_COMMUNITY_APIS and create gradido-federation.conf file
+rm -f $NGINX_CONFIG_DIR/gradido.conf.tmp
+rm -f $NGINX_CONFIG_DIR/gradido-federation.conf
+echo "===================================================================================================="
+IFS="," read -a API_ARRAY <<< $FEDERATION_COMMUNITY_APIS
+for api in "${API_ARRAY[@]}"
+do
+  export FEDERATION_APIVERSION=$api
+  # calculate port by remove '_' and add value of api to baseport
+  port=${api//_/}
+  FEDERATION_PORT=${FEDERATION_COMMUNITY_API_PORT:-5000}
+  FEDERATION_PORT=$(($FEDERATION_PORT + $port))
+  export FEDERATION_PORT
+  echo " create ngingx config: location /api/$FEDERATION_APIVERSION  to  http://127.0.0.1:$FEDERATION_PORT"
+  envsubst '$FEDERATION_APIVERSION, $FEDERATION_PORT' < $NGINX_CONFIG_DIR/gradido-federation.conf.template >> $NGINX_CONFIG_DIR/gradido-federation.conf
+done
+export FEDERATION_APIVERSION=
+export FEDERATION_PORT=
+echo "===================================================================================================="
+
+# *** 2nd read gradido-federation.conf file in env variable to be replaced in 3rd step
+export FEDERATION_NGINX_CONF=$(< $NGINX_CONFIG_DIR/gradido-federation.conf)
+
+# *** 3rd generate gradido nginx config including federation modules per api-version
 echo 'Generate new gradido nginx config' >> $UPDATE_HTML
 case "$NGINX_SSL" in
  true) TEMPLATE_FILE="gradido.conf.ssl.template" ;;
     *) TEMPLATE_FILE="gradido.conf.template" ;;
 esac
-envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $NGINX_CONFIG_DIR/$TEMPLATE_FILE > $NGINX_CONFIG_DIR/gradido.conf
+envsubst '$FEDERATION_NGINX_CONF' < $NGINX_CONFIG_DIR/$TEMPLATE_FILE > $NGINX_CONFIG_DIR/gradido.conf.tmp
+export FEDERATION_NGINX_CONF=
+envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $NGINX_CONFIG_DIR/gradido.conf.tmp > $NGINX_CONFIG_DIR/gradido.conf
+rm $NGINX_CONFIG_DIR/gradido.conf.tmp
+rm $NGINX_CONFIG_DIR/gradido-federation.conf
 
 # Generate update-page.conf from template
 echo 'Generate new update-page nginx config' >> $UPDATE_HTML
@@ -94,11 +124,13 @@ cp -f $PROJECT_ROOT/backend/.env $PROJECT_ROOT/backend/.env.bak
 cp -f $PROJECT_ROOT/frontend/.env $PROJECT_ROOT/frontend/.env.bak
 cp -f $PROJECT_ROOT/admin/.env $PROJECT_ROOT/admin/.env.bak
 cp -f $PROJECT_ROOT/dht-node/.env $PROJECT_ROOT/dht-node/.env.bak
+cp -f $PROJECT_ROOT/federation/.env $PROJECT_ROOT/federation/.env.bak
 envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $PROJECT_ROOT/database/.env.template > $PROJECT_ROOT/database/.env
 envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $PROJECT_ROOT/backend/.env.template > $PROJECT_ROOT/backend/.env
 envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $PROJECT_ROOT/frontend/.env.template > $PROJECT_ROOT/frontend/.env
 envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $PROJECT_ROOT/admin/.env.template > $PROJECT_ROOT/admin/.env
 envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $PROJECT_ROOT/dht-node/.env.template > $PROJECT_ROOT/dht-node/.env
+envsubst "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" < $PROJECT_ROOT/federation/.env.template > $PROJECT_ROOT/federation/.env
 
 # Install & build database
 echo 'Updating database' >> $UPDATE_HTML
@@ -124,7 +156,7 @@ if [ "$DEPLOY_SEED_DATA" = "true" ]; then
 fi
 # TODO maybe handle this differently?
 export NODE_ENV=production
-pm2 delete gradido-backend
+# pm2 delete gradido-backend
 pm2 start --name gradido-backend "yarn --cwd $PROJECT_ROOT/backend start" -l $GRADIDO_LOG_PATH/pm2.backend.$TODAY.log --log-date-format 'YYYY-MM-DD HH:mm:ss.SSS'
 pm2 save
 
@@ -137,7 +169,7 @@ yarn install
 yarn build
 # TODO maybe handle this differently?
 export NODE_ENV=production
-pm2 delete gradido-frontend
+# pm2 delete gradido-frontend
 pm2 start --name gradido-frontend "yarn --cwd $PROJECT_ROOT/frontend start" -l $GRADIDO_LOG_PATH/pm2.frontend.$TODAY.log --log-date-format 'YYYY-MM-DD HH:mm:ss.SSS'
 pm2 save
 
@@ -150,7 +182,7 @@ yarn install
 yarn build
 # TODO maybe handle this differently?
 export NODE_ENV=production
-pm2 delete gradido-admin
+# pm2 delete gradido-admin
 pm2 start --name gradido-admin "yarn --cwd $PROJECT_ROOT/admin start" -l $GRADIDO_LOG_PATH/pm2.admin.$TODAY.log --log-date-format 'YYYY-MM-DD HH:mm:ss.SSS'
 pm2 save
 
@@ -163,17 +195,53 @@ yarn install
 yarn build
 # TODO maybe handle this differently?
 export NODE_ENV=production
-pm2 delete gradido-dht-node
+# pm2 delete gradido-dht-node
 if [ ! -z $FEDERATION_DHT_TOPIC ]; then
   pm2 start --name gradido-dht-node "yarn --cwd $PROJECT_ROOT/dht-node start" -l $GRADIDO_LOG_PATH/pm2.dht-node.$TODAY.log --log-date-format 'YYYY-MM-DD HH:mm:ss.SSS'
+  pm2 save
 else
   echo "====================================================================="
   echo "WARNING: FEDERATION_DHT_TOPIC not configured. DHT-Node not started..."  
   echo "====================================================================="
 fi  
-pm2 save
+
+
+# Install & build federation
+echo 'Updating federation' >> $UPDATE_HTML
+cd $PROJECT_ROOT/federation
+# TODO maybe handle this differently?
+unset NODE_ENV
+yarn install
+yarn build
+# TODO maybe handle this differently?
+export NODE_ENV=production
+
+# set FEDERATION_PORT from FEDERATION_COMMUNITY_APIS
+IFS="," read -a API_ARRAY <<< $FEDERATION_COMMUNITY_APIS
+for api in "${API_ARRAY[@]}"
+do
+  export FEDERATION_API=$api
+  echo "FEDERATION_API=$FEDERATION_API"
+  export MODULENAME=gradido-federation-$api
+  echo "MODULENAME=$MODULENAME"
+  # calculate port by remove '_' and add value of api to baseport
+  port=${api//_/}
+  FEDERATION_PORT=${FEDERATION_COMMUNITY_API_PORT:-5000}
+  FEDERATION_PORT=$(($FEDERATION_PORT + $port))
+  export FEDERATION_PORT
+  echo "===================================================="
+  echo " start $MODULENAME listening on port=$FEDERATION_PORT"
+  echo "===================================================="
+#  pm2 delete $MODULENAME
+  pm2 start --name $MODULENAME "yarn --cwd $PROJECT_ROOT/federation start" -l $GRADIDO_LOG_PATH/pm2.$MODULENAME.$TODAY.log --log-date-format 'YYYY-MM-DD HH:mm:ss.SSS'
+  pm2 save
+done
+
+
+
 
 # let nginx showing gradido
+cd $SCRIPT_DIR
 echo 'Configuring nginx to serve gradido again' >> $UPDATE_HTML
 ln -s /etc/nginx/sites-available/gradido.conf /etc/nginx/sites-enabled/
 rm /etc/nginx/sites-enabled/update-page.conf

dht-node/src/dht_node/index.test.ts

@@ -719,11 +719,11 @@ describe('federation', () => {
                     JSON.stringify([
                       {
                         api: '1_0',
-                        url: 'http://localhost:5001/api/',
+                        url: 'http://localhost/api/',
                       },
                       {
                         api: '2_0',
-                        url: 'http://localhost:5002/api/',
+                        url: 'http://localhost/api/',
                       },
                     ]),
                   ),
@@ -747,7 +747,7 @@ describe('federation', () => {
                       foreign: true,
                       publicKey: expect.any(Buffer),
                       apiVersion: '1_0',
-                      endPoint: 'http://localhost:5001/api/',
+                      endPoint: 'http://localhost/api/',
                       lastAnnouncedAt: expect.any(Date),
                       createdAt: expect.any(Date),
                       updatedAt: null,
@@ -764,7 +764,7 @@ describe('federation', () => {
                       foreign: true,
                       publicKey: expect.any(Buffer),
                       apiVersion: '2_0',
-                      endPoint: 'http://localhost:5002/api/',
+                      endPoint: 'http://localhost/api/',
                       lastAnnouncedAt: expect.any(Date),
                       createdAt: expect.any(Date),
                       updatedAt: null,
@@ -786,15 +786,15 @@ describe('federation', () => {
                   JSON.stringify([
                     {
                       api: '1_0',
-                      url: 'http://localhost:5001/api/',
+                      url: 'http://localhost/api/',
                     },
                     {
                       api: '1_1',
-                      url: 'http://localhost:5002/api/',
+                      url: 'http://localhost/api/',
                     },
                     {
                       api: '2_0',
-                      url: 'http://localhost:5003/api/',
+                      url: 'http://localhost/api/',
                     },
                   ]),
                 ),

dht-node/src/dht_node/index.ts

@@ -181,11 +181,9 @@ export const startDHT = async (topic: string): Promise<void> => {
 
 async function writeHomeCommunityEnries(pubKey: any): Promise<CommunityApi[]> {
   const homeApiVersions: CommunityApi[] = Object.values(ApiVersionType).map(function (apiEnum) {
-    const port =
-      Number.parseInt(CONFIG.FEDERATION_COMMUNITY_API_PORT) + Number(apiEnum.replace('_', ''))
     const comApi: CommunityApi = {
       api: apiEnum,
-      url: CONFIG.FEDERATION_COMMUNITY_URL + ':' + port.toString() + '/api/',
+      url: CONFIG.FEDERATION_COMMUNITY_URL + '/api/',
     }
     return comApi
   })

federation/.env.dist

@@ -0,0 +1,17 @@
+LOG_LEVEL=debug
+PORT=4000
+GRAPHIQL=true
+
+# Database
+DB_PORT=3306
+DB_DATABASE=gradido_community
+
+
+COMMUNITY_NAME=Gradido Entwicklung
+COMMUNITY_URL=http://localhost:4000/
+COMMUNITY_DESCRIPTION=lokale Entwicklungsumgebung
+
+# Federation
+FEDERATION_API=1_0
+FEDERATION_PORT=5010
+FEDERATION_COMMUNITY_URL=http://localhost

federation/.env.dist.template

@@ -0,0 +1,17 @@
+LOG_LEVEL=$LOG_LEVEL
+PORT=$PORT
+GRAPHIQL=$GRAPHIQL
+
+# Database
+DB_PORT=$DB_PORT
+DB_DATABASE=$DB_DATABASE
+
+COMMUNITY_NAME=$COMMUNITY_NAME
+COMMUNITY_URL=$COMMUNITY_URL
+COMMUNITY_DESCRIPTION=$COMMUNITY_DESCRIPTION
+
+# Federation
+FEDERATION_API=$FEDERATION_API
+FEDERATION_PORT=$FEDERATION_PORT
+FEDERATION_COMMUNITY_URL=$FEDERATION_COMMUNITY_URL
+

federation/src/config/index.ts

@@ -73,8 +73,10 @@ if (
 const federation = {
   // FEDERATION_DHT_TOPIC: process.env.FEDERATION_DHT_TOPIC || null,
   // FEDERATION_DHT_SEED: process.env.FEDERATION_DHT_SEED || null,
-  FEDERATION_PORT: process.env.FEDERATION_PORT || 5010,
+  //  FEDERATION_COMMUNITY_API_PORT:
+  //    process.env.FEDERATION_COMMUNITY_API_PORT || 5000,
   FEDERATION_API: process.env.FEDERATION_API || '1_0',
+  FEDERATION_PORT: process.env.FEDERATION_PORT || 5010,
   FEDERATION_COMMUNITY_URL: process.env.FEDERATION_COMMUNITY_URL || null,
 }
 

federation/src/index.ts

@@ -20,7 +20,7 @@ async function main() {
     if (CONFIG.GRAPHIQL) {
       // eslint-disable-next-line no-console
       console.log(
-        `GraphIQL available at http://localhost:${CONFIG.FEDERATION_PORT}`
+        `GraphIQL available at ${CONFIG.FEDERATION_COMMUNITY_URL}/api/${CONFIG.FEDERATION_API}`
       )
     }
   })