Merge remote-tracking branch 'origin/master' into 3110-featuredlt-connector-gradido-transaktionen-auf-tangle-senden-und-empfangen

backend/log4js-config.json

@@ -10,6 +10,7 @@
       {
         "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m"
       },
+      "compress": true,
       "keepFileExt" : true,
       "fileNameSep" : "_",
       "numBackups" : 30
@@ -23,6 +24,7 @@
       {
         "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m"
       },
+      "compress": true,
       "keepFileExt" : true,
       "fileNameSep" : "_",
       "numBackups" : 30
@@ -36,6 +38,7 @@
       {
         "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m"
       },
+      "compress": true,
       "keepFileExt" : true,
       "fileNameSep" : "_",
       "numBackups" : 30
@@ -49,6 +52,7 @@
       {
         "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m"
       },
+      "compress": true,
       "keepFileExt" : true,
       "fileNameSep" : "_",
       "numBackups" : 30
@@ -62,6 +66,7 @@
       {
         "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m %s"
       },
+      "compress": true,
       "keepFileExt" : true,
       "fileNameSep" : "_",
       "numBackups" : 30
@@ -117,7 +122,6 @@
       "appenders":
       [
         "backend",
-        "out",
         "errors"
       ],
       "level": "debug",
@@ -128,7 +132,6 @@
       "appenders":
       [
         "klicktipp",
-        "out",
         "errors"
       ],
       "level": "debug",

backend/src/federation/client/1_0/FederationClient.ts

@@ -25,7 +25,7 @@ export class FederationClient {
   }
 
   getPublicKey = async (): Promise<string | undefined> => {
-    logger.info('Federation: getPublicKey from endpoint', this.endpoint)
+    logger.debug('Federation: getPublicKey from endpoint', this.endpoint)
     try {
       // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
       const { data } = await this.client.rawRequest(getPublicKey, {})
@@ -34,7 +34,7 @@ export class FederationClient {
         logger.warn('Federation: getPublicKey without response data from endpoint', this.endpoint)
         return
       }
-      logger.info(
+      logger.debug(
         'Federation: getPublicKey successful from endpoint',
         this.endpoint,
         // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access

backend/src/federation/validateCommunities.test.ts

@@ -97,7 +97,7 @@ describe('validate Communities', () => {
         expect(logger.debug).toBeCalledWith(`Federation: found 1 dbCommunities`)
       })
       it('logs requestGetPublicKey for community api 1_0 ', () => {
-        expect(logger.info).toBeCalledWith(
+        expect(logger.debug).toBeCalledWith(
           'Federation: getPublicKey from endpoint',
           'http//localhost:5001/api/1_0/',
         )
@@ -148,14 +148,14 @@ describe('validate Communities', () => {
         expect(logger.debug).toBeCalledWith(`Federation: found 1 dbCommunities`)
       })
       it('logs requestGetPublicKey for community api 1_0 ', () => {
-        expect(logger.info).toBeCalledWith(
+        expect(logger.debug).toBeCalledWith(
           'Federation: getPublicKey from endpoint',
           'http//localhost:5001/api/1_0/',
         )
       })
       it('logs community pubKey verified', () => {
-        expect(logger.info).toHaveBeenNthCalledWith(
+        expect(logger.debug).toHaveBeenNthCalledWith(
-          3,
+          6,
           'Federation: verified community with',
           'http//localhost:5001/api/',
         )
@@ -200,13 +200,13 @@ describe('validate Communities', () => {
         expect(logger.debug).toBeCalledWith(`Federation: found 2 dbCommunities`)
       })
       it('logs requestGetPublicKey for community api 1_0 ', () => {
-        expect(logger.info).toBeCalledWith(
+        expect(logger.debug).toBeCalledWith(
           'Federation: getPublicKey from endpoint',
           'http//localhost:5001/api/1_0/',
         )
       })
       it('logs requestGetPublicKey for community api 1_1 ', () => {
-        expect(logger.info).toBeCalledWith(
+        expect(logger.debug).toBeCalledWith(
           'Federation: getPublicKey from endpoint',
           'http//localhost:5001/api/1_1/',
         )
@@ -242,13 +242,13 @@ describe('validate Communities', () => {
         expect(logger.debug).toBeCalledWith(`Federation: found 3 dbCommunities`)
       })
       it('logs requestGetPublicKey for community api 1_0 ', () => {
-        expect(logger.info).toBeCalledWith(
+        expect(logger.debug).toBeCalledWith(
           'Federation: getPublicKey from endpoint',
           'http//localhost:5001/api/1_0/',
         )
       })
       it('logs requestGetPublicKey for community api 1_1 ', () => {
-        expect(logger.info).toBeCalledWith(
+        expect(logger.debug).toBeCalledWith(
           'Federation: getPublicKey from endpoint',
           'http//localhost:5001/api/1_1/',
         )

backend/src/federation/validateCommunities.ts

@@ -10,10 +10,13 @@ import { backendLogger as logger } from '@/server/logger'
 
 import { ApiVersionType } from './enum/apiVersionType'
 
-export function startValidateCommunities(timerInterval: number): void {
+export async function startValidateCommunities(timerInterval: number): Promise<void> {
   logger.info(
     `Federation: startValidateCommunities loop with an interval of ${timerInterval} ms...`,
   )
+  // delete all foreign federated community entries to avoid increasing validation efforts and log-files
+  await DbFederatedCommunity.delete({ foreign: true })
+
   // TODO: replace the timer-loop by an event-based communication to verify announced foreign communities
   // better to use setTimeout twice than setInterval once -> see https://javascript.info/settimeout-setinterval
   setTimeout(function run() {
@@ -45,7 +48,7 @@ export async function validateCommunities(): Promise<void> {
         const pubKey = await client.getPublicKey()
         if (pubKey && pubKey === dbCom.publicKey.toString()) {
           await DbFederatedCommunity.update({ id: dbCom.id }, { verifiedAt: new Date() })
-          logger.info('Federation: verified community with', dbCom.endPoint)
+          logger.debug('Federation: verified community with', dbCom.endPoint)
         } else {
           logger.warn(
             'Federation: received not matching publicKey:',

backend/src/index.ts

@@ -13,7 +13,7 @@ async function main() {
       console.log(`GraphIQL available at http://localhost:${CONFIG.PORT}`)
     }
   })
-  startValidateCommunities(Number(CONFIG.FEDERATION_VALIDATE_COMMUNITY_TIMER))
+  void startValidateCommunities(Number(CONFIG.FEDERATION_VALIDATE_COMMUNITY_TIMER))
 }
 
 main().catch((e) => {

deployment/bare_metal/removeLogFiles.sh

@@ -0,0 +1,20 @@
+find /home/gradido/gradido/deployment/bare_metal/log -name "pm2.admin.????-??-??.log*" -ctime +30 -exec rm -r {} \; > /dev/null
+find /home/gradido/gradido/deployment/bare_metal/log -name "pm2.backend.????-??-??.log*" -ctime +30 -exec rm -r {} \; > /dev/null
+find /home/gradido/gradido/deployment/bare_metal/log -name "pm2.dht-node.????-??-??.log*" -ctime +30 -exec rm -r {} \; > /dev/null
+find /home/gradido/gradido/deployment/bare_metal/log -name "pm2.frontend.????-??-??.log*" -ctime +30 -exec rm -r {} \; > /dev/null
+find /home/gradido/gradido/deployment/bare_metal/log -name "pm2.gradido-federation-*.????-??-??.log*" -ctime +30 -exec rm -r {} \; > /dev/null
+find /home/gradido/gradido/deployment/bare_metal/log -name "typeorm.backend.log.*" -atime +30 -exec rm -r {} \; > /dev/null
+
+find /home/gradido/gradido/logs/backend -name "access_*" -ctime +30 -exec rm -r {} \; > /dev/null
+find /home/gradido/gradido/logs/backend -name "apollo_*" -ctime +30 -exec rm -r {} \; > /dev/null
+find /home/gradido/gradido/logs/backend -name "backend_*" -ctime +30 -exec rm -r {} \; > /dev/null
+find /home/gradido/gradido/logs/backend -name "errors_*" -ctime +30 -exec rm -r {} \; > /dev/null
+find /home/gradido/gradido/logs/backend -name "klicktipp_*" -ctime +30 -exec rm -r {} \; > /dev/null
+
+find /home/gradido/gradido/logs/dht-node -name "dht-*" -ctime +30 -exec rm -r {} \; > /dev/null
+find /home/gradido/gradido/logs/dht-node -name "errors_*" -ctime +30 -exec rm -r {} \; > /dev/null
+
+find /home/gradido/gradido/logs/federation -name "access-*_????-??-??.*" -ctime +30 -exec rm -r {} \; > /dev/null
+find /home/gradido/gradido/logs/federation -name "apiversion-*_????-??-??.*" -ctime +30 -exec rm -r {} \; > /dev/null
+find /home/gradido/gradido/logs/federation -name "apollo-*_????-??-??.*" -ctime +30 -exec rm -r {} \; > /dev/null
+find /home/gradido/gradido/logs/federation -name "errors-*_????-??-??.*" -ctime +30 -exec rm -r {} \; > /dev/null

deployment/bare_metal/setup.md

@@ -224,9 +224,8 @@ to start the existing backup.sh script.
 
 ### On production / stage3 / stage2
 
-To check for existing cronjobs for the `gradido` user, please
+#### Check existing cronjobs
-
+To check for existing cronjobs for the `gradido` user, please process
-Run:
 
 ```bash
 crontab -l
@@ -234,10 +233,9 @@ crontab -l
 
 This show all existing entries of the crontab for user `gradido`
 
-To install/add the cronjob for a daily backup at 3:00am please,
+#### Add or install new cronjob
-To install/add the cronjob for a daily klicktipp export at 4:00am please,
+- Database Backup
-
+To install/add the cronjob for a daily backup at 3:00am please process
-Run:
 
 ```bash
 crontab -e
@@ -245,5 +243,26 @@ crontab -e
 and insert the following line
 ```bash
 0 3 * * * ~/gradido/deployment/bare_metal/backup.sh
+```
+
+- Logfile clearance
+To install/add the cronjob for a daily logfile clearance at 3:15 please process
+
+```bash
+crontab -e
+```
+and insert the following line
+```bash
+15 3 * * * ~/gradido/deployment/bare_metal/removeLogFiles.sh
+```
+
+- Klicktipp campains
+To install/add the cronjob for a daily klicktipp export at 4:00am please process
+
+```bash
+crontab -e
+```
+and insert the following line
+```bash
 0 4 * * * cd ~/gradido/backend/ && yarn klicktipp && cd
 ```

dht-node/log4js-config.json

@@ -10,6 +10,7 @@
       {
         "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] -  %m"
       },
+      "compress": true,
       "keepFileExt" : true,
       "fileNameSep" : "_",
       "numBackups" : 30
@@ -23,6 +24,7 @@
       {
         "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] -  %m"
       },
+      "compress": true,
       "keepFileExt" : true,
       "fileNameSep" : "_",
       "numBackups" : 30

dlt-connector/log4js-config.json

@@ -10,6 +10,7 @@
       {
         "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] -  %m"
       },
+      "compress": true,
       "keepFileExt" : true,
       "fileNameSep" : "_",
       "numBackups" : 30
@@ -23,6 +24,7 @@
       {
         "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] -  %m"
       },
+      "compress": true,
       "keepFileExt" : true,
       "fileNameSep" : "_",
       "numBackups" : 30

federation/log4js-config.json

@@ -10,6 +10,7 @@
       {
         "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] -  %m"
       },
+      "compress": true,
       "keepFileExt" : true,
       "fileNameSep" : "_",
       "numBackups" : 30
@@ -23,6 +24,7 @@
       {
         "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] -  %m"
       },
+      "compress": true,
       "keepFileExt" : true,
       "fileNameSep" : "_",
       "numBackups" : 30
@@ -36,6 +38,7 @@
       {
         "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] -  %m"
       },
+      "compress": true,
       "keepFileExt" : true,
       "fileNameSep" : "_",
       "numBackups" : 30
@@ -49,6 +52,7 @@
       {
         "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] -  %m"
       },
+      "compress": true,
       "keepFileExt" : true,
       "fileNameSep" : "_",
       "numBackups" : 30
@@ -62,6 +66,7 @@
       {
         "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] -  %m"
       },
+      "compress": true,
       "keepFileExt" : true,
       "fileNameSep" : "_",
       "numBackups" : 30