From fca7283671741bb445ead6d6d7496d9977d0405b Mon Sep 17 00:00:00 2001 From: Claus-Peter Huebner Date: Wed, 2 Aug 2023 23:14:49 +0200 Subject: [PATCH 01/11] delete foreign federated community entries on startCommunityValidation --- backend/src/federation/validateCommunities.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/src/federation/validateCommunities.ts b/backend/src/federation/validateCommunities.ts index 91c6ee724..f4b52c5dc 100644 --- a/backend/src/federation/validateCommunities.ts +++ b/backend/src/federation/validateCommunities.ts @@ -10,10 +10,13 @@ import { backendLogger as logger } from '@/server/logger' import { ApiVersionType } from './enum/apiVersionType' -export function startValidateCommunities(timerInterval: number): void { +export async function startValidateCommunities(timerInterval: number): Promise { logger.info( `Federation: startValidateCommunities loop with an interval of ${timerInterval} ms...`, ) + // delete all foreign federated community entries to avoid increasing validation efforts and log-files + await DbFederatedCommunity.delete({ foreign: true }) + // TODO: replace the timer-loop by an event-based communication to verify announced foreign communities // better to use setTimeout twice than setInterval once -> see https://javascript.info/settimeout-setinterval setTimeout(function run() { From 678517ca6bf172357b002fba5bdb0cdb5d2099a0 Mon Sep 17 00:00:00 2001 From: Claus-Peter Huebner Date: Wed, 2 Aug 2023 23:31:30 +0200 Subject: [PATCH 02/11] switch some log statements to debug level --- backend/src/federation/client/1_0/FederationClient.ts | 4 ++-- backend/src/federation/validateCommunities.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/federation/client/1_0/FederationClient.ts b/backend/src/federation/client/1_0/FederationClient.ts index ba446abe8..ff223b98a 100644 --- a/backend/src/federation/client/1_0/FederationClient.ts +++ b/backend/src/federation/client/1_0/FederationClient.ts @@ -25,7 +25,7 @@ export class FederationClient { } getPublicKey = async (): Promise => { - logger.info('Federation: getPublicKey from endpoint', this.endpoint) + logger.debug('Federation: getPublicKey from endpoint', this.endpoint) try { // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment const { data } = await this.client.rawRequest(getPublicKey, {}) @@ -34,7 +34,7 @@ export class FederationClient { logger.warn('Federation: getPublicKey without response data from endpoint', this.endpoint) return } - logger.info( + logger.debug( 'Federation: getPublicKey successful from endpoint', this.endpoint, // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access diff --git a/backend/src/federation/validateCommunities.ts b/backend/src/federation/validateCommunities.ts index f4b52c5dc..f94774e53 100644 --- a/backend/src/federation/validateCommunities.ts +++ b/backend/src/federation/validateCommunities.ts @@ -48,7 +48,7 @@ export async function validateCommunities(): Promise { const pubKey = await client.getPublicKey() if (pubKey && pubKey === dbCom.publicKey.toString()) { await DbFederatedCommunity.update({ id: dbCom.id }, { verifiedAt: new Date() }) - logger.info('Federation: verified community with', dbCom.endPoint) + logger.debug('Federation: verified community with', dbCom.endPoint) } else { logger.warn( 'Federation: received not matching publicKey:', From b0a4452ccb9523e8409c022d3c1213d665156802 Mon Sep 17 00:00:00 2001 From: Claus-Peter Huebner Date: Wed, 2 Aug 2023 23:38:36 +0200 Subject: [PATCH 03/11] adapt testcases on changed loglevels --- backend/src/federation/validateCommunities.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/src/federation/validateCommunities.test.ts b/backend/src/federation/validateCommunities.test.ts index ef2c30d6b..9fa1831ac 100644 --- a/backend/src/federation/validateCommunities.test.ts +++ b/backend/src/federation/validateCommunities.test.ts @@ -97,7 +97,7 @@ describe('validate Communities', () => { expect(logger.debug).toBeCalledWith(`Federation: found 1 dbCommunities`) }) it('logs requestGetPublicKey for community api 1_0 ', () => { - expect(logger.info).toBeCalledWith( + expect(logger.debug).toBeCalledWith( 'Federation: getPublicKey from endpoint', 'http//localhost:5001/api/1_0/', ) @@ -148,7 +148,7 @@ describe('validate Communities', () => { expect(logger.debug).toBeCalledWith(`Federation: found 1 dbCommunities`) }) it('logs requestGetPublicKey for community api 1_0 ', () => { - expect(logger.info).toBeCalledWith( + expect(logger.debug).toBeCalledWith( 'Federation: getPublicKey from endpoint', 'http//localhost:5001/api/1_0/', ) @@ -200,13 +200,13 @@ describe('validate Communities', () => { expect(logger.debug).toBeCalledWith(`Federation: found 2 dbCommunities`) }) it('logs requestGetPublicKey for community api 1_0 ', () => { - expect(logger.info).toBeCalledWith( + expect(logger.debug).toBeCalledWith( 'Federation: getPublicKey from endpoint', 'http//localhost:5001/api/1_0/', ) }) it('logs requestGetPublicKey for community api 1_1 ', () => { - expect(logger.info).toBeCalledWith( + expect(logger.debug).toBeCalledWith( 'Federation: getPublicKey from endpoint', 'http//localhost:5001/api/1_1/', ) @@ -242,13 +242,13 @@ describe('validate Communities', () => { expect(logger.debug).toBeCalledWith(`Federation: found 3 dbCommunities`) }) it('logs requestGetPublicKey for community api 1_0 ', () => { - expect(logger.info).toBeCalledWith( + expect(logger.debug).toBeCalledWith( 'Federation: getPublicKey from endpoint', 'http//localhost:5001/api/1_0/', ) }) it('logs requestGetPublicKey for community api 1_1 ', () => { - expect(logger.info).toBeCalledWith( + expect(logger.debug).toBeCalledWith( 'Federation: getPublicKey from endpoint', 'http//localhost:5001/api/1_1/', ) From 5704f6f3fe3471d96d374e254a01a84cf77cb016 Mon Sep 17 00:00:00 2001 From: Claus-Peter Huebner Date: Wed, 2 Aug 2023 23:51:07 +0200 Subject: [PATCH 04/11] remove 'out' from appenders in category backend and klicktipp --- backend/log4js-config.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/log4js-config.json b/backend/log4js-config.json index e614520df..1cbd4519c 100644 --- a/backend/log4js-config.json +++ b/backend/log4js-config.json @@ -122,7 +122,6 @@ "appenders": [ "backend", - "out", "errors" ], "level": "debug", @@ -133,7 +132,6 @@ "appenders": [ "klicktipp", - "out", "errors" ], "level": "debug", From 3470a38efba908fe2f0ae81452ccbee52c4b8cd6 Mon Sep 17 00:00:00 2001 From: Claus-Peter Huebner Date: Wed, 2 Aug 2023 23:54:33 +0200 Subject: [PATCH 05/11] linting --- backend/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index 6e0191155..86f78326d 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -13,7 +13,7 @@ async function main() { console.log(`GraphIQL available at http://localhost:${CONFIG.PORT}`) } }) - startValidateCommunities(Number(CONFIG.FEDERATION_VALIDATE_COMMUNITY_TIMER)) + void startValidateCommunities(Number(CONFIG.FEDERATION_VALIDATE_COMMUNITY_TIMER)) } main().catch((e) => { From 1f770cc3d7b3354b44291bac236c1dd3ce6499f0 Mon Sep 17 00:00:00 2001 From: Claus-Peter Huebner Date: Thu, 3 Aug 2023 00:09:08 +0200 Subject: [PATCH 06/11] adapt forgotten testcase on changed loglevel --- backend/src/federation/validateCommunities.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/federation/validateCommunities.test.ts b/backend/src/federation/validateCommunities.test.ts index 9fa1831ac..a8d555462 100644 --- a/backend/src/federation/validateCommunities.test.ts +++ b/backend/src/federation/validateCommunities.test.ts @@ -154,7 +154,7 @@ describe('validate Communities', () => { ) }) it('logs community pubKey verified', () => { - expect(logger.info).toHaveBeenNthCalledWith( + expect(logger.debug).toHaveBeenNthCalledWith( 3, 'Federation: verified community with', 'http//localhost:5001/api/', From b9a1c893d7fc87108a98b9ff40d6de5151848155 Mon Sep 17 00:00:00 2001 From: Claus-Peter Huebner Date: Thu, 3 Aug 2023 00:36:59 +0200 Subject: [PATCH 07/11] correct testcase NthcalledWith --- backend/src/federation/validateCommunities.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/federation/validateCommunities.test.ts b/backend/src/federation/validateCommunities.test.ts index a8d555462..f6b3b3080 100644 --- a/backend/src/federation/validateCommunities.test.ts +++ b/backend/src/federation/validateCommunities.test.ts @@ -155,7 +155,7 @@ describe('validate Communities', () => { }) it('logs community pubKey verified', () => { expect(logger.debug).toHaveBeenNthCalledWith( - 3, + 6, 'Federation: verified community with', 'http//localhost:5001/api/', ) From cbb427e3592dbc31fec5becfc0ab30f7cd9b7c39 Mon Sep 17 00:00:00 2001 From: Claus-Peter Huebner Date: Thu, 3 Aug 2023 21:39:44 +0200 Subject: [PATCH 08/11] add script to remove log files older than 30 days --- deployment/bare_metal/removeLogFiles.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 deployment/bare_metal/removeLogFiles.sh diff --git a/deployment/bare_metal/removeLogFiles.sh b/deployment/bare_metal/removeLogFiles.sh new file mode 100644 index 000000000..f55a567b8 --- /dev/null +++ b/deployment/bare_metal/removeLogFiles.sh @@ -0,0 +1,16 @@ +find /home/gradido/gradido/deployment/bare_metal/log -name "pm2.*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/deployment/bare_metal/log -name "nginx-*" -ctime +30 -exec rm -r {} \; > /dev/null + +find /home/gradido/gradido/logs/backend -name "access_*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/logs/backend -name "apollo_*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/logs/backend -name "backend_*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/logs/backend -name "errors_*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/logs/backend -name "klicktipp_*" -ctime +30 -exec rm -r {} \; > /dev/null + +find /home/gradido/gradido/logs/dht-node -name "dht-*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/logs/dht-node -name "errors_*" -ctime +30 -exec rm -r {} \; > /dev/null + +find /home/gradido/gradido/logs/federation -name "access-*_????-??-??.*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/logs/federation -name "apiversion-*_????-??-??.*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/logs/federation -name "apollo-*_????-??-??.*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/logs/federation -name "errors-*_????-??-??.*" -ctime +30 -exec rm -r {} \; > /dev/null From 52014be46f8ab3e8390a94b6ba068fc12a8954c5 Mon Sep 17 00:00:00 2001 From: Claus-Peter Huebner Date: Thu, 3 Aug 2023 22:06:49 +0200 Subject: [PATCH 09/11] remove no nginx-files, but more detailed pm2-files --- deployment/bare_metal/removeLogFiles.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/deployment/bare_metal/removeLogFiles.sh b/deployment/bare_metal/removeLogFiles.sh index f55a567b8..75a54fa60 100644 --- a/deployment/bare_metal/removeLogFiles.sh +++ b/deployment/bare_metal/removeLogFiles.sh @@ -1,5 +1,9 @@ -find /home/gradido/gradido/deployment/bare_metal/log -name "pm2.*" -ctime +30 -exec rm -r {} \; > /dev/null -find /home/gradido/gradido/deployment/bare_metal/log -name "nginx-*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/deployment/bare_metal/log -name "pm2.admin.????-??-??.log*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/deployment/bare_metal/log -name "pm2.backend.????-??-??.log*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/deployment/bare_metal/log -name "pm2.dht-node.????-??-??.log*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/deployment/bare_metal/log -name "pm2.frontend.??-??-??.log*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/deployment/bare_metal/log -name "pm2.gradido-federation-*.????-??-??.log*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/deployment/bare_metal/log -name "typeorm.backend.log.*" -ctime +30 -exec rm -r {} \; > /dev/null find /home/gradido/gradido/logs/backend -name "access_*" -ctime +30 -exec rm -r {} \; > /dev/null find /home/gradido/gradido/logs/backend -name "apollo_*" -ctime +30 -exec rm -r {} \; > /dev/null From 9cdd6dbba8c1e8480d5d36c0c7867abfba4e1dd6 Mon Sep 17 00:00:00 2001 From: Claus-Peter Huebner Date: Thu, 3 Aug 2023 22:30:11 +0200 Subject: [PATCH 10/11] setup description for new logfile clearance cronjob --- deployment/bare_metal/setup.md | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/deployment/bare_metal/setup.md b/deployment/bare_metal/setup.md index 717dd8547..2e4105979 100644 --- a/deployment/bare_metal/setup.md +++ b/deployment/bare_metal/setup.md @@ -224,9 +224,8 @@ to start the existing backup.sh script. ### On production / stage3 / stage2 -To check for existing cronjobs for the `gradido` user, please - -Run: +#### Check existing cronjobs +To check for existing cronjobs for the `gradido` user, please process ```bash crontab -l @@ -234,10 +233,9 @@ crontab -l This show all existing entries of the crontab for user `gradido` -To install/add the cronjob for a daily backup at 3:00am please, -To install/add the cronjob for a daily klicktipp export at 4:00am please, - -Run: +#### Add or install new cronjob +- Database Backup +To install/add the cronjob for a daily backup at 3:00am please process ```bash crontab -e @@ -245,5 +243,26 @@ crontab -e and insert the following line ```bash 0 3 * * * ~/gradido/deployment/bare_metal/backup.sh +``` + +- Logfile clearance +To install/add the cronjob for a daily logfile clearance at 3:15 please process + +```bash +crontab -e +``` +and insert the following line +```bash +15 3 * * * ~/gradido/deployment/bare_metal/removeLogFiles.sh +``` + +- Klicktipp campains +To install/add the cronjob for a daily klicktipp export at 4:00am please process + +```bash +crontab -e +``` +and insert the following line +```bash 0 4 * * * cd ~/gradido/backend/ && yarn klicktipp && cd ``` From c35a6512021242b993c1993ec575346dd089ddeb Mon Sep 17 00:00:00 2001 From: Claus-Peter Huebner Date: Thu, 3 Aug 2023 22:51:51 +0200 Subject: [PATCH 11/11] make the script executable --- deployment/bare_metal/removeLogFiles.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 deployment/bare_metal/removeLogFiles.sh diff --git a/deployment/bare_metal/removeLogFiles.sh b/deployment/bare_metal/removeLogFiles.sh old mode 100644 new mode 100755