diff --git a/backend/log4js-config.json b/backend/log4js-config.json index 160883eb3..1cbd4519c 100644 --- a/backend/log4js-config.json +++ b/backend/log4js-config.json @@ -10,6 +10,7 @@ { "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m" }, + "compress": true, "keepFileExt" : true, "fileNameSep" : "_", "numBackups" : 30 @@ -23,6 +24,7 @@ { "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m" }, + "compress": true, "keepFileExt" : true, "fileNameSep" : "_", "numBackups" : 30 @@ -36,6 +38,7 @@ { "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m" }, + "compress": true, "keepFileExt" : true, "fileNameSep" : "_", "numBackups" : 30 @@ -49,6 +52,7 @@ { "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m" }, + "compress": true, "keepFileExt" : true, "fileNameSep" : "_", "numBackups" : 30 @@ -62,6 +66,7 @@ { "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m %s" }, + "compress": true, "keepFileExt" : true, "fileNameSep" : "_", "numBackups" : 30 @@ -117,7 +122,6 @@ "appenders": [ "backend", - "out", "errors" ], "level": "debug", @@ -128,7 +132,6 @@ "appenders": [ "klicktipp", - "out", "errors" ], "level": "debug", diff --git a/backend/src/federation/client/1_0/FederationClient.ts b/backend/src/federation/client/1_0/FederationClient.ts index ba446abe8..ff223b98a 100644 --- a/backend/src/federation/client/1_0/FederationClient.ts +++ b/backend/src/federation/client/1_0/FederationClient.ts @@ -25,7 +25,7 @@ export class FederationClient { } getPublicKey = async (): Promise => { - logger.info('Federation: getPublicKey from endpoint', this.endpoint) + logger.debug('Federation: getPublicKey from endpoint', this.endpoint) try { // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment const { data } = await this.client.rawRequest(getPublicKey, {}) @@ -34,7 +34,7 @@ export class FederationClient { logger.warn('Federation: getPublicKey without response data from endpoint', this.endpoint) return } - logger.info( + logger.debug( 'Federation: getPublicKey successful from endpoint', this.endpoint, // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access diff --git a/backend/src/federation/validateCommunities.test.ts b/backend/src/federation/validateCommunities.test.ts index ef2c30d6b..f6b3b3080 100644 --- a/backend/src/federation/validateCommunities.test.ts +++ b/backend/src/federation/validateCommunities.test.ts @@ -97,7 +97,7 @@ describe('validate Communities', () => { expect(logger.debug).toBeCalledWith(`Federation: found 1 dbCommunities`) }) it('logs requestGetPublicKey for community api 1_0 ', () => { - expect(logger.info).toBeCalledWith( + expect(logger.debug).toBeCalledWith( 'Federation: getPublicKey from endpoint', 'http//localhost:5001/api/1_0/', ) @@ -148,14 +148,14 @@ describe('validate Communities', () => { expect(logger.debug).toBeCalledWith(`Federation: found 1 dbCommunities`) }) it('logs requestGetPublicKey for community api 1_0 ', () => { - expect(logger.info).toBeCalledWith( + expect(logger.debug).toBeCalledWith( 'Federation: getPublicKey from endpoint', 'http//localhost:5001/api/1_0/', ) }) it('logs community pubKey verified', () => { - expect(logger.info).toHaveBeenNthCalledWith( - 3, + expect(logger.debug).toHaveBeenNthCalledWith( + 6, 'Federation: verified community with', 'http//localhost:5001/api/', ) @@ -200,13 +200,13 @@ describe('validate Communities', () => { expect(logger.debug).toBeCalledWith(`Federation: found 2 dbCommunities`) }) it('logs requestGetPublicKey for community api 1_0 ', () => { - expect(logger.info).toBeCalledWith( + expect(logger.debug).toBeCalledWith( 'Federation: getPublicKey from endpoint', 'http//localhost:5001/api/1_0/', ) }) it('logs requestGetPublicKey for community api 1_1 ', () => { - expect(logger.info).toBeCalledWith( + expect(logger.debug).toBeCalledWith( 'Federation: getPublicKey from endpoint', 'http//localhost:5001/api/1_1/', ) @@ -242,13 +242,13 @@ describe('validate Communities', () => { expect(logger.debug).toBeCalledWith(`Federation: found 3 dbCommunities`) }) it('logs requestGetPublicKey for community api 1_0 ', () => { - expect(logger.info).toBeCalledWith( + expect(logger.debug).toBeCalledWith( 'Federation: getPublicKey from endpoint', 'http//localhost:5001/api/1_0/', ) }) it('logs requestGetPublicKey for community api 1_1 ', () => { - expect(logger.info).toBeCalledWith( + expect(logger.debug).toBeCalledWith( 'Federation: getPublicKey from endpoint', 'http//localhost:5001/api/1_1/', ) diff --git a/backend/src/federation/validateCommunities.ts b/backend/src/federation/validateCommunities.ts index 91c6ee724..f94774e53 100644 --- a/backend/src/federation/validateCommunities.ts +++ b/backend/src/federation/validateCommunities.ts @@ -10,10 +10,13 @@ import { backendLogger as logger } from '@/server/logger' import { ApiVersionType } from './enum/apiVersionType' -export function startValidateCommunities(timerInterval: number): void { +export async function startValidateCommunities(timerInterval: number): Promise { logger.info( `Federation: startValidateCommunities loop with an interval of ${timerInterval} ms...`, ) + // delete all foreign federated community entries to avoid increasing validation efforts and log-files + await DbFederatedCommunity.delete({ foreign: true }) + // TODO: replace the timer-loop by an event-based communication to verify announced foreign communities // better to use setTimeout twice than setInterval once -> see https://javascript.info/settimeout-setinterval setTimeout(function run() { @@ -45,7 +48,7 @@ export async function validateCommunities(): Promise { const pubKey = await client.getPublicKey() if (pubKey && pubKey === dbCom.publicKey.toString()) { await DbFederatedCommunity.update({ id: dbCom.id }, { verifiedAt: new Date() }) - logger.info('Federation: verified community with', dbCom.endPoint) + logger.debug('Federation: verified community with', dbCom.endPoint) } else { logger.warn( 'Federation: received not matching publicKey:', diff --git a/backend/src/index.ts b/backend/src/index.ts index 6e0191155..86f78326d 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -13,7 +13,7 @@ async function main() { console.log(`GraphIQL available at http://localhost:${CONFIG.PORT}`) } }) - startValidateCommunities(Number(CONFIG.FEDERATION_VALIDATE_COMMUNITY_TIMER)) + void startValidateCommunities(Number(CONFIG.FEDERATION_VALIDATE_COMMUNITY_TIMER)) } main().catch((e) => { diff --git a/deployment/bare_metal/removeLogFiles.sh b/deployment/bare_metal/removeLogFiles.sh new file mode 100755 index 000000000..82e4717e3 --- /dev/null +++ b/deployment/bare_metal/removeLogFiles.sh @@ -0,0 +1,20 @@ +find /home/gradido/gradido/deployment/bare_metal/log -name "pm2.admin.????-??-??.log*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/deployment/bare_metal/log -name "pm2.backend.????-??-??.log*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/deployment/bare_metal/log -name "pm2.dht-node.????-??-??.log*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/deployment/bare_metal/log -name "pm2.frontend.????-??-??.log*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/deployment/bare_metal/log -name "pm2.gradido-federation-*.????-??-??.log*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/deployment/bare_metal/log -name "typeorm.backend.log.*" -atime +30 -exec rm -r {} \; > /dev/null + +find /home/gradido/gradido/logs/backend -name "access_*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/logs/backend -name "apollo_*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/logs/backend -name "backend_*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/logs/backend -name "errors_*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/logs/backend -name "klicktipp_*" -ctime +30 -exec rm -r {} \; > /dev/null + +find /home/gradido/gradido/logs/dht-node -name "dht-*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/logs/dht-node -name "errors_*" -ctime +30 -exec rm -r {} \; > /dev/null + +find /home/gradido/gradido/logs/federation -name "access-*_????-??-??.*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/logs/federation -name "apiversion-*_????-??-??.*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/logs/federation -name "apollo-*_????-??-??.*" -ctime +30 -exec rm -r {} \; > /dev/null +find /home/gradido/gradido/logs/federation -name "errors-*_????-??-??.*" -ctime +30 -exec rm -r {} \; > /dev/null diff --git a/deployment/bare_metal/setup.md b/deployment/bare_metal/setup.md index 717dd8547..2e4105979 100644 --- a/deployment/bare_metal/setup.md +++ b/deployment/bare_metal/setup.md @@ -224,9 +224,8 @@ to start the existing backup.sh script. ### On production / stage3 / stage2 -To check for existing cronjobs for the `gradido` user, please - -Run: +#### Check existing cronjobs +To check for existing cronjobs for the `gradido` user, please process ```bash crontab -l @@ -234,10 +233,9 @@ crontab -l This show all existing entries of the crontab for user `gradido` -To install/add the cronjob for a daily backup at 3:00am please, -To install/add the cronjob for a daily klicktipp export at 4:00am please, - -Run: +#### Add or install new cronjob +- Database Backup +To install/add the cronjob for a daily backup at 3:00am please process ```bash crontab -e @@ -245,5 +243,26 @@ crontab -e and insert the following line ```bash 0 3 * * * ~/gradido/deployment/bare_metal/backup.sh +``` + +- Logfile clearance +To install/add the cronjob for a daily logfile clearance at 3:15 please process + +```bash +crontab -e +``` +and insert the following line +```bash +15 3 * * * ~/gradido/deployment/bare_metal/removeLogFiles.sh +``` + +- Klicktipp campains +To install/add the cronjob for a daily klicktipp export at 4:00am please process + +```bash +crontab -e +``` +and insert the following line +```bash 0 4 * * * cd ~/gradido/backend/ && yarn klicktipp && cd ``` diff --git a/dht-node/log4js-config.json b/dht-node/log4js-config.json index 3e5cc59ec..ee5207550 100644 --- a/dht-node/log4js-config.json +++ b/dht-node/log4js-config.json @@ -10,6 +10,7 @@ { "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m" }, + "compress": true, "keepFileExt" : true, "fileNameSep" : "_", "numBackups" : 30 @@ -23,6 +24,7 @@ { "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m" }, + "compress": true, "keepFileExt" : true, "fileNameSep" : "_", "numBackups" : 30 diff --git a/dlt-connector/log4js-config.json b/dlt-connector/log4js-config.json index d67db87c8..8b2f1b348 100644 --- a/dlt-connector/log4js-config.json +++ b/dlt-connector/log4js-config.json @@ -10,6 +10,7 @@ { "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m" }, + "compress": true, "keepFileExt" : true, "fileNameSep" : "_", "numBackups" : 30 @@ -23,6 +24,7 @@ { "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m" }, + "compress": true, "keepFileExt" : true, "fileNameSep" : "_", "numBackups" : 30 diff --git a/federation/log4js-config.json b/federation/log4js-config.json index f9fbc8bf5..e9336f83c 100644 --- a/federation/log4js-config.json +++ b/federation/log4js-config.json @@ -10,6 +10,7 @@ { "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m" }, + "compress": true, "keepFileExt" : true, "fileNameSep" : "_", "numBackups" : 30 @@ -23,6 +24,7 @@ { "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m" }, + "compress": true, "keepFileExt" : true, "fileNameSep" : "_", "numBackups" : 30 @@ -36,6 +38,7 @@ { "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m" }, + "compress": true, "keepFileExt" : true, "fileNameSep" : "_", "numBackups" : 30 @@ -49,6 +52,7 @@ { "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m" }, + "compress": true, "keepFileExt" : true, "fileNameSep" : "_", "numBackups" : 30 @@ -62,6 +66,7 @@ { "type": "pattern", "pattern": "%d{ISO8601} %p %c [%X{user}] [%f : %l] - %m" }, + "compress": true, "keepFileExt" : true, "fileNameSep" : "_", "numBackups" : 30