From 4afa24ed1840eccad35c8e1ab9503367216347cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus-Peter=20H=C3=BCbner?= Date: Fri, 30 Dec 2022 02:35:18 +0100 Subject: [PATCH 001/124] add graphql-client for requestGetPublicKey --- backend/package.json | 1 + .../federation/client/1_0/FederationClient.ts | 50 +++++++++++++++++++ .../graphql/1_0/model/FdCommunity.ts | 41 +++++++++++++++ backend/yarn.lock | 29 ++++++++++- 4 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 backend/src/federation/client/1_0/FederationClient.ts create mode 100644 backend/src/federation/graphql/1_0/model/FdCommunity.ts diff --git a/backend/package.json b/backend/package.json index 69a436563..33926f45b 100644 --- a/backend/package.json +++ b/backend/package.json @@ -30,6 +30,7 @@ "email-templates": "^10.0.1", "express": "^4.17.1", "graphql": "^15.5.1", + "graphql-request": "5.0.0", "i18n": "^0.15.1", "jest": "^27.2.4", "jsonwebtoken": "^8.5.1", diff --git a/backend/src/federation/client/1_0/FederationClient.ts b/backend/src/federation/client/1_0/FederationClient.ts new file mode 100644 index 000000000..12b5fdac0 --- /dev/null +++ b/backend/src/federation/client/1_0/FederationClient.ts @@ -0,0 +1,50 @@ +import { GraphQLClient, gql } from 'graphql-request' +import { backendLogger as logger } from '@/server/logger' +import { FdCommunity } from '@/federation/graphql/1_0/model/FdCommunity' + +export async function requestGetPublicKey(fdCom: FdCommunity): Promise { + let endpoint = fdCom.url.endsWith('/') ? fdCom.url : fdCom.url + '/' + endpoint = `${endpoint}graphql/${fdCom.apiVersion}/getPublicKey` + logger.info(`requestGetPublicKey with endpoint='${endpoint}'...`) + + const graphQLClient = new GraphQLClient(endpoint, { + method: 'GET', + jsonSerializer: { + parse: JSON.parse, + stringify: JSON.stringify, + }, + }) + logger.info(`graphQLClient=${JSON.stringify(graphQLClient)}`) + const query = gql` + query { + getPublicKey { + publicKey + } + } + ` + + const variables = {} + + try { + const { data, errors, extensions, headers, status } = await graphQLClient.rawRequest( + query, + variables, + ) + logger.debug( + `Response-Data: ${JSON.stringify( + { data, errors, extensions, headers, status }, + undefined, + 2, + )}`, + ) + if (data) { + logger.debug(`Response-PublicKey: ${data.getPublicKey.publicKey}`) + logger.info(`requestGetPublicKey processed successfully`) + return data.getPublicKey.publicKey + } + logger.warn(`requestGetPublicKey processed without response data`) + return undefined + } catch (err) { + logger.error(`Request-Error: ${JSON.stringify(err)}`) + } +} diff --git a/backend/src/federation/graphql/1_0/model/FdCommunity.ts b/backend/src/federation/graphql/1_0/model/FdCommunity.ts new file mode 100644 index 000000000..cf0ce3388 --- /dev/null +++ b/backend/src/federation/graphql/1_0/model/FdCommunity.ts @@ -0,0 +1,41 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ +import { Community as DbCommunity } from '@entity/Community' +import { ObjectType, Field } from 'type-graphql' + +@ObjectType() +export class FdCommunity { + constructor(dbCommunity: DbCommunity) { + this.apiVersion = dbCommunity.apiVersion + this.createdAt = dbCommunity.createdAt + this.id = dbCommunity.id + this.lastAnnouncedAt = dbCommunity.lastAnnouncedAt + this.publicKey = dbCommunity.publicKey.toString('hex') + this.updatedAt = dbCommunity.updatedAt ? dbCommunity.updatedAt : null + this.url = dbCommunity.endPoint + } + + @Field(() => Number, { nullable: true }) + id: number + + @Field(() => String) + publicKey: string + + @Field(() => String) + apiVersion: string + + @Field(() => String) + url: string + + @Field(() => Date) + createdAt: Date + + @Field(() => Date, { nullable: true }) + lastAnnouncedAt: Date | null + + @Field(() => Date, { nullable: true }) + verifiedAt: Date | null + + @Field(() => Date, { nullable: true }) + updatedAt: Date | null +} diff --git a/backend/yarn.lock b/backend/yarn.lock index 82bcd6b1f..72ee7518a 100644 --- a/backend/yarn.lock +++ b/backend/yarn.lock @@ -404,6 +404,11 @@ minimatch "^3.0.4" strip-json-comments "^3.1.1" +"@graphql-typed-document-node/core@^3.1.1": + version "3.1.1" + resolved "https://registry.yarnpkg.com/@graphql-typed-document-node/core/-/core-3.1.1.tgz#076d78ce99822258cf813ecc1e7fa460fa74d052" + integrity sha512-NQ17ii0rK1b34VZonlmT2QMJFI70m0TRwbknO/ihlbatXyaktDhN/98vBiUU6kNBPljqGqyIrl2T4nY2RpFANg== + "@hapi/boom@^10.0.0": version "10.0.0" resolved "https://registry.yarnpkg.com/@hapi/boom/-/boom-10.0.0.tgz#3624831d0a26b3378423b246f50eacea16e04a08" @@ -2193,6 +2198,13 @@ cross-env@^7.0.3: dependencies: cross-spawn "^7.0.1" +cross-fetch@^3.1.5: + version "3.1.5" + resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.5.tgz#e1389f44d9e7ba767907f7af8454787952ab534f" + integrity sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw== + dependencies: + node-fetch "2.6.7" + cross-spawn@^6.0.0: version "6.0.5" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4" @@ -2985,6 +2997,11 @@ express@^4.17.1: utils-merge "1.0.1" vary "~1.1.2" +extract-files@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/extract-files/-/extract-files-9.0.0.tgz#8a7744f2437f81f5ed3250ed9f1550de902fe54a" + integrity sha512-CvdFfHkC95B4bBBk36hcEmvdR2awOdhhVUYH6S/zrVj3477zven/fJMYg7121h4T1xHZC+tetUpubpAhxwI7hQ== + faker@^5.5.3: version "5.5.3" resolved "https://registry.yarnpkg.com/faker/-/faker-5.5.3.tgz#c57974ee484431b25205c2c8dc09fda861e51e0e" @@ -3340,6 +3357,16 @@ graphql-query-complexity@^0.7.0: dependencies: lodash.get "^4.4.2" +graphql-request@5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/graphql-request/-/graphql-request-5.0.0.tgz#7504a807d0e11be11a3c448e900f0cc316aa18ef" + integrity sha512-SpVEnIo2J5k2+Zf76cUkdvIRaq5FMZvGQYnA4lUWYbc99m+fHh4CZYRRO/Ff4tCLQ613fzCm3SiDT64ubW5Gyw== + dependencies: + "@graphql-typed-document-node/core" "^3.1.1" + cross-fetch "^3.1.5" + extract-files "^9.0.0" + form-data "^3.0.0" + graphql-subscriptions@^1.0.0, graphql-subscriptions@^1.1.0: version "1.2.1" resolved "https://registry.yarnpkg.com/graphql-subscriptions/-/graphql-subscriptions-1.2.1.tgz#2142b2d729661ddf967b7388f7cf1dd4cf2e061d" @@ -4977,7 +5004,7 @@ nice-try@^1.0.4: resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== -node-fetch@^2.6.0: +node-fetch@2.6.7, node-fetch@^2.6.0: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== From 72cc65d14a82da738c4578efd63529375f76de4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus-Peter=20H=C3=BCbner?= Date: Tue, 3 Jan 2023 03:12:59 +0100 Subject: [PATCH 002/124] add verifiedAt to communities --- .../Community.ts | 45 +++++++++++++++++++ database/entity/Community.ts | 2 +- .../0059-add_varified_at_to communities.ts | 17 +++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 database/entity/0059-add_verified_at_to_communities/Community.ts create mode 100644 database/migrations/0059-add_varified_at_to communities.ts diff --git a/database/entity/0059-add_verified_at_to_communities/Community.ts b/database/entity/0059-add_verified_at_to_communities/Community.ts new file mode 100644 index 000000000..eb4296e35 --- /dev/null +++ b/database/entity/0059-add_verified_at_to_communities/Community.ts @@ -0,0 +1,45 @@ +import { + BaseEntity, + Entity, + PrimaryGeneratedColumn, + Column, + CreateDateColumn, + UpdateDateColumn, +} from 'typeorm' + +@Entity('communities') +export class Community extends BaseEntity { + @PrimaryGeneratedColumn('increment', { unsigned: true }) + id: number + + @Column({ name: 'public_key', type: 'binary', length: 64, default: null, nullable: true }) + publicKey: Buffer + + @Column({ name: 'api_version', length: 10, nullable: false }) + apiVersion: string + + @Column({ name: 'end_point', length: 255, nullable: false }) + endPoint: string + + @Column({ name: 'last_announced_at', type: 'datetime', nullable: false }) + lastAnnouncedAt: Date + + @Column({ name: 'verified_at', type: 'datetime', nullable: true }) + verifiedAt: Date + + @CreateDateColumn({ + name: 'created_at', + type: 'datetime', + default: () => 'CURRENT_TIMESTAMP(3)', + nullable: false, + }) + createdAt: Date + + @UpdateDateColumn({ + name: 'updated_at', + type: 'datetime', + onUpdate: 'CURRENT_TIMESTAMP(3)', + nullable: true, + }) + updatedAt: Date | null +} diff --git a/database/entity/Community.ts b/database/entity/Community.ts index 457d03eae..8efc2735d 100644 --- a/database/entity/Community.ts +++ b/database/entity/Community.ts @@ -1 +1 @@ -export { Community } from './0058-add_communities_table/Community' +export { Community } from './0059-add_verified_at_to_communities/Community' diff --git a/database/migrations/0059-add_varified_at_to communities.ts b/database/migrations/0059-add_varified_at_to communities.ts new file mode 100644 index 000000000..2531b6cbc --- /dev/null +++ b/database/migrations/0059-add_varified_at_to communities.ts @@ -0,0 +1,17 @@ +/* MIGRATION TO CREATE THE FEDERATION COMMUNITY TABLES + * + * This migration creates the `community` and 'communityfederation' tables in the `apollo` database (`gradido_community`). + */ + +/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +export async function upgrade(queryFn: (query: string, values?: any[]) => Promise>) { + await queryFn( + `ALTER TABLE communities ADD COLUMN verified_at datetime(3) DEFAULT NULL AFTER last_announced_at;`, + ) +} + +export async function downgrade(queryFn: (query: string, values?: any[]) => Promise>) { + await queryFn(`ALTER TABLE communities DROP COLUMN verified_at;`) +} From 290987c40cb264287596300d0361efead205839a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus-Peter=20H=C3=BCbner?= Date: Tue, 3 Jan 2023 03:13:57 +0100 Subject: [PATCH 003/124] insert startValidateCommunities loop --- backend/src/config/index.ts | 1 + backend/src/federation/validateCommunities.ts | 29 +++++++++++++++++++ backend/src/index.ts | 2 ++ 3 files changed, 32 insertions(+) create mode 100644 backend/src/federation/validateCommunities.ts diff --git a/backend/src/config/index.ts b/backend/src/config/index.ts index 698b17e67..1559e7571 100644 --- a/backend/src/config/index.ts +++ b/backend/src/config/index.ts @@ -126,6 +126,7 @@ const federation = { : process.env.FEDERATION_COMMUNITY_URL.endsWith('/') ? process.env.FEDERATION_COMMUNITY_URL : process.env.FEDERATION_COMMUNITY_URL + '/', + FEDERATION_VALIDATE_COMMUNITY_TIMER: process.env.FEDERATION_VALIDATE_COMMUNITY_TIMER || 60000, } const CONFIG = { diff --git a/backend/src/federation/validateCommunities.ts b/backend/src/federation/validateCommunities.ts new file mode 100644 index 000000000..8a9429973 --- /dev/null +++ b/backend/src/federation/validateCommunities.ts @@ -0,0 +1,29 @@ +import { Community as DbCommunity } from '@entity/Community' +import { IsNull, LessThan, Raw } from '@dbTools/typeorm' +import { requestGetPublicKey } from './client/1_0/FederationClient' +import { FdCommunity } from './graphql/1_0/model/FdCommunity' + +export async function startValidateCommunities(timerInterval: number): Promise { + while (true) { + const dbCommunities: DbCommunity[] = await DbCommunity.find({ + where: [ + { verifiedAt: IsNull() }, + { verifiedAt: LessThan(Raw((lastAnnouncedAt) => `${lastAnnouncedAt}`)) }, + ], + }) + if (dbCommunities) { + dbCommunities.forEach(async function (dbCom) { + const fdCom = new FdCommunity(dbCom) + const pubKey = await requestGetPublicKey(fdCom) + if (pubKey && pubKey === dbCom.publicKey.toString('hex')) { + DbCommunity.update({ verifiedAt: new Date() }, { id: dbCom.id }) + } + }) + } + await sleep(timerInterval) + } +} + +function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} diff --git a/backend/src/index.ts b/backend/src/index.ts index 329e63f87..a549172be 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -5,6 +5,7 @@ import { startDHT } from '@/federation/index' // config import CONFIG from './config' +import { startValidateCommunities } from './federation/validateCommunities' async function main() { const { app } = await createServer() @@ -17,6 +18,7 @@ async function main() { console.log(`GraphIQL available at http://localhost:${CONFIG.PORT}`) } }) + startValidateCommunities(Number(CONFIG.FEDERATION_VALIDATE_COMMUNITY_TIMER)) // start DHT hyperswarm when DHT_TOPIC is set in .env if (CONFIG.FEDERATION_DHT_TOPIC) { From f227bb8d07f9227e3658214bf0ea850abea726eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus-Peter=20H=C3=BCbner?= Date: Wed, 4 Jan 2023 00:18:02 +0100 Subject: [PATCH 004/124] add verified_at to communities table --- ...t_to communities.ts => 0059-add_verified_at_to communities.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename database/migrations/{0059-add_varified_at_to communities.ts => 0059-add_verified_at_to communities.ts} (100%) diff --git a/database/migrations/0059-add_varified_at_to communities.ts b/database/migrations/0059-add_verified_at_to communities.ts similarity index 100% rename from database/migrations/0059-add_varified_at_to communities.ts rename to database/migrations/0059-add_verified_at_to communities.ts From bac9d7bd4d2332d4a034c4752ada93a0b75b6c3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus-Peter=20H=C3=BCbner?= Date: Wed, 4 Jan 2023 00:19:38 +0100 Subject: [PATCH 005/124] invoke request getPubkicKey on each endpoint read from communities table --- backend/src/config/index.ts | 2 +- backend/src/federation/client/1_0/FederationClient.ts | 2 +- backend/src/federation/validateCommunities.ts | 5 +---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/backend/src/config/index.ts b/backend/src/config/index.ts index 1559e7571..5e73eb76f 100644 --- a/backend/src/config/index.ts +++ b/backend/src/config/index.ts @@ -10,7 +10,7 @@ Decimal.set({ }) const constants = { - DB_VERSION: '0058-add_communities_table', + DB_VERSION: '0059-add_verified_at_to communities', DECAY_START_TIME: new Date('2021-05-13 17:46:31-0000'), // GMT+0 LOG4JS_CONFIG: 'log4js-config.json', // default log level on production should be info diff --git a/backend/src/federation/client/1_0/FederationClient.ts b/backend/src/federation/client/1_0/FederationClient.ts index 12b5fdac0..d1c1c00af 100644 --- a/backend/src/federation/client/1_0/FederationClient.ts +++ b/backend/src/federation/client/1_0/FederationClient.ts @@ -43,8 +43,8 @@ export async function requestGetPublicKey(fdCom: FdCommunity): Promise { while (true) { const dbCommunities: DbCommunity[] = await DbCommunity.find({ - where: [ - { verifiedAt: IsNull() }, - { verifiedAt: LessThan(Raw((lastAnnouncedAt) => `${lastAnnouncedAt}`)) }, - ], + where: [{ verifiedAt: IsNull() }, { verifiedAt: LessThan(`last_announced_at:`) }], }) if (dbCommunities) { dbCommunities.forEach(async function (dbCom) { From ed43c19ff423ebebc248ee2e8d5b977316bd6787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus-Peter=20H=C3=BCbner?= Date: Wed, 4 Jan 2023 00:40:43 +0100 Subject: [PATCH 006/124] add FEDERATION_VALIDATE_COMMUNITY_TIMER to config --- backend/.env.dist | 1 + backend/.env.template | 1 + backend/src/config/index.ts | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/.env.dist b/backend/.env.dist index b238388f6..7d2ec63f9 100644 --- a/backend/.env.dist +++ b/backend/.env.dist @@ -68,3 +68,4 @@ EVENT_PROTOCOL_DISABLED=false # FEDERATION_DHT_TOPIC=GRADIDO_HUB # FEDERATION_DHT_SEED=64ebcb0e3ad547848fef4197c6e2332f # FEDERATION_COMMUNITY_URL=http://localhost:4000/api +# FEDERATION_VALIDATE_COMMUNITY_TIMER=60000 \ No newline at end of file diff --git a/backend/.env.template b/backend/.env.template index f73b87353..b2c47d340 100644 --- a/backend/.env.template +++ b/backend/.env.template @@ -61,3 +61,4 @@ EVENT_PROTOCOL_DISABLED=$EVENT_PROTOCOL_DISABLED FEDERATION_DHT_TOPIC=$FEDERATION_DHT_TOPIC FEDERATION_DHT_SEED=$FEDERATION_DHT_SEED FEDERATION_COMMUNITY_URL=$FEDERATION_COMMUNITY_URL +FEDERATION_VALIDATE_COMMUNITY_TIMER=$FEDERATION_VALIDATE_COMMUNITY_TIMER \ No newline at end of file diff --git a/backend/src/config/index.ts b/backend/src/config/index.ts index 5e73eb76f..01e507cf5 100644 --- a/backend/src/config/index.ts +++ b/backend/src/config/index.ts @@ -17,7 +17,7 @@ const constants = { LOG_LEVEL: process.env.LOG_LEVEL || 'info', CONFIG_VERSION: { DEFAULT: 'DEFAULT', - EXPECTED: 'v14.2022-11-22', + EXPECTED: 'v15.2023-01-03', CURRENT: '', }, } From 998f673dd0e1b4d78d74982663f425918371cdfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus-Peter=20H=C3=BCbner?= Date: Wed, 4 Jan 2023 23:06:52 +0100 Subject: [PATCH 007/124] correct query for announced communities --- .../federation/client/1_0/FederationClient.ts | 2 +- .../graphql/1_0/model/FdCommunity.ts | 16 +++++++------- backend/src/federation/validateCommunities.ts | 22 +++++++++++++++++-- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/backend/src/federation/client/1_0/FederationClient.ts b/backend/src/federation/client/1_0/FederationClient.ts index d1c1c00af..6615d716e 100644 --- a/backend/src/federation/client/1_0/FederationClient.ts +++ b/backend/src/federation/client/1_0/FederationClient.ts @@ -44,7 +44,7 @@ export async function requestGetPublicKey(fdCom: FdCommunity): Promise Number, { nullable: true }) diff --git a/backend/src/federation/validateCommunities.ts b/backend/src/federation/validateCommunities.ts index 0924583f0..a07b4f099 100644 --- a/backend/src/federation/validateCommunities.ts +++ b/backend/src/federation/validateCommunities.ts @@ -2,22 +2,40 @@ import { Community as DbCommunity } from '@entity/Community' import { IsNull, LessThan, Raw } from '@dbTools/typeorm' import { requestGetPublicKey } from './client/1_0/FederationClient' import { FdCommunity } from './graphql/1_0/model/FdCommunity' +import { backendLogger as logger } from '@/server/logger' export async function startValidateCommunities(timerInterval: number): Promise { + logger.info( + `Federation: startValidateCommunities loop with an interval of ${timerInterval} ms...`, + ) while (true) { + /* const dbCommunities: DbCommunity[] = await DbCommunity.find({ - where: [{ verifiedAt: IsNull() }, { verifiedAt: LessThan(`last_announced_at:`) }], + where: [{ verifiedAt: IsNull() }, { verifiedAt: LessThan('Community.last_announced_at') }], }) + */ + const dbCommunities: DbCommunity[] = await DbCommunity.getRepository().manager.query( + 'SELECT * FROM `communities` `Community` WHERE (`Community`.`verified_at` IS NULL OR `Community`.`verified_at` < `Community`.`last_announced_at`)', + ) + + logger.debug(`Federation: found ${dbCommunities.length} dbCommunities`) if (dbCommunities) { dbCommunities.forEach(async function (dbCom) { + logger.debug(`Federation: validate publicKey for dbCom: ${JSON.stringify(dbCom)}`) const fdCom = new FdCommunity(dbCom) const pubKey = await requestGetPublicKey(fdCom) + logger.debug(`Federation: received publicKey: ${pubKey}`) if (pubKey && pubKey === dbCom.publicKey.toString('hex')) { - DbCommunity.update({ verifiedAt: new Date() }, { id: dbCom.id }) + // if (!pubKey) { + logger.debug(`Federation: matching publicKey: ${pubKey}`) + DbCommunity.update({ id: dbCom.id }, { verifiedAt: new Date() }) + logger.debug(`Federation: updated dbCom: ${JSON.stringify(dbCom)}`) } }) } + logger.debug(`Federation: loop starts sleeping...`) await sleep(timerInterval) + logger.debug(`Federation: loop ends sleeping`) } } From 8cd70ef05d53582e1b8157b7e57a41990f704fd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus-Peter=20H=C3=BCbner?= Date: Wed, 4 Jan 2023 23:42:17 +0100 Subject: [PATCH 008/124] linting --- backend/src/federation/validateCommunities.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/federation/validateCommunities.ts b/backend/src/federation/validateCommunities.ts index a07b4f099..f0a6d3d2a 100644 --- a/backend/src/federation/validateCommunities.ts +++ b/backend/src/federation/validateCommunities.ts @@ -1,5 +1,5 @@ import { Community as DbCommunity } from '@entity/Community' -import { IsNull, LessThan, Raw } from '@dbTools/typeorm' +// import { IsNull, LessThan, Raw } from '@dbTools/typeorm' import { requestGetPublicKey } from './client/1_0/FederationClient' import { FdCommunity } from './graphql/1_0/model/FdCommunity' import { backendLogger as logger } from '@/server/logger' @@ -26,7 +26,7 @@ export async function startValidateCommunities(timerInterval: number): Promise Date: Thu, 5 Jan 2023 03:50:09 +0100 Subject: [PATCH 009/124] change raw-query back to querybuilder-statement --- .../federation/client/1_0/FederationClient.ts | 2 +- backend/src/federation/enum/apiVersionType.ts | 3 ++ .../graphql/1_0/model/FdCommunity.ts | 16 ++++---- backend/src/federation/validateCommunities.ts | 39 ++++++++++++------- 4 files changed, 37 insertions(+), 23 deletions(-) create mode 100644 backend/src/federation/enum/apiVersionType.ts diff --git a/backend/src/federation/client/1_0/FederationClient.ts b/backend/src/federation/client/1_0/FederationClient.ts index 6615d716e..8c3f13c77 100644 --- a/backend/src/federation/client/1_0/FederationClient.ts +++ b/backend/src/federation/client/1_0/FederationClient.ts @@ -4,7 +4,7 @@ import { FdCommunity } from '@/federation/graphql/1_0/model/FdCommunity' export async function requestGetPublicKey(fdCom: FdCommunity): Promise { let endpoint = fdCom.url.endsWith('/') ? fdCom.url : fdCom.url + '/' - endpoint = `${endpoint}graphql/${fdCom.apiVersion}/getPublicKey` + endpoint = `${endpoint}${fdCom.apiVersion}/getPublicKey` logger.info(`requestGetPublicKey with endpoint='${endpoint}'...`) const graphQLClient = new GraphQLClient(endpoint, { diff --git a/backend/src/federation/enum/apiVersionType.ts b/backend/src/federation/enum/apiVersionType.ts new file mode 100644 index 000000000..0322b67a7 --- /dev/null +++ b/backend/src/federation/enum/apiVersionType.ts @@ -0,0 +1,3 @@ +export enum ApiVersionType { + V1_0 = '1_0', +} diff --git a/backend/src/federation/graphql/1_0/model/FdCommunity.ts b/backend/src/federation/graphql/1_0/model/FdCommunity.ts index f08583490..2169a294c 100644 --- a/backend/src/federation/graphql/1_0/model/FdCommunity.ts +++ b/backend/src/federation/graphql/1_0/model/FdCommunity.ts @@ -1,18 +1,18 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import { ObjectType, Field } from 'type-graphql' +import { Community as DbCommunity } from '@entity/Community' @ObjectType() export class FdCommunity { - // using NOT the entity DbCommunity, because of necessary RAW-Sql to find the correct announced communities - constructor(dbCommunity: any) { - this.apiVersion = dbCommunity.api_version - this.createdAt = dbCommunity.created_at + constructor(dbCommunity: DbCommunity) { + this.apiVersion = dbCommunity.apiVersion + this.createdAt = dbCommunity.createdAt this.id = dbCommunity.id - this.lastAnnouncedAt = dbCommunity.last_announced_at - this.publicKey = dbCommunity.public_key.toString('hex') - this.updatedAt = dbCommunity.updated_at - this.url = dbCommunity.end_point + this.lastAnnouncedAt = dbCommunity.lastAnnouncedAt + this.publicKey = dbCommunity.publicKey.toString('hex') + this.updatedAt = dbCommunity.updatedAt + this.url = dbCommunity.endPoint } @Field(() => Number, { nullable: true }) diff --git a/backend/src/federation/validateCommunities.ts b/backend/src/federation/validateCommunities.ts index f0a6d3d2a..61f9e724f 100644 --- a/backend/src/federation/validateCommunities.ts +++ b/backend/src/federation/validateCommunities.ts @@ -1,35 +1,46 @@ import { Community as DbCommunity } from '@entity/Community' -// import { IsNull, LessThan, Raw } from '@dbTools/typeorm' +import { IsNull, LessThan, Raw } from '@dbTools/typeorm' import { requestGetPublicKey } from './client/1_0/FederationClient' import { FdCommunity } from './graphql/1_0/model/FdCommunity' import { backendLogger as logger } from '@/server/logger' +import { ApiVersionType } from './enum/apiVersionType' export async function startValidateCommunities(timerInterval: number): Promise { logger.info( `Federation: startValidateCommunities loop with an interval of ${timerInterval} ms...`, ) while (true) { + const dbCommunities: DbCommunity[] = await DbCommunity.createQueryBuilder() + .where({ verifiedAt: IsNull() }) + .orWhere('verified_at < last_announced_at') + .getMany() /* - const dbCommunities: DbCommunity[] = await DbCommunity.find({ - where: [{ verifiedAt: IsNull() }, { verifiedAt: LessThan('Community.last_announced_at') }], - }) - */ const dbCommunities: DbCommunity[] = await DbCommunity.getRepository().manager.query( 'SELECT * FROM `communities` `Community` WHERE (`Community`.`verified_at` IS NULL OR `Community`.`verified_at` < `Community`.`last_announced_at`)', ) - + */ logger.debug(`Federation: found ${dbCommunities.length} dbCommunities`) if (dbCommunities) { dbCommunities.forEach(async function (dbCom) { - logger.debug(`Federation: validate publicKey for dbCom: ${JSON.stringify(dbCom)}`) + logger.debug(`Federation: dbCom: ${JSON.stringify(dbCom)}`) const fdCom = new FdCommunity(dbCom) - const pubKey = await requestGetPublicKey(fdCom) - logger.debug(`Federation: received publicKey: ${pubKey}`) - if (pubKey && pubKey === dbCom.publicKey.toString('hex')) { - // if (!pubKey) { - logger.debug(`Federation: matching publicKey: ${pubKey}`) - DbCommunity.update({ id: dbCom.id }, { verifiedAt: new Date() }) - logger.debug(`Federation: updated dbCom: ${JSON.stringify(dbCom)}`) + console.log(`ApiVersionType=`, ApiVersionType) + const apiValueStrings: string[] = Object.values(ApiVersionType) + if (apiValueStrings.includes(fdCom.apiVersion)) { + logger.debug( + `Federation: validate publicKey for dbCom: ${dbCom.id} with apiVersion=${dbCom.apiVersion}`, + ) + const pubKey = await requestGetPublicKey(fdCom) + logger.debug(`Federation: received publicKey: ${pubKey}`) + if (pubKey && pubKey === fdCom.publicKey) { + logger.debug(`Federation: matching publicKey: ${pubKey}`) + DbCommunity.update({ id: fdCom.id }, { verifiedAt: new Date() }) + logger.debug(`Federation: updated dbCom: ${JSON.stringify(dbCom)}`) + } + } else { + logger.debug( + `Federation: dbCom: ${fdCom.id} with unsupported apiVersion=${fdCom.apiVersion}; supported versions=${apiValueStrings}`, + ) } }) } From 49c131a6847a9603c24186e30e58f9b9cf82b5af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus-Peter=20H=C3=BCbner?= Date: Mon, 9 Jan 2023 22:46:38 +0100 Subject: [PATCH 010/124] use own versioned classes to request versioned publicKey --- .../federation/client/1_0/FederationClient.ts | 6 ++- .../federation/client/1_1/FederationClient.ts | 52 +++++++++++++++++++ backend/src/federation/enum/apiVersionType.ts | 1 + .../{FdCommunity.ts => V1_0_FdCommunity.ts} | 3 +- .../graphql/1_1/model/V1_1_FdCommunity.ts | 42 +++++++++++++++ backend/src/federation/validateCommunities.ts | 49 ++++++++++++++--- 6 files changed, 144 insertions(+), 9 deletions(-) create mode 100644 backend/src/federation/client/1_1/FederationClient.ts rename backend/src/federation/graphql/1_0/model/{FdCommunity.ts => V1_0_FdCommunity.ts} (93%) create mode 100644 backend/src/federation/graphql/1_1/model/V1_1_FdCommunity.ts diff --git a/backend/src/federation/client/1_0/FederationClient.ts b/backend/src/federation/client/1_0/FederationClient.ts index 8c3f13c77..77a8ec842 100644 --- a/backend/src/federation/client/1_0/FederationClient.ts +++ b/backend/src/federation/client/1_0/FederationClient.ts @@ -1,8 +1,10 @@ import { GraphQLClient, gql } from 'graphql-request' import { backendLogger as logger } from '@/server/logger' -import { FdCommunity } from '@/federation/graphql/1_0/model/FdCommunity' +// eslint-disable-next-line camelcase +import { V1_0_FdCommunity } from '@/federation/graphql/1_0/model/V1_0_FdCommunity' -export async function requestGetPublicKey(fdCom: FdCommunity): Promise { +// eslint-disable-next-line camelcase +export async function requestGetPublicKey(fdCom: V1_0_FdCommunity): Promise { let endpoint = fdCom.url.endsWith('/') ? fdCom.url : fdCom.url + '/' endpoint = `${endpoint}${fdCom.apiVersion}/getPublicKey` logger.info(`requestGetPublicKey with endpoint='${endpoint}'...`) diff --git a/backend/src/federation/client/1_1/FederationClient.ts b/backend/src/federation/client/1_1/FederationClient.ts new file mode 100644 index 000000000..798fb4a76 --- /dev/null +++ b/backend/src/federation/client/1_1/FederationClient.ts @@ -0,0 +1,52 @@ +import { GraphQLClient, gql } from 'graphql-request' +import { backendLogger as logger } from '@/server/logger' +// eslint-disable-next-line camelcase +import { V1_1_FdCommunity } from '@/federation/graphql/1_1/model/V1_1_FdCommunity' + +// eslint-disable-next-line camelcase +export async function requestGetPublicKey(fdCom: V1_1_FdCommunity): Promise { + let endpoint = fdCom.url.endsWith('/') ? fdCom.url : fdCom.url + '/' + endpoint = `${endpoint}${fdCom.apiVersion}/getPublicKey` + logger.info(`requestGetPublicKey with endpoint='${endpoint}'...`) + + const graphQLClient = new GraphQLClient(endpoint, { + method: 'GET', + jsonSerializer: { + parse: JSON.parse, + stringify: JSON.stringify, + }, + }) + logger.info(`graphQLClient=${JSON.stringify(graphQLClient)}`) + const query = gql` + query { + getPublicKey { + publicKey + } + } + ` + + const variables = {} + + try { + const { data, errors, extensions, headers, status } = await graphQLClient.rawRequest( + query, + variables, + ) + logger.debug( + `Response-Data: ${JSON.stringify( + { data, errors, extensions, headers, status }, + undefined, + 2, + )}`, + ) + if (data) { + logger.debug(`Response-PublicKey: ${data.getPublicKey.publicKey}`) + logger.info(`requestGetPublicKey processed successfully`) + return data.getPublicKey.publicKey + } + logger.warn(`requestGetPublicKey processed without response data`) + } catch (err) { + logger.error(`Request-Error:`, err) // ${JSON.stringify(err)}`) + } + return undefined +} diff --git a/backend/src/federation/enum/apiVersionType.ts b/backend/src/federation/enum/apiVersionType.ts index 0322b67a7..60da9de57 100644 --- a/backend/src/federation/enum/apiVersionType.ts +++ b/backend/src/federation/enum/apiVersionType.ts @@ -1,3 +1,4 @@ export enum ApiVersionType { V1_0 = '1_0', + V1_1 = '1_1', } diff --git a/backend/src/federation/graphql/1_0/model/FdCommunity.ts b/backend/src/federation/graphql/1_0/model/V1_0_FdCommunity.ts similarity index 93% rename from backend/src/federation/graphql/1_0/model/FdCommunity.ts rename to backend/src/federation/graphql/1_0/model/V1_0_FdCommunity.ts index 2169a294c..9c9ef305c 100644 --- a/backend/src/federation/graphql/1_0/model/FdCommunity.ts +++ b/backend/src/federation/graphql/1_0/model/V1_0_FdCommunity.ts @@ -4,7 +4,8 @@ import { ObjectType, Field } from 'type-graphql' import { Community as DbCommunity } from '@entity/Community' @ObjectType() -export class FdCommunity { +// eslint-disable-next-line camelcase +export class V1_0_FdCommunity { constructor(dbCommunity: DbCommunity) { this.apiVersion = dbCommunity.apiVersion this.createdAt = dbCommunity.createdAt diff --git a/backend/src/federation/graphql/1_1/model/V1_1_FdCommunity.ts b/backend/src/federation/graphql/1_1/model/V1_1_FdCommunity.ts new file mode 100644 index 000000000..4c44c3a2d --- /dev/null +++ b/backend/src/federation/graphql/1_1/model/V1_1_FdCommunity.ts @@ -0,0 +1,42 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ +import { ObjectType, Field } from 'type-graphql' +import { Community as DbCommunity } from '@entity/Community' + +@ObjectType() +// eslint-disable-next-line camelcase +export class V1_1_FdCommunity { + constructor(dbCommunity: DbCommunity) { + this.apiVersion = dbCommunity.apiVersion + this.createdAt = dbCommunity.createdAt + this.id = dbCommunity.id + this.lastAnnouncedAt = dbCommunity.lastAnnouncedAt + this.publicKey = dbCommunity.publicKey.toString('hex') + this.updatedAt = dbCommunity.updatedAt + this.url = dbCommunity.endPoint + } + + @Field(() => Number, { nullable: true }) + id: number + + @Field(() => String) + publicKey: string + + @Field(() => String) + apiVersion: string + + @Field(() => String) + url: string + + @Field(() => Date) + createdAt: Date + + @Field(() => Date, { nullable: true }) + lastAnnouncedAt: Date | null + + @Field(() => Date, { nullable: true }) + verifiedAt: Date | null + + @Field(() => Date, { nullable: true }) + updatedAt: Date | null +} diff --git a/backend/src/federation/validateCommunities.ts b/backend/src/federation/validateCommunities.ts index 61f9e724f..398009d9a 100644 --- a/backend/src/federation/validateCommunities.ts +++ b/backend/src/federation/validateCommunities.ts @@ -1,7 +1,13 @@ import { Community as DbCommunity } from '@entity/Community' -import { IsNull, LessThan, Raw } from '@dbTools/typeorm' -import { requestGetPublicKey } from './client/1_0/FederationClient' -import { FdCommunity } from './graphql/1_0/model/FdCommunity' +import { IsNull } from '@dbTools/typeorm' +// eslint-disable-next-line camelcase +import { requestGetPublicKey as v1_0_requestGetPublicKey } from './client/1_0/FederationClient' +// eslint-disable-next-line camelcase +import { requestGetPublicKey as v1_1_requestGetPublicKey } from './client/1_1/FederationClient' +// eslint-disable-next-line camelcase +import { V1_0_FdCommunity } from './graphql/1_0/model/V1_0_FdCommunity' +// eslint-disable-next-line camelcase +import { V1_1_FdCommunity } from './graphql/1_1/model/V1_1_FdCommunity' import { backendLogger as logger } from '@/server/logger' import { ApiVersionType } from './enum/apiVersionType' @@ -23,14 +29,20 @@ export async function startValidateCommunities(timerInterval: number): Promise setTimeout(resolve, ms)) } + +function getVersionedFdCommunity(dbCom: DbCommunity) { + switch (dbCom.apiVersion) { + case ApiVersionType.V1_0: + // eslint-disable-next-line new-cap + return new V1_0_FdCommunity(dbCom) + case ApiVersionType.V1_1: + // eslint-disable-next-line new-cap + return new V1_1_FdCommunity(dbCom) + } + return undefined +} + +async function invokeVersionedRequestGetPublicKey( + // eslint-disable-next-line camelcase + fdCom: V1_0_FdCommunity | V1_1_FdCommunity, +): Promise { + switch (fdCom.apiVersion) { + case ApiVersionType.V1_0: + return v1_0_requestGetPublicKey(fdCom) + case ApiVersionType.V1_1: + return v1_1_requestGetPublicKey(fdCom) + } + return undefined +} From 885834338f73997950abb17bc298a81dcb8f75c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus-Peter=20H=C3=BCbner?= Date: Thu, 12 Jan 2023 00:45:30 +0100 Subject: [PATCH 011/124] remove versioned graphql models --- .../federation/client/1_0/FederationClient.ts | 9 ++-- .../federation/client/1_1/FederationClient.ts | 9 ++-- .../graphql/1_0/model/V1_0_FdCommunity.ts | 42 ------------------- .../graphql/1_1/model/V1_1_FdCommunity.ts | 42 ------------------- backend/src/federation/validateCommunities.ts | 42 ++++--------------- 5 files changed, 16 insertions(+), 128 deletions(-) delete mode 100644 backend/src/federation/graphql/1_0/model/V1_0_FdCommunity.ts delete mode 100644 backend/src/federation/graphql/1_1/model/V1_1_FdCommunity.ts diff --git a/backend/src/federation/client/1_0/FederationClient.ts b/backend/src/federation/client/1_0/FederationClient.ts index 77a8ec842..e9ec5da54 100644 --- a/backend/src/federation/client/1_0/FederationClient.ts +++ b/backend/src/federation/client/1_0/FederationClient.ts @@ -1,12 +1,11 @@ import { GraphQLClient, gql } from 'graphql-request' import { backendLogger as logger } from '@/server/logger' -// eslint-disable-next-line camelcase -import { V1_0_FdCommunity } from '@/federation/graphql/1_0/model/V1_0_FdCommunity' +import { Community as DbCommunity } from '@entity/Community' // eslint-disable-next-line camelcase -export async function requestGetPublicKey(fdCom: V1_0_FdCommunity): Promise { - let endpoint = fdCom.url.endsWith('/') ? fdCom.url : fdCom.url + '/' - endpoint = `${endpoint}${fdCom.apiVersion}/getPublicKey` +export async function requestGetPublicKey(dbCom: DbCommunity): Promise { + let endpoint = dbCom.endPoint.endsWith('/') ? dbCom.endPoint : dbCom.endPoint + '/' + endpoint = `${endpoint}${dbCom.apiVersion}/getPublicKey` logger.info(`requestGetPublicKey with endpoint='${endpoint}'...`) const graphQLClient = new GraphQLClient(endpoint, { diff --git a/backend/src/federation/client/1_1/FederationClient.ts b/backend/src/federation/client/1_1/FederationClient.ts index 798fb4a76..e9ec5da54 100644 --- a/backend/src/federation/client/1_1/FederationClient.ts +++ b/backend/src/federation/client/1_1/FederationClient.ts @@ -1,12 +1,11 @@ import { GraphQLClient, gql } from 'graphql-request' import { backendLogger as logger } from '@/server/logger' -// eslint-disable-next-line camelcase -import { V1_1_FdCommunity } from '@/federation/graphql/1_1/model/V1_1_FdCommunity' +import { Community as DbCommunity } from '@entity/Community' // eslint-disable-next-line camelcase -export async function requestGetPublicKey(fdCom: V1_1_FdCommunity): Promise { - let endpoint = fdCom.url.endsWith('/') ? fdCom.url : fdCom.url + '/' - endpoint = `${endpoint}${fdCom.apiVersion}/getPublicKey` +export async function requestGetPublicKey(dbCom: DbCommunity): Promise { + let endpoint = dbCom.endPoint.endsWith('/') ? dbCom.endPoint : dbCom.endPoint + '/' + endpoint = `${endpoint}${dbCom.apiVersion}/getPublicKey` logger.info(`requestGetPublicKey with endpoint='${endpoint}'...`) const graphQLClient = new GraphQLClient(endpoint, { diff --git a/backend/src/federation/graphql/1_0/model/V1_0_FdCommunity.ts b/backend/src/federation/graphql/1_0/model/V1_0_FdCommunity.ts deleted file mode 100644 index 9c9ef305c..000000000 --- a/backend/src/federation/graphql/1_0/model/V1_0_FdCommunity.ts +++ /dev/null @@ -1,42 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ -import { ObjectType, Field } from 'type-graphql' -import { Community as DbCommunity } from '@entity/Community' - -@ObjectType() -// eslint-disable-next-line camelcase -export class V1_0_FdCommunity { - constructor(dbCommunity: DbCommunity) { - this.apiVersion = dbCommunity.apiVersion - this.createdAt = dbCommunity.createdAt - this.id = dbCommunity.id - this.lastAnnouncedAt = dbCommunity.lastAnnouncedAt - this.publicKey = dbCommunity.publicKey.toString('hex') - this.updatedAt = dbCommunity.updatedAt - this.url = dbCommunity.endPoint - } - - @Field(() => Number, { nullable: true }) - id: number - - @Field(() => String) - publicKey: string - - @Field(() => String) - apiVersion: string - - @Field(() => String) - url: string - - @Field(() => Date) - createdAt: Date - - @Field(() => Date, { nullable: true }) - lastAnnouncedAt: Date | null - - @Field(() => Date, { nullable: true }) - verifiedAt: Date | null - - @Field(() => Date, { nullable: true }) - updatedAt: Date | null -} diff --git a/backend/src/federation/graphql/1_1/model/V1_1_FdCommunity.ts b/backend/src/federation/graphql/1_1/model/V1_1_FdCommunity.ts deleted file mode 100644 index 4c44c3a2d..000000000 --- a/backend/src/federation/graphql/1_1/model/V1_1_FdCommunity.ts +++ /dev/null @@ -1,42 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ -import { ObjectType, Field } from 'type-graphql' -import { Community as DbCommunity } from '@entity/Community' - -@ObjectType() -// eslint-disable-next-line camelcase -export class V1_1_FdCommunity { - constructor(dbCommunity: DbCommunity) { - this.apiVersion = dbCommunity.apiVersion - this.createdAt = dbCommunity.createdAt - this.id = dbCommunity.id - this.lastAnnouncedAt = dbCommunity.lastAnnouncedAt - this.publicKey = dbCommunity.publicKey.toString('hex') - this.updatedAt = dbCommunity.updatedAt - this.url = dbCommunity.endPoint - } - - @Field(() => Number, { nullable: true }) - id: number - - @Field(() => String) - publicKey: string - - @Field(() => String) - apiVersion: string - - @Field(() => String) - url: string - - @Field(() => Date) - createdAt: Date - - @Field(() => Date, { nullable: true }) - lastAnnouncedAt: Date | null - - @Field(() => Date, { nullable: true }) - verifiedAt: Date | null - - @Field(() => Date, { nullable: true }) - updatedAt: Date | null -} diff --git a/backend/src/federation/validateCommunities.ts b/backend/src/federation/validateCommunities.ts index 398009d9a..da76698c5 100644 --- a/backend/src/federation/validateCommunities.ts +++ b/backend/src/federation/validateCommunities.ts @@ -4,10 +4,6 @@ import { IsNull } from '@dbTools/typeorm' import { requestGetPublicKey as v1_0_requestGetPublicKey } from './client/1_0/FederationClient' // eslint-disable-next-line camelcase import { requestGetPublicKey as v1_1_requestGetPublicKey } from './client/1_1/FederationClient' -// eslint-disable-next-line camelcase -import { V1_0_FdCommunity } from './graphql/1_0/model/V1_0_FdCommunity' -// eslint-disable-next-line camelcase -import { V1_1_FdCommunity } from './graphql/1_1/model/V1_1_FdCommunity' import { backendLogger as logger } from '@/server/logger' import { ApiVersionType } from './enum/apiVersionType' @@ -29,29 +25,22 @@ export async function startValidateCommunities(timerInterval: number): Promise setTimeout(resolve, ms)) } -function getVersionedFdCommunity(dbCom: DbCommunity) { +async function invokeVersionedRequestGetPublicKey(dbCom: DbCommunity): Promise { switch (dbCom.apiVersion) { case ApiVersionType.V1_0: - // eslint-disable-next-line new-cap - return new V1_0_FdCommunity(dbCom) + return v1_0_requestGetPublicKey(dbCom) case ApiVersionType.V1_1: - // eslint-disable-next-line new-cap - return new V1_1_FdCommunity(dbCom) - } - return undefined -} - -async function invokeVersionedRequestGetPublicKey( - // eslint-disable-next-line camelcase - fdCom: V1_0_FdCommunity | V1_1_FdCommunity, -): Promise { - switch (fdCom.apiVersion) { - case ApiVersionType.V1_0: - return v1_0_requestGetPublicKey(fdCom) - case ApiVersionType.V1_1: - return v1_1_requestGetPublicKey(fdCom) + return v1_1_requestGetPublicKey(dbCom) } return undefined } From 15cb8dff3a1844786873cee2de534c42e28f6d8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus-Peter=20H=C3=BCbner?= Date: Wed, 18 Jan 2023 02:05:49 +0100 Subject: [PATCH 012/124] Merge remote-tracking branch 'origin/master' into 2428-feature-federation-implement-multiple-apollo-graphql-endpoints plus validateCommunities-Tests --- .../federation/validateCommunities.test.ts | 159 ++++++++++++++++++ backend/src/federation/validateCommunities.ts | 70 ++++---- 2 files changed, 196 insertions(+), 33 deletions(-) create mode 100644 backend/src/federation/validateCommunities.test.ts diff --git a/backend/src/federation/validateCommunities.test.ts b/backend/src/federation/validateCommunities.test.ts new file mode 100644 index 000000000..4ccc05a3b --- /dev/null +++ b/backend/src/federation/validateCommunities.test.ts @@ -0,0 +1,159 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ + +import CONFIG from '@/config' +import { logger } from '@test/testSetup' +import { Community as DbCommunity } from '@entity/Community' +import { testEnvironment, cleanDB } from '@test/helpers' +import { startValidateCommunities, validateCommunities } from './validateCommunities' + +let con: any +let testEnv: any + +beforeAll(async () => { + testEnv = await testEnvironment(logger) + con = testEnv.con + await cleanDB() +}) + +afterAll(async () => { + // await cleanDB() + await con.close() +}) + +describe('validate Communities', () => { + /* + describe('start validation loop', () => { + beforeEach(async () => { + jest.clearAllMocks() + startValidateCommunities(0) + }) + + it('logs loop started', () => { + expect(logger.info).toBeCalledWith( + `Federation: startValidateCommunities loop with an interval of 0 ms...`, + ) + }) + }) + */ + describe('start validation logic without loop', () => { + beforeEach(async () => { + jest.clearAllMocks() + await validateCommunities() + }) + + it('logs zero communities found', () => { + expect(logger.debug).toBeCalledWith(`Federation: found 0 dbCommunities`) + }) + + describe('with one Community of api 1_0', () => { + beforeEach(async () => { + const variables1 = { + publicKey: Buffer.from('11111111111111111111111111111111'), + apiVersion: '1_0', + endPoint: 'http//localhost:5001/api/', + lastAnnouncedAt: new Date(), + } + await DbCommunity.createQueryBuilder() + .insert() + .into(DbCommunity) + .values(variables1) + .orUpdate({ + conflict_target: ['id', 'publicKey', 'apiVersion'], + overwrite: ['end_point', 'last_announced_at'], + }) + .execute() + + jest.clearAllMocks() + await validateCommunities() + }) + + it('logs one community found', () => { + expect(logger.debug).toBeCalledWith(`Federation: found 1 dbCommunities`) + }) + it('logs requestGetPublicKey for community api 1_0 ', () => { + expect(logger.info).toBeCalledWith( + `requestGetPublicKey with endpoint='http//localhost:5001/api/1_0/getPublicKey'...`, + ) + }) + }) + describe('with two Communities of api 1_0 and 1_1', () => { + beforeEach(async () => { + const variables2 = { + publicKey: Buffer.from('11111111111111111111111111111111'), + apiVersion: '1_1', + endPoint: 'http//localhost:5001/api/', + lastAnnouncedAt: new Date(), + } + await DbCommunity.createQueryBuilder() + .insert() + .into(DbCommunity) + .values(variables2) + .orUpdate({ + conflict_target: ['id', 'publicKey', 'apiVersion'], + overwrite: ['end_point', 'last_announced_at'], + }) + .execute() + + jest.clearAllMocks() + await validateCommunities() + }) + it('logs two communities found', () => { + expect(logger.debug).toBeCalledWith(`Federation: found 2 dbCommunities`) + }) + it('logs requestGetPublicKey for community api 1_0 ', () => { + expect(logger.info).toBeCalledWith( + `requestGetPublicKey with endpoint='http//localhost:5001/api/1_0/getPublicKey'...`, + ) + }) + it('logs requestGetPublicKey for community api 1_1 ', () => { + expect(logger.info).toBeCalledWith( + `requestGetPublicKey with endpoint='http//localhost:5001/api/1_1/getPublicKey'...`, + ) + }) + }) + describe('with three Communities of api 1_0, 1_1 and 2_0', () => { + let dbCom: DbCommunity + beforeEach(async () => { + const variables3 = { + publicKey: Buffer.from('11111111111111111111111111111111'), + apiVersion: '2_0', + endPoint: 'http//localhost:5001/api/', + lastAnnouncedAt: new Date(), + } + await DbCommunity.createQueryBuilder() + .insert() + .into(DbCommunity) + .values(variables3) + .orUpdate({ + conflict_target: ['id', 'publicKey', 'apiVersion'], + overwrite: ['end_point', 'last_announced_at'], + }) + .execute() + dbCom = await DbCommunity.findOneOrFail({ + where: { publicKey: variables3.publicKey, apiVersion: variables3.apiVersion }, + }) + jest.clearAllMocks() + await validateCommunities() + }) + it('logs three community found', () => { + expect(logger.debug).toBeCalledWith(`Federation: found 3 dbCommunities`) + }) + it('logs requestGetPublicKey for community api 1_0 ', () => { + expect(logger.info).toBeCalledWith( + `requestGetPublicKey with endpoint='http//localhost:5001/api/1_0/getPublicKey'...`, + ) + }) + it('logs requestGetPublicKey for community api 1_1 ', () => { + expect(logger.info).toBeCalledWith( + `requestGetPublicKey with endpoint='http//localhost:5001/api/1_1/getPublicKey'...`, + ) + }) + it('logs unsupported api for community with api 2_0 ', () => { + expect(logger.warn).toBeCalledWith( + `Federation: dbCom: ${dbCom.id} with unsupported apiVersion=2_0; supported versions=1_0,1_1`, + ) + }) + }) + }) +}) diff --git a/backend/src/federation/validateCommunities.ts b/backend/src/federation/validateCommunities.ts index da76698c5..9a978fbb9 100644 --- a/backend/src/federation/validateCommunities.ts +++ b/backend/src/federation/validateCommunities.ts @@ -12,45 +12,49 @@ export async function startValidateCommunities(timerInterval: number): Promise { + const dbCommunities: DbCommunity[] = await DbCommunity.createQueryBuilder() + .where({ verifiedAt: IsNull() }) + .orWhere('verified_at < last_announced_at') + .getMany() + /* + const dbCommunities: DbCommunity[] = await DbCommunity.getRepository().manager.query( + 'SELECT * FROM `communities` `Community` WHERE (`Community`.`verified_at` IS NULL OR `Community`.`verified_at` < `Community`.`last_announced_at`)', + ) + */ + logger.debug(`Federation: found ${dbCommunities.length} dbCommunities`) + if (dbCommunities) { + dbCommunities.forEach(async function (dbCom) { + logger.debug(`Federation: dbCom: ${JSON.stringify(dbCom)}`) + const apiValueStrings: string[] = Object.values(ApiVersionType) + logger.debug(`suppported ApiVersions=`, apiValueStrings) + if (apiValueStrings.includes(dbCom.apiVersion)) { + logger.debug( + `Federation: validate publicKey for dbCom: ${dbCom.id} with apiVersion=${dbCom.apiVersion}`, + ) + const pubKey = await invokeVersionedRequestGetPublicKey(dbCom) + logger.debug(`Federation: received publicKey: ${pubKey}`) + if (pubKey && pubKey === dbCom.publicKey.toString('hex')) { + logger.debug(`Federation: matching publicKey: ${pubKey}`) + DbCommunity.update({ id: dbCom.id }, { verifiedAt: new Date() }) + logger.debug(`Federation: updated dbCom: ${JSON.stringify(dbCom)}`) + } + } else { + logger.warn( + `Federation: dbCom: ${dbCom.id} with unsupported apiVersion=${dbCom.apiVersion}; supported versions=${apiValueStrings}`, + ) + } + }) + } +} + function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)) } From 6be123e3bfae906012d75578d4cc692358a9bd17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus-Peter=20H=C3=BCbner?= Date: Wed, 18 Jan 2023 02:13:36 +0100 Subject: [PATCH 013/124] linting --- backend/src/federation/validateCommunities.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/src/federation/validateCommunities.test.ts b/backend/src/federation/validateCommunities.test.ts index 4ccc05a3b..12e8e9fb9 100644 --- a/backend/src/federation/validateCommunities.test.ts +++ b/backend/src/federation/validateCommunities.test.ts @@ -1,11 +1,10 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ -import CONFIG from '@/config' import { logger } from '@test/testSetup' import { Community as DbCommunity } from '@entity/Community' import { testEnvironment, cleanDB } from '@test/helpers' -import { startValidateCommunities, validateCommunities } from './validateCommunities' +import { validateCommunities } from './validateCommunities' let con: any let testEnv: any From 3c669a17cb418c6699d0bb235812e07112388548 Mon Sep 17 00:00:00 2001 From: ogerly Date: Tue, 24 Jan 2023 15:23:34 +0100 Subject: [PATCH 014/124] change open creation-page, add tabs for all state on contributions --- admin/src/components/NavBar.vue | 14 ++++------- admin/src/locales/de.json | 8 ++++++- admin/src/locales/en.json | 8 ++++++- admin/src/pages/CreationConfirm.vue | 37 ++++++++++++++++++++++------- 4 files changed, 47 insertions(+), 20 deletions(-) diff --git a/admin/src/components/NavBar.vue b/admin/src/components/NavBar.vue index 6bed8e6e4..399a7dcac 100644 --- a/admin/src/components/NavBar.vue +++ b/admin/src/components/NavBar.vue @@ -12,12 +12,11 @@ {{ $t('navbar.overview') }} {{ $t('navbar.user_search') }} {{ $t('navbar.multi_creation') }} - - {{ $store.state.openCreations }} {{ $t('navbar.open_creation') }} + + {{ $t('creation') }} + + {{ $store.state.openCreations }} + {{ $t('navbar.automaticContributions') }} @@ -57,7 +56,4 @@ export default { height: 2rem; padding-left: 10px; } -.bg-color-creation { - background-color: #cf1010dc; -} diff --git a/admin/src/locales/de.json b/admin/src/locales/de.json index ad7a668e2..74cfffe3b 100644 --- a/admin/src/locales/de.json +++ b/admin/src/locales/de.json @@ -29,6 +29,13 @@ "validFrom": "Startdatum", "validTo": "Enddatum" }, + "contributions": { + "all": "Alle", + "confirmed": "Bestätigte", + "deleted": "Gelöscht", + "denied": "Abgelehnt", + "open": "Offen" + }, "creation": "Schöpfung", "creationList": "Schöpfungsliste", "creation_form": { @@ -101,7 +108,6 @@ "logout": "Abmelden", "multi_creation": "Mehrfachschöpfung", "my-account": "Mein Konto", - "open_creation": "Offene Schöpfungen", "overview": "Übersicht", "statistic": "Statistik", "user_search": "Nutzersuche" diff --git a/admin/src/locales/en.json b/admin/src/locales/en.json index 3f8751fa1..c61d3f9b8 100644 --- a/admin/src/locales/en.json +++ b/admin/src/locales/en.json @@ -29,6 +29,13 @@ "validFrom": "Start-date", "validTo": "End-Date" }, + "contributions": { + "all": "All", + "confirmed": "Confirmed", + "deleted": "Deleted", + "denied": "Denied", + "open": "Open" + }, "creation": "Creation", "creationList": "Creation list", "creation_form": { @@ -101,7 +108,6 @@ "logout": "Logout", "multi_creation": "Multiple creation", "my-account": "My Account", - "open_creation": "Open creations", "overview": "Overview", "statistic": "Statistic", "user_search": "User search" diff --git a/admin/src/pages/CreationConfirm.vue b/admin/src/pages/CreationConfirm.vue index 23871635d..956174f4e 100644 --- a/admin/src/pages/CreationConfirm.vue +++ b/admin/src/pages/CreationConfirm.vue @@ -1,17 +1,36 @@ From 22c8de606c36ec50bec66ab6c801f670b612ab89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus-Peter=20H=C3=BCbner?= Date: Wed, 1 Feb 2023 22:29:39 +0100 Subject: [PATCH 026/124] upgrade db-version in dht-modul --- dht-node/src/config/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dht-node/src/config/index.ts b/dht-node/src/config/index.ts index 02cbb20e9..70dd829bb 100644 --- a/dht-node/src/config/index.ts +++ b/dht-node/src/config/index.ts @@ -3,7 +3,7 @@ import dotenv from 'dotenv' dotenv.config() const constants = { - DB_VERSION: '0059-add_hide_amount_to_users', + DB_VERSION: '0060-add_verified_at_to communities', LOG4JS_CONFIG: 'log4js-config.json', // default log level on production should be info LOG_LEVEL: process.env.LOG_LEVEL || 'info', From bfcef736361174589d85933cc8fd07fecd59a068 Mon Sep 17 00:00:00 2001 From: ogerly Date: Thu, 2 Feb 2023 07:53:29 +0100 Subject: [PATCH 027/124] add badge, count open contribution on tab open --- admin/src/pages/CreationConfirm.vue | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/admin/src/pages/CreationConfirm.vue b/admin/src/pages/CreationConfirm.vue index 69a483943..75356c588 100644 --- a/admin/src/pages/CreationConfirm.vue +++ b/admin/src/pages/CreationConfirm.vue @@ -3,7 +3,13 @@
- + + Date: Thu, 2 Feb 2023 09:29:59 +0100 Subject: [PATCH 028/124] change query listAllContribution add statusFilter --- admin/src/graphql/listAllContributions.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/admin/src/graphql/listAllContributions.js b/admin/src/graphql/listAllContributions.js index 1aacc4824..c9ad60441 100644 --- a/admin/src/graphql/listAllContributions.js +++ b/admin/src/graphql/listAllContributions.js @@ -1,8 +1,18 @@ import gql from 'graphql-tag' export const listAllContributions = gql` - query ($currentPage: Int = 1, $pageSize: Int = 25, $order: Order = DESC) { - listAllContributions(currentPage: $currentPage, pageSize: $pageSize, order: $order) { + query ( + $currentPage: Int = 1 + $pageSize: Int = 5 + $order: Order = DESC + $statusFilter: [ContributionStatus!] + ) { + listAllContributions( + currentPage: $currentPage + pageSize: $pageSize + order: $order + statusFilter: $statusFilter + ) { contributionCount contributionList { id From 0298bbd740aa91c74f5481ed17df692e6ffe0480 Mon Sep 17 00:00:00 2001 From: ogerly Date: Thu, 2 Feb 2023 16:26:26 +0100 Subject: [PATCH 029/124] add pagination on contributions lists --- .../components/Tables/OpenCreationsTable.vue | 30 +++- admin/src/graphql/listAllContributions.js | 2 +- admin/src/locales/de.json | 5 + admin/src/locales/en.json | 4 + admin/src/pages/CreationConfirm.vue | 167 ++++++++++++++---- admin/src/pages/Overview.vue | 37 ++-- .../graphql/resolver/ContributionResolver.ts | 1 + 7 files changed, 195 insertions(+), 51 deletions(-) diff --git a/admin/src/components/Tables/OpenCreationsTable.vue b/admin/src/components/Tables/OpenCreationsTable.vue index e4d289c8e..aa5e88dc5 100644 --- a/admin/src/components/Tables/OpenCreationsTable.vue +++ b/admin/src/components/Tables/OpenCreationsTable.vue @@ -1,11 +1,20 @@