Merge remote-tracking branch

'origin/2650-feature-federation-harmonize-and-sync-modules-of-federation'
into 2501-feature-federation-implement-a-graphql-client-to-request-getpublickey

backend/src/config/index.ts

@@ -10,7 +10,7 @@ Decimal.set({
 })
 
 const constants = {
-  DB_VERSION: '0060-add_verified_at_to communities',
+  DB_VERSION: '0061-update_communities_table',
   DECAY_START_TIME: new Date('2021-05-13 17:46:31-0000'), // GMT+0
   LOG4JS_CONFIG: 'log4js-config.json',
   // default log level on production should be info

database/entity/0061-update_communities_table/Community.ts

@@ -0,0 +1,51 @@
+import {
+  BaseEntity,
+  Entity,
+  PrimaryGeneratedColumn,
+  Column,
+  CreateDateColumn,
+  UpdateDateColumn,
+} from 'typeorm'
+
+@Entity('communities')
+export class Community extends BaseEntity {
+  @PrimaryGeneratedColumn('increment', { unsigned: true })
+  id: number
+
+  @Column({ name: 'foreign', type: 'bool', nullable: false, default: true })
+  foreign: boolean
+
+  @Column({ name: 'public_key', type: 'binary', length: 64, default: null, nullable: true })
+  publicKey: Buffer
+
+  @Column({ name: 'api_version', length: 10, nullable: false })
+  apiVersion: string
+
+  @Column({ name: 'end_point', length: 255, nullable: false })
+  endPoint: string
+
+  @Column({ name: 'last_announced_at', type: 'datetime', nullable: true })
+  lastAnnouncedAt: Date
+
+  @Column({ name: 'verified_at', type: 'datetime', nullable: true })
+  verifiedAt: Date
+
+  @Column({ name: 'last_error_at', type: 'datetime', nullable: true })
+  lastErrorAt: Date
+
+  @CreateDateColumn({
+    name: 'created_at',
+    type: 'datetime',
+    default: () => 'CURRENT_TIMESTAMP(3)',
+    nullable: false,
+  })
+  createdAt: Date
+
+  @UpdateDateColumn({
+    name: 'updated_at',
+    type: 'datetime',
+    onUpdate: 'CURRENT_TIMESTAMP(3)',
+    nullable: true,
+  })
+  updatedAt: Date | null
+}

database/entity/Community.ts

@@ -1 +1 @@
-export { Community } from './0060-add_verified_at_to_communities/Community'
+export { Community } from './0061-update_communities_table/Community'

database/migrations/0061-update_communities_table.ts

@@ -0,0 +1,34 @@
+/* MIGRATION TO CREATE THE FEDERATION COMMUNITY TABLES
+ *
+ * This migration creates the `community` and 'communityfederation' tables in the `apollo` database (`gradido_community`).
+ */
+
+/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+export async function upgrade(queryFn: (query: string, values?: any[]) => Promise<Array<any>>) {
+  await queryFn(
+    'ALTER TABLE `communities` MODIFY COLUMN `last_announced_at` datetime(3) AFTER `end_point`;',
+  )
+  await queryFn(
+    'ALTER TABLE `communities` ADD COLUMN `foreign` tinyint(4) NOT NULL DEFAULT 1 AFTER `id`;',
+  )
+  await queryFn(
+    'ALTER TABLE `communities` ADD COLUMN `verified_at` datetime(3) AFTER `last_announced_at`;',
+  )
+  /*
+  await queryFn(
+    'ALTER TABLE `communities` ADD COLUMN `last_error_at` datetime(3) AFTER `verified_at`;',
+  )
+  */
+}
+
+export async function downgrade(queryFn: (query: string, values?: any[]) => Promise<Array<any>>) {
+  // write downgrade logic as parameter of queryFn
+  await queryFn(
+    'ALTER TABLE `communities` MODIFY COLUMN `last_announced_at` datetime(3) NOT NULL AFTER `end_point`;',
+  )
+  await queryFn('ALTER TABLE `communities` DROP COLUMN `foreign`;')
+  // await queryFn('ALTER TABLE `communities` DROP COLUMN `verified_at`;')
+  await queryFn('ALTER TABLE `communities` DROP COLUMN `last_error_at`;')
+}

dht-node/.env.dist

@@ -1,4 +1,4 @@
-CONFIG_VERSION=v1.2023-01-01
+CONFIG_VERSION=v2.2023-02-03
 
 # Database
 DB_HOST=localhost
@@ -20,3 +20,5 @@ EVENT_PROTOCOL_DISABLED=false
 # on an hash created from this topic
 FEDERATION_DHT_TOPIC=GRADIDO_HUB
 # FEDERATION_DHT_SEED=64ebcb0e3ad547848fef4197c6e2332f 
+# FEDERATION_COMMUNITY_URL=http://localhost
+# FEDERATION_COMMUNITY_API_PORT=5000

dht-node/.env.template

@@ -15,3 +15,4 @@ EVENT_PROTOCOL_DISABLED=$EVENT_PROTOCOL_DISABLED
 FEDERATION_DHT_TOPIC=$FEDERATION_DHT_TOPIC
 FEDERATION_DHT_SEED=$FEDERATION_DHT_SEED
 FEDERATION_COMMUNITY_URL=$FEDERATION_COMMUNITY_URL
+FEDERATION_COMMUNITY_API_PORT=FEDERATION_COMMUNITY_API_PORT

dht-node/src/config/index.ts

@@ -3,13 +3,13 @@ import dotenv from 'dotenv'
 dotenv.config()
 
 const constants = {
-  DB_VERSION: '0060-add_verified_at_to communities',
+  DB_VERSION: '0061-update_communities_table',
   LOG4JS_CONFIG: 'log4js-config.json',
   // default log level on production should be info
   LOG_LEVEL: process.env.LOG_LEVEL || 'info',
   CONFIG_VERSION: {
     DEFAULT: 'DEFAULT',
-    EXPECTED: 'v1.2023-01-01',
+    EXPECTED: 'v2.2023-02-03',
     CURRENT: '',
   },
 }
@@ -36,7 +36,8 @@ const eventProtocol = {
 const federation = {
   FEDERATION_DHT_TOPIC: process.env.FEDERATION_DHT_TOPIC || 'GRADIDO_HUB',
   FEDERATION_DHT_SEED: process.env.FEDERATION_DHT_SEED || null,
-  FEDERATION_COMMUNITY_URL: process.env.FEDERATION_COMMUNITY_URL || null,
+  FEDERATION_COMMUNITY_URL: process.env.FEDERATION_COMMUNITY_URL || 'http://localhost',
+  FEDERATION_COMMUNITY_API_PORT: process.env.FEDERATION_COMMUNITY_API_PORT || '5000',
 }
 
 // Check config version

dht-node/src/dht_node/index.test.ts

@@ -44,10 +44,11 @@ const lookupResultMock = {
   token: Buffer.from(TEST_TOPIC),
   from: {
     id: Buffer.from('somone'),
+    foreign: true,
     host: '188.95.53.5',
     port: 63561,
   },
-  to: { id: null, host: '83.53.31.27', port: 55723 },
+  to: { id: null, foreign: true, host: '83.53.31.27', port: 55723 },
   peers: [
     {
       publicKey: Buffer.from('some-public-key'),
@@ -116,6 +117,7 @@ describe('federation', () => {
     beforeEach(async () => {
       DHT.mockClear()
       jest.clearAllMocks()
+      await cleanDB()
       await startDHT(TEST_TOPIC)
     })
 
@@ -234,18 +236,18 @@ describe('federation', () => {
               beforeEach(async () => {
                 jest.clearAllMocks()
                 jsonArray = [
-                  { api: 'v1_0', url: 'too much versions at the same time test' },
+                  { api: '1_0', url: 'too much versions at the same time test' },
-                  { api: 'v1_0', url: 'url2' },
+                  { api: '1_0', url: 'url2' },
-                  { api: 'v1_0', url: 'url3' },
+                  { api: '1_0', url: 'url3' },
-                  { api: 'v1_0', url: 'url4' },
+                  { api: '1_0', url: 'url4' },
-                  { api: 'v1_0', url: 'url5' },
+                  { api: '1_0', url: 'url5' },
                 ]
                 await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
               })
 
               it('logs the received data', () => {
                 expect(logger.info).toBeCalledWith(
-                  'data: [{"api":"v1_0","url":"too much versions at the same time test"},{"api":"v1_0","url":"url2"},{"api":"v1_0","url":"url3"},{"api":"v1_0","url":"url4"},{"api":"v1_0","url":"url5"}]',
+                  'data: [{"api":"1_0","url":"too much versions at the same time test"},{"api":"1_0","url":"url2"},{"api":"1_0","url":"url3"},{"api":"1_0","url":"url4"},{"api":"1_0","url":"url5"}]',
                 )
               })
 
@@ -266,17 +268,17 @@ describe('federation', () => {
                 jsonArray = [
                   {
                     wrong: 'wrong but tolerated property test',
-                    api: 'v1_0',
+                    api: '1_0',
                     url: 'url1',
                   },
                   {
-                    api: 'v2_0',
+                    api: '2_0',
                     url: 'url2',
                     wrong: 'wrong but tolerated property test',
                   },
                 ]
                 await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
-                result = await DbCommunity.find()
+                result = await DbCommunity.find({ foreign: true })
               })
 
               afterAll(async () => {
@@ -287,13 +289,14 @@ describe('federation', () => {
                 expect(result).toHaveLength(2)
               })
 
-              it('has an entry for api version v1_0', () => {
+              it('has an entry for api version 1_0', () => {
                 expect(result).toEqual(
                   expect.arrayContaining([
                     expect.objectContaining({
                       id: expect.any(Number),
+                      foreign: true,
                       publicKey: expect.any(Buffer),
-                      apiVersion: 'v1_0',
+                      apiVersion: '1_0',
                       endPoint: 'url1',
                       lastAnnouncedAt: expect.any(Date),
                       createdAt: expect.any(Date),
@@ -303,13 +306,14 @@ describe('federation', () => {
                 )
               })
 
-              it('has an entry for api version v2_0', () => {
+              it('has an entry for api version 2_0', () => {
                 expect(result).toEqual(
                   expect.arrayContaining([
                     expect.objectContaining({
                       id: expect.any(Number),
+                      foreign: true,
                       publicKey: expect.any(Buffer),
-                      apiVersion: 'v2_0',
+                      apiVersion: '2_0',
                       endPoint: 'url2',
                       lastAnnouncedAt: expect.any(Date),
                       createdAt: expect.any(Date),
@@ -535,7 +539,7 @@ describe('federation', () => {
                   { api: 'toolong api', url: 'some valid url' },
                 ]
                 await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
-                result = await DbCommunity.find()
+                result = await DbCommunity.find({ foreign: true })
               })
 
               afterAll(async () => {
@@ -551,6 +555,7 @@ describe('federation', () => {
                   expect.arrayContaining([
                     expect.objectContaining({
                       id: expect.any(Number),
+                      foreign: true,
                       publicKey: expect.any(Buffer),
                       apiVersion: 'valid  api',
                       endPoint:
@@ -588,7 +593,7 @@ describe('federation', () => {
                   },
                 ]
                 await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
-                result = await DbCommunity.find()
+                result = await DbCommunity.find({ foreign: true })
               })
 
               afterAll(async () => {
@@ -604,6 +609,7 @@ describe('federation', () => {
                   expect.arrayContaining([
                     expect.objectContaining({
                       id: expect.any(Number),
+                      foreign: true,
                       publicKey: expect.any(Buffer),
                       apiVersion: 'valid api1',
                       endPoint:
@@ -621,6 +627,7 @@ describe('federation', () => {
                   expect.arrayContaining([
                     expect.objectContaining({
                       id: expect.any(Number),
+                      foreign: true,
                       publicKey: expect.any(Buffer),
                       apiVersion: 'valid api2',
                       endPoint:
@@ -638,6 +645,7 @@ describe('federation', () => {
                   expect.arrayContaining([
                     expect.objectContaining({
                       id: expect.any(Number),
+                      foreign: true,
                       publicKey: expect.any(Buffer),
                       apiVersion: 'valid api3',
                       endPoint:
@@ -655,6 +663,7 @@ describe('federation', () => {
                   expect.arrayContaining([
                     expect.objectContaining({
                       id: expect.any(Number),
+                      foreign: true,
                       publicKey: expect.any(Buffer),
                       apiVersion: 'valid api4',
                       endPoint:
@@ -710,17 +719,17 @@ describe('federation', () => {
                   Buffer.from(
                     JSON.stringify([
                       {
-                        api: 'v1_0',
+                        api: '1_0',
-                        url: 'http://localhost:4000/api/v1_0',
+                        url: 'http://localhost:5001/api/',
                       },
                       {
-                        api: 'v2_0',
+                        api: '2_0',
-                        url: 'http://localhost:4000/api/v2_0',
+                        url: 'http://localhost:5002/api/',
                       },
                     ]),
                   ),
                 )
-                result = await DbCommunity.find()
+                result = await DbCommunity.find({ foreign: true })
               })
 
               afterAll(async () => {
@@ -736,9 +745,10 @@ describe('federation', () => {
                   expect.arrayContaining([
                     expect.objectContaining({
                       id: expect.any(Number),
+                      foreign: true,
                       publicKey: expect.any(Buffer),
-                      apiVersion: 'v1_0',
+                      apiVersion: '1_0',
-                      endPoint: 'http://localhost:4000/api/v1_0',
+                      endPoint: 'http://localhost:5001/api/',
                       lastAnnouncedAt: expect.any(Date),
                       createdAt: expect.any(Date),
                       updatedAt: null,
@@ -747,14 +757,15 @@ describe('federation', () => {
                 )
               })
 
-              it('has an entry for api version v2_0', () => {
+              it('has an entry for api version 2_0', () => {
                 expect(result).toEqual(
                   expect.arrayContaining([
                     expect.objectContaining({
                       id: expect.any(Number),
+                      foreign: true,
                       publicKey: expect.any(Buffer),
-                      apiVersion: 'v2_0',
+                      apiVersion: '2_0',
-                      endPoint: 'http://localhost:4000/api/v2_0',
+                      endPoint: 'http://localhost:5002/api/',
                       lastAnnouncedAt: expect.any(Date),
                       createdAt: expect.any(Date),
                       updatedAt: null,
@@ -775,16 +786,16 @@ describe('federation', () => {
                 Buffer.from(
                   JSON.stringify([
                     {
-                      api: 'v1_0',
+                      api: '1_0',
-                      url: 'http://localhost:4000/api/v1_0',
+                      url: 'http://localhost:5001/api/',
                     },
                     {
-                      api: 'v1_1',
+                      api: '1_1',
-                      url: 'http://localhost:4000/api/v1_1',
+                      url: 'http://localhost:5002/api/',
                     },
                     {
-                      api: 'v2_0',
+                      api: '2_0',
-                      url: 'http://localhost:4000/api/v2_0',
+                      url: 'http://localhost:5003/api/',
                     },
                   ]),
                 ),

dht-node/src/dht_node/index.ts

@@ -15,9 +15,9 @@ const ERRORTIME = 240000
 const ANNOUNCETIME = 30000
 
 enum ApiVersionType {
-  V1_0 = 'v1_0',
+  V1_0 = '1_0',
-  V1_1 = 'v1_1',
+  V1_1 = '1_1',
-  V2_0 = 'v2_0',
+  V2_0 = '2_0',
 }
 type CommunityApi = {
   api: string
@@ -31,13 +31,16 @@ export const startDHT = async (topic: string): Promise<void> => {
     logger.info(`keyPairDHT: publicKey=${keyPair.publicKey.toString('hex')}`)
     logger.debug(`keyPairDHT: secretKey=${keyPair.secretKey.toString('hex')}`)
 
+    const ownApiVersions = writeHomeCommunityEnries(keyPair.publicKey)
+    /*
     const ownApiVersions = Object.values(ApiVersionType).map(function (apiEnum) {
       const comApi: CommunityApi = {
         api: apiEnum,
-        url: CONFIG.FEDERATION_COMMUNITY_URL + apiEnum,
+        url: CONFIG.FEDERATION_COMMUNITY_URL,
       }
       return comApi
     })
+    */
     logger.debug(`ApiList: ${JSON.stringify(ownApiVersions)}`)
 
     const node = new DHT({ keyPair })
@@ -184,3 +187,40 @@ export const startDHT = async (topic: string): Promise<void> => {
     logger.error('DHT unexpected error:', err)
   }
 }
+
+async function writeHomeCommunityEnries(pubKey: any): Promise<CommunityApi[]> {
+  const homeApiVersions: CommunityApi[] = Object.values(ApiVersionType).map(function (
+    apiEnum,
+    idx,
+  ) {
+    const port = Number.parseInt(CONFIG.FEDERATION_COMMUNITY_API_PORT) + idx + 1
+    const comApi: CommunityApi = {
+      api: apiEnum,
+      url: CONFIG.FEDERATION_COMMUNITY_URL + ':' + port.toString() + '/api/',
+    }
+    return comApi
+  })
+  try {
+    // first remove privious existing homeCommunity entries
+    const homeComs = await DbCommunity.find({ foreign: false })
+    if (homeComs.length > 0) {
+      await DbCommunity.remove(homeComs)
+    }
+
+    homeApiVersions.forEach(async function (homeApi) {
+      const homeCom = new DbCommunity()
+      homeCom.foreign = false
+      homeCom.apiVersion = homeApi.api
+      homeCom.endPoint = homeApi.url
+      homeCom.publicKey = pubKey.toString('hex')
+
+      // this will NOT update the updatedAt column, to distingue between a normal update and the last announcement
+      await DbCommunity.insert(homeCom)
+      logger.info(`federation home-community inserted successfully: ${JSON.stringify(homeCom)}`)
+    })
+  } catch (err) {
+    throw new Error(`Federation: Error writing HomeCommunity-Entries: ${err}`)
+  }
+
+  return homeApiVersions
+}