Merge pull request #2427 from gradido/2420-feature-federation-implement-exchange-of-api-versions-persist-in-table

feat(backend): federation implement exchange of api versions persist in table

backend/.env.dist

@@ -1,4 +1,4 @@
-CONFIG_VERSION=v13.2022-12-20
+CONFIG_VERSION=v14.2022-12-22
 
 # Server
 PORT=4000
@@ -67,3 +67,4 @@ EVENT_PROTOCOL_DISABLED=false
 # on an hash created from this topic
 # FEDERATION_DHT_TOPIC=GRADIDO_HUB
 # FEDERATION_DHT_SEED=64ebcb0e3ad547848fef4197c6e2332f 
+# FEDERATION_COMMUNITY_URL=http://localhost:4000/api

backend/.env.template

@@ -60,3 +60,4 @@ EVENT_PROTOCOL_DISABLED=$EVENT_PROTOCOL_DISABLED
 # Federation
 FEDERATION_DHT_TOPIC=$FEDERATION_DHT_TOPIC
 FEDERATION_DHT_SEED=$FEDERATION_DHT_SEED
+FEDERATION_COMMUNITY_URL=$FEDERATION_COMMUNITY_URL

backend/src/config/index.ts

@@ -10,14 +10,14 @@ Decimal.set({
 })
 
 const constants = {
-  DB_VERSION: '0057-clear_old_password_junk',
+  DB_VERSION: '0058-add_communities_table',
   DECAY_START_TIME: new Date('2021-05-13 17:46:31-0000'), // GMT+0
   LOG4JS_CONFIG: 'log4js-config.json',
   // default log level on production should be info
   LOG_LEVEL: process.env.LOG_LEVEL || 'info',
   CONFIG_VERSION: {
     DEFAULT: 'DEFAULT',
-    EXPECTED: 'v13.2022-12-20',
+    EXPECTED: 'v14.2022-11-22',
     CURRENT: '',
   },
 }
@@ -120,6 +120,12 @@ if (
 const federation = {
   FEDERATION_DHT_TOPIC: process.env.FEDERATION_DHT_TOPIC || null,
   FEDERATION_DHT_SEED: process.env.FEDERATION_DHT_SEED || null,
+  FEDERATION_COMMUNITY_URL:
+    process.env.FEDERATION_COMMUNITY_URL === undefined
+      ? null
+      : process.env.FEDERATION_COMMUNITY_URL.endsWith('/')
+      ? process.env.FEDERATION_COMMUNITY_URL
+      : process.env.FEDERATION_COMMUNITY_URL + '/',
 }
 
 const CONFIG = {

backend/src/federation/index.test.ts

@@ -0,0 +1,798 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
+
+import { startDHT } from './index'
+import DHT from '@hyperswarm/dht'
+import CONFIG from '@/config'
+import { logger } from '@test/testSetup'
+import { Community as DbCommunity } from '@entity/Community'
+import { testEnvironment, cleanDB } from '@test/helpers'
+
+CONFIG.FEDERATION_DHT_SEED = '64ebcb0e3ad547848fef4197c6e2332f'
+
+jest.mock('@hyperswarm/dht')
+
+const TEST_TOPIC = 'gradido_test_topic'
+
+const keyPairMock = {
+  publicKey: Buffer.from('publicKey'),
+  secretKey: Buffer.from('secretKey'),
+}
+
+const serverListenSpy = jest.fn()
+
+const serverEventMocks: { [key: string]: any } = {}
+
+const serverOnMock = jest.fn().mockImplementation((key: string, callback) => {
+  serverEventMocks[key] = callback
+})
+
+const nodeCreateServerMock = jest.fn().mockImplementation(() => {
+  return {
+    on: serverOnMock,
+    listen: serverListenSpy,
+  }
+})
+
+const nodeAnnounceMock = jest.fn().mockImplementation(() => {
+  return {
+    finished: jest.fn(),
+  }
+})
+
+const lookupResultMock = {
+  token: Buffer.from(TEST_TOPIC),
+  from: {
+    id: Buffer.from('somone'),
+    host: '188.95.53.5',
+    port: 63561,
+  },
+  to: { id: null, host: '83.53.31.27', port: 55723 },
+  peers: [
+    {
+      publicKey: Buffer.from('some-public-key'),
+      relayAddresses: [],
+    },
+  ],
+}
+
+const nodeLookupMock = jest.fn().mockResolvedValue([lookupResultMock])
+
+const socketEventMocks: { [key: string]: any } = {}
+
+const socketOnMock = jest.fn().mockImplementation((key: string, callback) => {
+  socketEventMocks[key] = callback
+})
+
+const socketWriteMock = jest.fn()
+
+const nodeConnectMock = jest.fn().mockImplementation(() => {
+  return {
+    on: socketOnMock,
+    once: socketOnMock,
+    write: socketWriteMock,
+  }
+})
+
+DHT.hash.mockImplementation(() => {
+  return Buffer.from(TEST_TOPIC)
+})
+
+DHT.keyPair.mockImplementation(() => {
+  return keyPairMock
+})
+
+DHT.mockImplementation(() => {
+  return {
+    createServer: nodeCreateServerMock,
+    announce: nodeAnnounceMock,
+    lookup: nodeLookupMock,
+    connect: nodeConnectMock,
+  }
+})
+
+let con: any
+let testEnv: any
+
+beforeAll(async () => {
+  testEnv = await testEnvironment(logger)
+  con = testEnv.con
+  await cleanDB()
+})
+
+afterAll(async () => {
+  await cleanDB()
+  await con.close()
+})
+
+describe('federation', () => {
+  beforeAll(() => {
+    jest.useFakeTimers()
+  })
+
+  describe('call startDHT', () => {
+    const hashSpy = jest.spyOn(DHT, 'hash')
+    const keyPairSpy = jest.spyOn(DHT, 'keyPair')
+    beforeEach(async () => {
+      DHT.mockClear()
+      jest.clearAllMocks()
+      await startDHT(TEST_TOPIC)
+    })
+
+    it('calls DHT.hash', () => {
+      expect(hashSpy).toBeCalledWith(Buffer.from(TEST_TOPIC))
+    })
+
+    it('creates a key pair', () => {
+      expect(keyPairSpy).toBeCalledWith(expect.any(Buffer))
+    })
+
+    it('initializes a new DHT object', () => {
+      expect(DHT).toBeCalledWith({ keyPair: keyPairMock })
+    })
+
+    describe('DHT node', () => {
+      it('creates a server', () => {
+        expect(nodeCreateServerMock).toBeCalled()
+      })
+
+      it('listens on the server', () => {
+        expect(serverListenSpy).toBeCalled()
+      })
+
+      describe('timers', () => {
+        beforeEach(() => {
+          jest.runOnlyPendingTimers()
+        })
+
+        it('announces on topic', () => {
+          expect(nodeAnnounceMock).toBeCalledWith(Buffer.from(TEST_TOPIC), keyPairMock)
+        })
+
+        it('looks up on topic', () => {
+          expect(nodeLookupMock).toBeCalledWith(Buffer.from(TEST_TOPIC))
+        })
+      })
+
+      describe('server connection event', () => {
+        beforeEach(() => {
+          serverEventMocks.connection({
+            remotePublicKey: Buffer.from('another-public-key'),
+            on: socketOnMock,
+          })
+        })
+
+        it('can be triggered', () => {
+          expect(socketOnMock).toBeCalled()
+        })
+
+        describe('socket events', () => {
+          describe('on data', () => {
+            describe('with receiving simply a string', () => {
+              beforeEach(() => {
+                jest.clearAllMocks()
+                socketEventMocks.data(Buffer.from('no-json string'))
+              })
+
+              it('logs the received data', () => {
+                expect(logger.info).toBeCalledWith('data: no-json string')
+              })
+
+              it('logs an error of unexpected data format and structure', () => {
+                expect(logger.error).toBeCalledWith(
+                  'Error on receiving data from socket:',
+                  new SyntaxError('Unexpected token o in JSON at position 1'),
+                )
+              })
+            })
+
+            describe('with receiving array of strings', () => {
+              beforeEach(() => {
+                jest.clearAllMocks()
+                const strArray: string[] = ['invalid type test', 'api', 'url']
+                socketEventMocks.data(Buffer.from(strArray.toString()))
+              })
+
+              it('logs the received data', () => {
+                expect(logger.info).toBeCalledWith('data: invalid type test,api,url')
+              })
+
+              it('logs an error of unexpected data format and structure', () => {
+                expect(logger.error).toBeCalledWith(
+                  'Error on receiving data from socket:',
+                  new SyntaxError('Unexpected token i in JSON at position 0'),
+                )
+              })
+            })
+
+            describe('with receiving array of string-arrays', () => {
+              beforeEach(async () => {
+                jest.clearAllMocks()
+                const strArray: string[][] = [
+                  [`api`, `url`, `invalid type in array test`],
+                  [`wrong`, `api`, `url`],
+                ]
+                await socketEventMocks.data(Buffer.from(strArray.toString()))
+              })
+
+              it('logs the received data', () => {
+                expect(logger.info).toBeCalledWith(
+                  'data: api,url,invalid type in array test,wrong,api,url',
+                )
+              })
+
+              it('logs an error of unexpected data format and structure', () => {
+                expect(logger.error).toBeCalledWith(
+                  'Error on receiving data from socket:',
+                  new SyntaxError('Unexpected token a in JSON at position 0'),
+                )
+              })
+            })
+
+            describe('with receiving JSON-Array with too much entries', () => {
+              let jsonArray: { api: string; url: string }[]
+              beforeEach(async () => {
+                jest.clearAllMocks()
+                jsonArray = [
+                  { api: 'v1_0', url: 'too much versions at the same time test' },
+                  { api: 'v1_0', url: 'url2' },
+                  { api: 'v1_0', url: 'url3' },
+                  { api: 'v1_0', url: 'url4' },
+                  { api: 'v1_0', url: 'url5' },
+                ]
+                await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
+              })
+
+              it('logs the received data', () => {
+                expect(logger.info).toBeCalledWith(
+                  'data: [{"api":"v1_0","url":"too much versions at the same time test"},{"api":"v1_0","url":"url2"},{"api":"v1_0","url":"url3"},{"api":"v1_0","url":"url4"},{"api":"v1_0","url":"url5"}]',
+                )
+              })
+
+              it('logs a warning of too much apiVersion-Definitions', () => {
+                expect(logger.warn).toBeCalledWith(
+                  `received totaly wrong or too much apiVersions-Definition JSON-String: ${JSON.stringify(
+                    jsonArray,
+                  )}`,
+                )
+              })
+            })
+
+            describe('with receiving wrong but tolerated property data', () => {
+              let jsonArray: any[]
+              let result: DbCommunity[] = []
+              beforeAll(async () => {
+                jest.clearAllMocks()
+                jsonArray = [
+                  {
+                    wrong: 'wrong but tolerated property test',
+                    api: 'v1_0',
+                    url: 'url1',
+                  },
+                  {
+                    api: 'v2_0',
+                    url: 'url2',
+                    wrong: 'wrong but tolerated property test',
+                  },
+                ]
+                await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
+                result = await DbCommunity.find()
+              })
+
+              afterAll(async () => {
+                await cleanDB()
+              })
+
+              it('has two Communty entries in database', () => {
+                expect(result).toHaveLength(2)
+              })
+
+              it('has an entry for api version v1_0', () => {
+                expect(result).toEqual(
+                  expect.arrayContaining([
+                    expect.objectContaining({
+                      id: expect.any(Number),
+                      publicKey: expect.any(Buffer),
+                      apiVersion: 'v1_0',
+                      endPoint: 'url1',
+                      lastAnnouncedAt: expect.any(Date),
+                      createdAt: expect.any(Date),
+                      updatedAt: null,
+                    }),
+                  ]),
+                )
+              })
+
+              it('has an entry for api version v2_0', () => {
+                expect(result).toEqual(
+                  expect.arrayContaining([
+                    expect.objectContaining({
+                      id: expect.any(Number),
+                      publicKey: expect.any(Buffer),
+                      apiVersion: 'v2_0',
+                      endPoint: 'url2',
+                      lastAnnouncedAt: expect.any(Date),
+                      createdAt: expect.any(Date),
+                      updatedAt: null,
+                    }),
+                  ]),
+                )
+              })
+            })
+
+            describe('with receiving data but missing api property', () => {
+              let jsonArray: any[]
+              beforeEach(async () => {
+                jest.clearAllMocks()
+                jsonArray = [
+                  { test1: 'missing api proterty test', url: 'any url definition as string' },
+                  { api: 'some api', test2: 'missing url property test' },
+                ]
+                await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
+              })
+
+              it('logs the received data', () => {
+                expect(logger.info).toBeCalledWith(`data: ${JSON.stringify(jsonArray)}`)
+              })
+
+              it('logs a warning of invalid apiVersion-Definition', () => {
+                expect(logger.warn).toBeCalledWith(
+                  `received invalid apiVersion-Definition: ${JSON.stringify(jsonArray[0])}`,
+                )
+              })
+            })
+
+            describe('with receiving data but missing url property', () => {
+              let jsonArray: any[]
+              beforeEach(async () => {
+                jest.clearAllMocks()
+                jsonArray = [
+                  { api: 'some api', test2: 'missing url property test' },
+                  { test1: 'missing api proterty test', url: 'any url definition as string' },
+                ]
+                await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
+              })
+
+              it('logs the received data', () => {
+                expect(logger.info).toBeCalledWith(`data: ${JSON.stringify(jsonArray)}`)
+              })
+
+              it('logs a warning of invalid apiVersion-Definition', () => {
+                expect(logger.warn).toBeCalledWith(
+                  `received invalid apiVersion-Definition: ${JSON.stringify(jsonArray[0])}`,
+                )
+              })
+            })
+
+            describe('with receiving data but wrong type of api property', () => {
+              let jsonArray: any[]
+              beforeEach(async () => {
+                jest.clearAllMocks()
+                jsonArray = [
+                  { api: 1, url: 'wrong property type tests' },
+                  { api: 'urltyptest', url: 2 },
+                  { api: 1, url: 2 },
+                ]
+                await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
+              })
+
+              it('logs the received data', () => {
+                expect(logger.info).toBeCalledWith(`data: ${JSON.stringify(jsonArray)}`)
+              })
+
+              it('logs a warning of invalid apiVersion-Definition', () => {
+                expect(logger.warn).toBeCalledWith(
+                  `received invalid apiVersion-Definition: ${JSON.stringify(jsonArray[0])}`,
+                )
+              })
+            })
+
+            describe('with receiving data but wrong type of url property', () => {
+              let jsonArray: any[]
+              beforeEach(async () => {
+                jest.clearAllMocks()
+                jsonArray = [
+                  { api: 'urltyptest', url: 2 },
+                  { api: 1, url: 'wrong property type tests' },
+                  { api: 1, url: 2 },
+                ]
+                await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
+              })
+
+              it('logs the received data', () => {
+                expect(logger.info).toBeCalledWith(`data: ${JSON.stringify(jsonArray)}`)
+              })
+
+              it('logs a warning of invalid apiVersion-Definition', () => {
+                expect(logger.warn).toBeCalledWith(
+                  `received invalid apiVersion-Definition: ${JSON.stringify(jsonArray[0])}`,
+                )
+              })
+            })
+
+            describe('with receiving data but wrong type of both properties', () => {
+              let jsonArray: any[]
+              beforeEach(async () => {
+                jest.clearAllMocks()
+                jsonArray = [
+                  { api: 1, url: 2 },
+                  { api: 'urltyptest', url: 2 },
+                  { api: 1, url: 'wrong property type tests' },
+                ]
+                await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
+              })
+
+              it('logs the received data', () => {
+                expect(logger.info).toBeCalledWith(`data: ${JSON.stringify(jsonArray)}`)
+              })
+
+              it('logs a warning of invalid apiVersion-Definition', () => {
+                expect(logger.warn).toBeCalledWith(
+                  `received invalid apiVersion-Definition: ${JSON.stringify(jsonArray[0])}`,
+                )
+              })
+            })
+
+            describe('with receiving data but too long api string', () => {
+              let jsonArray: any[]
+              beforeEach(async () => {
+                jest.clearAllMocks()
+                jsonArray = [
+                  { api: 'toolong api', url: 'some valid url' },
+                  {
+                    api: 'valid  api',
+                    url: 'this is a too long url definition with exact one character more than the allowed two hundert and fiftyfive characters. and here begins the fill characters with no sense of content menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmic',
+                  },
+                  {
+                    api: 'toolong api',
+                    url: 'this is a too long url definition with exact one character more than the allowed two hundert and fiftyfive characters. and here begins the fill characters with no sense of content menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmic',
+                  },
+                ]
+                await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
+              })
+
+              it('logs the received data', () => {
+                expect(logger.info).toBeCalledWith(`data: ${JSON.stringify(jsonArray)}`)
+              })
+
+              it('logs a warning of invalid apiVersion-Definition', () => {
+                expect(logger.warn).toBeCalledWith(
+                  `received apiVersion with content longer than max length: ${JSON.stringify(
+                    jsonArray[0],
+                  )}`,
+                )
+              })
+            })
+
+            describe('with receiving data but too long url string', () => {
+              let jsonArray: any[]
+              beforeEach(async () => {
+                jest.clearAllMocks()
+                jsonArray = [
+                  {
+                    api: 'api',
+                    url: 'this is a too long url definition with exact one character more than the allowed two hundert and fiftyfive characters. and here begins the fill characters with no sense of content menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmic',
+                  },
+                  { api: 'toolong api', url: 'some valid url' },
+                  {
+                    api: 'toolong api',
+                    url: 'this is a too long url definition with exact one character more than the allowed two hundert and fiftyfive characters. and here begins the fill characters with no sense of content menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmic',
+                  },
+                ]
+                await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
+              })
+
+              it('logs the received data', () => {
+                expect(logger.info).toBeCalledWith(`data: ${JSON.stringify(jsonArray)}`)
+              })
+
+              it('logs a warning of invalid apiVersion-Definition', () => {
+                expect(logger.warn).toBeCalledWith(
+                  `received apiVersion with content longer than max length: ${JSON.stringify(
+                    jsonArray[0],
+                  )}`,
+                )
+              })
+            })
+
+            describe('with receiving data but both properties with too long strings', () => {
+              let jsonArray: any[]
+              beforeEach(async () => {
+                jest.clearAllMocks()
+                jsonArray = [
+                  {
+                    api: 'toolong api',
+                    url: 'this is a too long url definition with exact one character more than the allowed two hundert and fiftyfive characters. and here begins the fill characters with no sense of content menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmic',
+                  },
+                  {
+                    api: 'api',
+                    url: 'this is a too long url definition with exact one character more than the allowed two hundert and fiftyfive characters. and here begins the fill characters with no sense of content menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmic',
+                  },
+                  { api: 'toolong api', url: 'some valid url' },
+                ]
+                await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
+              })
+
+              it('logs the received data', () => {
+                expect(logger.info).toBeCalledWith(`data: ${JSON.stringify(jsonArray)}`)
+              })
+            })
+
+            describe('with receiving data of exact max allowed properties length', () => {
+              let jsonArray: any[]
+              let result: DbCommunity[] = []
+              beforeAll(async () => {
+                jest.clearAllMocks()
+                jsonArray = [
+                  {
+                    api: 'valid  api',
+                    url: 'this is a valid url definition with exact the max allowed length of two hundert and fiftyfive characters. and here begins the fill characters with no sense of content kuhwarmiga menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmich',
+                  },
+                  {
+                    api: 'api',
+                    url: 'this is a too long url definition with exact one character more than the allowed two hundert and fiftyfive characters. and here begins the fill characters with no sense of content menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmic',
+                  },
+                  { api: 'toolong api', url: 'some valid url' },
+                ]
+                await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
+                result = await DbCommunity.find()
+              })
+
+              afterAll(async () => {
+                await cleanDB()
+              })
+
+              it('has one Communty entry in database', () => {
+                expect(result).toHaveLength(1)
+              })
+
+              it(`has an entry with max content length for api and url`, () => {
+                expect(result).toEqual(
+                  expect.arrayContaining([
+                    expect.objectContaining({
+                      id: expect.any(Number),
+                      publicKey: expect.any(Buffer),
+                      apiVersion: 'valid  api',
+                      endPoint:
+                        'this is a valid url definition with exact the max allowed length of two hundert and fiftyfive characters. and here begins the fill characters with no sense of content kuhwarmiga menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmich',
+                      lastAnnouncedAt: expect.any(Date),
+                      createdAt: expect.any(Date),
+                      updatedAt: null,
+                    }),
+                  ]),
+                )
+              })
+            })
+
+            describe('with receiving data of exact max allowed buffer length', () => {
+              let jsonArray: any[]
+              let result: DbCommunity[] = []
+              beforeAll(async () => {
+                jest.clearAllMocks()
+                jsonArray = [
+                  {
+                    api: 'valid api1',
+                    url: 'this is a valid url definition with exact the max allowed length of two hundert and fiftyfive characters. and here begins the fill characters with no sense of content kuhwarmiga menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmich',
+                  },
+                  {
+                    api: 'valid api2',
+                    url: 'this is a valid url definition with exact the max allowed length of two hundert and fiftyfive characters. and here begins the fill characters with no sense of content kuhwarmiga menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmich',
+                  },
+                  {
+                    api: 'valid api3',
+                    url: 'this is a valid url definition with exact the max allowed length of two hundert and fiftyfive characters. and here begins the fill characters with no sense of content kuhwarmiga menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmich',
+                  },
+                  {
+                    api: 'valid api4',
+                    url: 'this is a valid url definition with exact the max allowed length of two hundert and fiftyfive characters. and here begins the fill characters with no sense of content kuhwarmiga menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmich',
+                  },
+                ]
+                await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
+                result = await DbCommunity.find()
+              })
+
+              afterAll(async () => {
+                await cleanDB()
+              })
+
+              it('has five Communty entries in database', () => {
+                expect(result).toHaveLength(4)
+              })
+
+              it(`has an entry 'valid api1' with max content length for api and url`, () => {
+                expect(result).toEqual(
+                  expect.arrayContaining([
+                    expect.objectContaining({
+                      id: expect.any(Number),
+                      publicKey: expect.any(Buffer),
+                      apiVersion: 'valid api1',
+                      endPoint:
+                        'this is a valid url definition with exact the max allowed length of two hundert and fiftyfive characters. and here begins the fill characters with no sense of content kuhwarmiga menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmich',
+                      lastAnnouncedAt: expect.any(Date),
+                      createdAt: expect.any(Date),
+                      updatedAt: null,
+                    }),
+                  ]),
+                )
+              })
+
+              it(`has an entry 'valid api2' with max content length for api and url`, () => {
+                expect(result).toEqual(
+                  expect.arrayContaining([
+                    expect.objectContaining({
+                      id: expect.any(Number),
+                      publicKey: expect.any(Buffer),
+                      apiVersion: 'valid api2',
+                      endPoint:
+                        'this is a valid url definition with exact the max allowed length of two hundert and fiftyfive characters. and here begins the fill characters with no sense of content kuhwarmiga menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmich',
+                      lastAnnouncedAt: expect.any(Date),
+                      createdAt: expect.any(Date),
+                      updatedAt: null,
+                    }),
+                  ]),
+                )
+              })
+
+              it(`has an entry 'valid api3' with max content length for api and url`, () => {
+                expect(result).toEqual(
+                  expect.arrayContaining([
+                    expect.objectContaining({
+                      id: expect.any(Number),
+                      publicKey: expect.any(Buffer),
+                      apiVersion: 'valid api3',
+                      endPoint:
+                        'this is a valid url definition with exact the max allowed length of two hundert and fiftyfive characters. and here begins the fill characters with no sense of content kuhwarmiga menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmich',
+                      lastAnnouncedAt: expect.any(Date),
+                      createdAt: expect.any(Date),
+                      updatedAt: null,
+                    }),
+                  ]),
+                )
+              })
+
+              it(`has an entry 'valid api4' with max content length for api and url`, () => {
+                expect(result).toEqual(
+                  expect.arrayContaining([
+                    expect.objectContaining({
+                      id: expect.any(Number),
+                      publicKey: expect.any(Buffer),
+                      apiVersion: 'valid api4',
+                      endPoint:
+                        'this is a valid url definition with exact the max allowed length of two hundert and fiftyfive characters. and here begins the fill characters with no sense of content kuhwarmiga menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmich',
+                      lastAnnouncedAt: expect.any(Date),
+                      createdAt: expect.any(Date),
+                      updatedAt: null,
+                    }),
+                  ]),
+                )
+              })
+            })
+
+            describe('with receiving data longer than max allowed buffer length', () => {
+              let jsonArray: any[]
+              beforeEach(async () => {
+                jest.clearAllMocks()
+                jsonArray = [
+                  {
+                    api: 'Xvalid api1',
+                    url: 'this is a valid url definition with exact the max allowed length of two hundert and fiftyfive characters. and here begins the fill characters with no sense of content kuhwarmiga menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmich',
+                  },
+                  {
+                    api: 'valid api2',
+                    url: 'this is a valid url definition with exact the max allowed length of two hundert and fiftyfive characters. and here begins the fill characters with no sense of content kuhwarmiga menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmich',
+                  },
+                  {
+                    api: 'valid api3',
+                    url: 'this is a valid url definition with exact the max allowed length of two hundert and fiftyfive characters. and here begins the fill characters with no sense of content kuhwarmiga menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmich',
+                  },
+                  {
+                    api: 'valid api4',
+                    url: 'this is a valid url definition with exact the max allowed length of two hundert and fiftyfive characters. and here begins the fill characters with no sense of content kuhwarmiga menschhabicheinhungerdassichnichtweiswoichheutnachtschlafensollsofriertesmich',
+                  },
+                ]
+                await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
+              })
+
+              it('logs the received data', () => {
+                expect(logger.warn).toBeCalledWith(
+                  `received more than max allowed length of data buffer: ${
+                    JSON.stringify(jsonArray).length
+                  } against 1141 max allowed`,
+                )
+              })
+            })
+
+            describe('with proper data', () => {
+              let result: DbCommunity[] = []
+              beforeAll(async () => {
+                jest.clearAllMocks()
+                await socketEventMocks.data(
+                  Buffer.from(
+                    JSON.stringify([
+                      {
+                        api: 'v1_0',
+                        url: 'http://localhost:4000/api/v1_0',
+                      },
+                      {
+                        api: 'v2_0',
+                        url: 'http://localhost:4000/api/v2_0',
+                      },
+                    ]),
+                  ),
+                )
+                result = await DbCommunity.find()
+              })
+
+              afterAll(async () => {
+                await cleanDB()
+              })
+
+              it('has two Communty entries in database', () => {
+                expect(result).toHaveLength(2)
+              })
+
+              it('has an entry for api version v1_0', () => {
+                expect(result).toEqual(
+                  expect.arrayContaining([
+                    expect.objectContaining({
+                      id: expect.any(Number),
+                      publicKey: expect.any(Buffer),
+                      apiVersion: 'v1_0',
+                      endPoint: 'http://localhost:4000/api/v1_0',
+                      lastAnnouncedAt: expect.any(Date),
+                      createdAt: expect.any(Date),
+                      updatedAt: null,
+                    }),
+                  ]),
+                )
+              })
+
+              it('has an entry for api version v2_0', () => {
+                expect(result).toEqual(
+                  expect.arrayContaining([
+                    expect.objectContaining({
+                      id: expect.any(Number),
+                      publicKey: expect.any(Buffer),
+                      apiVersion: 'v2_0',
+                      endPoint: 'http://localhost:4000/api/v2_0',
+                      lastAnnouncedAt: expect.any(Date),
+                      createdAt: expect.any(Date),
+                      updatedAt: null,
+                    }),
+                  ]),
+                )
+              })
+            })
+          })
+
+          describe('on open', () => {
+            beforeEach(() => {
+              socketEventMocks.open()
+            })
+
+            it.skip('calls socket write with own api versions', () => {
+              expect(socketWriteMock).toBeCalledWith(
+                Buffer.from(
+                  JSON.stringify([
+                    {
+                      api: 'v1_0',
+                      url: 'http://localhost:4000/api/v1_0',
+                    },
+                    {
+                      api: 'v1_1',
+                      url: 'http://localhost:4000/api/v1_1',
+                    },
+                    {
+                      api: 'v2_0',
+                      url: 'http://localhost:4000/api/v2_0',
+                    },
+                  ]),
+                ),
+              )
+            })
+          })
+        })
+      })
+    })
+  })
+})

backend/src/federation/index.ts

@@ -1,14 +1,9 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 /* eslint-disable @typescript-eslint/explicit-module-boundary-types */
-
 import DHT from '@hyperswarm/dht'
-// import { Connection } from '@dbTools/typeorm'
 import { backendLogger as logger } from '@/server/logger'
 import CONFIG from '@/config'
-
-function between(min: number, max: number) {
-  return Math.floor(Math.random() * (max - min + 1) + min)
-}
+import { Community as DbCommunity } from '@entity/Community'
 
 const KEY_SECRET_SEEDBYTES = 32
 const getSeed = (): Buffer | null =>
@@ -18,37 +13,107 @@ const POLLTIME = 20000
 const SUCCESSTIME = 120000
 const ERRORTIME = 240000
 const ANNOUNCETIME = 30000
-const nodeRand = between(1, 99)
-const nodeURL = `https://test${nodeRand}.org`
-const nodeAPI = {
-  API_1_00: `${nodeURL}/api/1_00/`,
-  API_1_01: `${nodeURL}/api/1_01/`,
-  API_2_00: `${nodeURL}/graphql/2_00/`,
+
+enum ApiVersionType {
+  V1_0 = 'v1_0',
+  V1_1 = 'v1_1',
+  V2_0 = 'v2_0',
+}
+type CommunityApi = {
+  api: string
+  url: string
 }
 
-export const startDHT = async (
-  // connection: Connection,
-  topic: string,
-): Promise<void> => {
+export const startDHT = async (topic: string): Promise<void> => {
   try {
     const TOPIC = DHT.hash(Buffer.from(topic))
     const keyPair = DHT.keyPair(getSeed())
     logger.info(`keyPairDHT: publicKey=${keyPair.publicKey.toString('hex')}`)
     logger.debug(`keyPairDHT: secretKey=${keyPair.secretKey.toString('hex')}`)
 
+    const ownApiVersions = Object.values(ApiVersionType).map(function (apiEnum) {
+      const comApi: CommunityApi = {
+        api: apiEnum,
+        url: CONFIG.FEDERATION_COMMUNITY_URL + apiEnum,
+      }
+      return comApi
+    })
+    logger.debug(`ApiList: ${JSON.stringify(ownApiVersions)}`)
+
     const node = new DHT({ keyPair })
 
     const server = node.createServer()
 
     server.on('connection', function (socket: any) {
-      // noiseSocket is E2E between you and the other peer
-      // pipe it somewhere like any duplex stream
-      logger.info(`Remote public key: ${socket.remotePublicKey.toString('hex')}`)
-      // console.log("Local public key", noiseSocket.publicKey.toString("hex")); // same as keyPair.publicKey
+      logger.info(`server on... with Remote public key: ${socket.remotePublicKey.toString('hex')}`)
 
-      socket.on('data', (data: Buffer) => logger.info(`data: ${data.toString('ascii')}`))
+      socket.on('data', async (data: Buffer) => {
+        try {
+          if (data.length > 1141) {
+            logger.warn(
+              `received more than max allowed length of data buffer: ${data.length} against 1141 max allowed`,
+            )
+            return
+          }
+          logger.info(`data: ${data.toString('ascii')}`)
+          const recApiVersions: CommunityApi[] = JSON.parse(data.toString('ascii'))
 
-      // process.stdin.pipe(noiseSocket).pipe(process.stdout);
+          // TODO better to introduce the validation by https://github.com/typestack/class-validato
+          if (recApiVersions && Array.isArray(recApiVersions) && recApiVersions.length < 5) {
+            for (const recApiVersion of recApiVersions) {
+              if (
+                !recApiVersion.api ||
+                typeof recApiVersion.api !== 'string' ||
+                !recApiVersion.url ||
+                typeof recApiVersion.url !== 'string'
+              ) {
+                logger.warn(
+                  `received invalid apiVersion-Definition: ${JSON.stringify(recApiVersion)}`,
+                )
+                // in a forEach-loop use return instead of continue
+                return
+              }
+              // TODO better to introduce the validation on entity-Level by https://github.com/typestack/class-validator
+              if (recApiVersion.api.length > 10 || recApiVersion.url.length > 255) {
+                logger.warn(
+                  `received apiVersion with content longer than max length: ${JSON.stringify(
+                    recApiVersion,
+                  )}`,
+                )
+                // in a forEach-loop use return instead of continue
+                return
+              }
+
+              const variables = {
+                apiVersion: recApiVersion.api,
+                endPoint: recApiVersion.url,
+                publicKey: socket.remotePublicKey.toString('hex'),
+                lastAnnouncedAt: new Date(),
+              }
+              logger.debug(`upsert with variables=${JSON.stringify(variables)}`)
+              // this will NOT update the updatedAt column, to distingue between a normal update and the last announcement
+              await DbCommunity.createQueryBuilder()
+                .insert()
+                .into(DbCommunity)
+                .values(variables)
+                .orUpdate({
+                  conflict_target: ['id', 'publicKey', 'apiVersion'],
+                  overwrite: ['end_point', 'last_announced_at'],
+                })
+                .execute()
+              logger.info(`federation community upserted successfully...`)
+            }
+          } else {
+            logger.warn(
+              `received totaly wrong or too much apiVersions-Definition JSON-String: ${JSON.stringify(
+                recApiVersions,
+              )}`,
+            )
+          }
+        } catch (e) {
+          logger.error('Error on receiving data from socket:', e)
+        }
+      })
     })
 
     await server.listen()
@@ -93,7 +158,6 @@ export const startDHT = async (
       logger.info(`Found new peers: ${collectedPubKeys}`)
 
       collectedPubKeys.forEach((remotePubKey) => {
-        // publicKey here is keyPair.publicKey from above
         const socket = node.connect(Buffer.from(remotePubKey, 'hex'))
 
         // socket.once("connect", function () {
@@ -110,17 +174,12 @@ export const startDHT = async (
         })
 
         socket.on('open', function () {
-          // noiseSocket fully open with the other peer
-          // console.log("writing to socket");
-          socket.write(Buffer.from(`${nodeRand}`))
-          socket.write(Buffer.from(JSON.stringify(nodeAPI)))
+          socket.write(Buffer.from(JSON.stringify(ownApiVersions)))
           successfulRequests.push(remotePubKey)
         })
-        // pipe it somewhere like any duplex stream
-        // process.stdin.pipe(noiseSocket).pipe(process.stdout)
       })
     }, POLLTIME)
   } catch (err) {
-    logger.error(err)
+    logger.error('DHT unexpected error:', err)
   }
 }

backend/src/index.ts

@@ -20,6 +20,9 @@ async function main() {
 
   // start DHT hyperswarm when DHT_TOPIC is set in .env
   if (CONFIG.FEDERATION_DHT_TOPIC) {
+    if (CONFIG.FEDERATION_COMMUNITY_URL === null) {
+      throw Error(`Config-Error: missing configuration of property FEDERATION_COMMUNITY_URL`)
+    }
     // eslint-disable-next-line no-console
     console.log(
       `starting Federation on ${CONFIG.FEDERATION_DHT_TOPIC} ${

database/entity/0058-add_communities_table/Community.ts

@@ -0,0 +1,42 @@
+import {
+  BaseEntity,
+  Entity,
+  PrimaryGeneratedColumn,
+  Column,
+  CreateDateColumn,
+  UpdateDateColumn,
+} from 'typeorm'
+
+@Entity('communities')
+export class Community extends BaseEntity {
+  @PrimaryGeneratedColumn('increment', { unsigned: true })
+  id: number
+
+  @Column({ name: 'public_key', type: 'binary', length: 64, default: null, nullable: true })
+  publicKey: Buffer
+
+  @Column({ name: 'api_version', length: 10, nullable: false })
+  apiVersion: string
+
+  @Column({ name: 'end_point', length: 255, nullable: false })
+  endPoint: string
+
+  @Column({ name: 'last_announced_at', type: 'datetime', nullable: false })
+  lastAnnouncedAt: Date
+
+  @CreateDateColumn({
+    name: 'created_at',
+    type: 'datetime',
+    default: () => 'CURRENT_TIMESTAMP(3)',
+    nullable: false,
+  })
+  createdAt: Date
+
+  @UpdateDateColumn({
+    name: 'updated_at',
+    type: 'datetime',
+    onUpdate: 'CURRENT_TIMESTAMP(3)',
+    nullable: true,
+  })
+  updatedAt: Date | null
+}

database/entity/Community.ts

@@ -0,0 +1 @@
+export { Community } from './0058-add_communities_table/Community'

database/entity/index.ts

@@ -9,6 +9,7 @@ import { UserContact } from './UserContact'
 import { Contribution } from './Contribution'
 import { EventProtocol } from './EventProtocol'
 import { ContributionMessage } from './ContributionMessage'
+import { Community } from './Community'
 
 export const entities = [
   Contribution,
@@ -22,4 +23,5 @@ export const entities = [
   EventProtocol,
   ContributionMessage,
   UserContact,
+  Community,
 ]

database/migrations/0058-add_communities_table.ts

@@ -0,0 +1,28 @@
+/* MIGRATION TO CREATE THE FEDERATION COMMUNITY TABLES
+ *
+ * This migration creates the `community` and 'communityfederation' tables in the `apollo` database (`gradido_community`).
+ */
+
+/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+export async function upgrade(queryFn: (query: string, values?: any[]) => Promise<Array<any>>) {
+  await queryFn(`
+    CREATE TABLE communities (
+      id int unsigned NOT NULL AUTO_INCREMENT,
+      public_key binary(64),
+      api_version varchar(10) NOT NULL,
+      end_point varchar(255) NOT NULL,
+      last_announced_at datetime(3) NOT NULL,
+      created_at datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
+      updated_at datetime(3),
+      PRIMARY KEY (id),
+      UNIQUE KEY public_api_key (public_key, api_version)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+  `)
+}
+
+export async function downgrade(queryFn: (query: string, values?: any[]) => Promise<Array<any>>) {
+  // write downgrade logic as parameter of queryFn
+  await queryFn(`DROP TABLE communities;`)
+}