IT4Change/gradido
3
0
Fork 0
mirror of https://github.com/IT4Change/gradido.git synced 2025-12-12 23:35:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch

Browse Source 
'origin/2650-feature-federation-harmonize-and-sync-modules-of-federation'
into 2501-feature-federation-implement-a-graphql-client-to-request-getpublickey
This commit is contained in:
Claus-Peter Hübner 2023-02-07 00:55:08 +01:00
commit e4082fbeb1
9 changed files with 182 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/config/index.ts
View File

@ -10,7 +10,7 @@ Decimal.set({
})
const constants = {
DB_VERSION: '0060-add_verified_at_to communities',
DB_VERSION: '0061-update_communities_table',
DECAY_START_TIME: new Date('2021-05-13 17:46:31-0000'), // GMT+0
LOG4JS_CONFIG: 'log4js-config.json',
// default log level on production should be info

51
database/entity/0061-update_communities_table/Community.ts Normal file
View File

@ -0,0 +1,51 @@
import {
BaseEntity,
Entity,
PrimaryGeneratedColumn,
Column,
CreateDateColumn,
UpdateDateColumn,
} from 'typeorm'
@Entity('communities')
export class Community extends BaseEntity {
@PrimaryGeneratedColumn('increment', { unsigned: true })
id: number
@Column({ name: 'foreign', type: 'bool', nullable: false, default: true })
foreign: boolean
@Column({ name: 'public_key', type: 'binary', length: 64, default: null, nullable: true })
publicKey: Buffer
@Column({ name: 'api_version', length: 10, nullable: false })
apiVersion: string
@Column({ name: 'end_point', length: 255, nullable: false })
endPoint: string
@Column({ name: 'last_announced_at', type: 'datetime', nullable: true })
lastAnnouncedAt: Date
@Column({ name: 'verified_at', type: 'datetime', nullable: true })
verifiedAt: Date
@Column({ name: 'last_error_at', type: 'datetime', nullable: true })
lastErrorAt: Date
@CreateDateColumn({
name: 'created_at',
type: 'datetime',
default: () => 'CURRENT_TIMESTAMP(3)',
nullable: false,
})
createdAt: Date
@UpdateDateColumn({
name: 'updated_at',
type: 'datetime',
onUpdate: 'CURRENT_TIMESTAMP(3)',
nullable: true,
})
updatedAt: Date | null
}

2
database/entity/Community.ts
View File

@ -1 +1 @@
export { Community } from './0060-add_verified_at_to_communities/Community'
export { Community } from './0061-update_communities_table/Community'

34
database/migrations/0061-update_communities_table.ts Normal file
View File

@ -0,0 +1,34 @@
/* MIGRATION TO CREATE THE FEDERATION COMMUNITY TABLES
*
* This migration creates the `community` and 'communityfederation' tables in the `apollo` database (`gradido_community`).
*/
/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
/* eslint-disable @typescript-eslint/no-explicit-any */
export async function upgrade(queryFn: (query: string, values?: any[]) => Promise<Array<any>>) {
await queryFn(
'ALTER TABLE `communities` MODIFY COLUMN `last_announced_at` datetime(3) AFTER `end_point`;',
)
await queryFn(
'ALTER TABLE `communities` ADD COLUMN `foreign` tinyint(4) NOT NULL DEFAULT 1 AFTER `id`;',
)
await queryFn(
'ALTER TABLE `communities` ADD COLUMN `verified_at` datetime(3) AFTER `last_announced_at`;',
)
/*
await queryFn(
'ALTER TABLE `communities` ADD COLUMN `last_error_at` datetime(3) AFTER `verified_at`;',
)
*/
}
export async function downgrade(queryFn: (query: string, values?: any[]) => Promise<Array<any>>) {
// write downgrade logic as parameter of queryFn
await queryFn(
'ALTER TABLE `communities` MODIFY COLUMN `last_announced_at` datetime(3) NOT NULL AFTER `end_point`;',
)
await queryFn('ALTER TABLE `communities` DROP COLUMN `foreign`;')
// await queryFn('ALTER TABLE `communities` DROP COLUMN `verified_at`;')
await queryFn('ALTER TABLE `communities` DROP COLUMN `last_error_at`;')
}

4
dht-node/.env.dist
View File

@ -1,4 +1,4 @@
CONFIG_VERSION=v1.2023-01-01
CONFIG_VERSION=v2.2023-02-03
# Database
DB_HOST=localhost
@ -20,3 +20,5 @@ EVENT_PROTOCOL_DISABLED=false
# on an hash created from this topic
FEDERATION_DHT_TOPIC=GRADIDO_HUB
# FEDERATION_DHT_SEED=64ebcb0e3ad547848fef4197c6e2332f
# FEDERATION_COMMUNITY_URL=http://localhost
# FEDERATION_COMMUNITY_API_PORT=5000

1
dht-node/.env.template
View File

@ -15,3 +15,4 @@ EVENT_PROTOCOL_DISABLED=$EVENT_PROTOCOL_DISABLED
FEDERATION_DHT_TOPIC=$FEDERATION_DHT_TOPIC
FEDERATION_DHT_SEED=$FEDERATION_DHT_SEED
FEDERATION_COMMUNITY_URL=$FEDERATION_COMMUNITY_URL
FEDERATION_COMMUNITY_API_PORT=FEDERATION_COMMUNITY_API_PORT

7
dht-node/src/config/index.ts
View File

@ -3,13 +3,13 @@ import dotenv from 'dotenv'
dotenv.config()
const constants = {
DB_VERSION: '0060-add_verified_at_to communities',
DB_VERSION: '0061-update_communities_table',
LOG4JS_CONFIG: 'log4js-config.json',
// default log level on production should be info
LOG_LEVEL: process.env.LOG_LEVEL || 'info',
CONFIG_VERSION: {
DEFAULT: 'DEFAULT',
EXPECTED: 'v1.2023-01-01',
EXPECTED: 'v2.2023-02-03',
CURRENT: '',
},
}
@ -36,7 +36,8 @@ const eventProtocol = {
const federation = {
FEDERATION_DHT_TOPIC: process.env.FEDERATION_DHT_TOPIC || 'GRADIDO_HUB',
FEDERATION_DHT_SEED: process.env.FEDERATION_DHT_SEED || null,
FEDERATION_COMMUNITY_URL: process.env.FEDERATION_COMMUNITY_URL || null,
FEDERATION_COMMUNITY_URL: process.env.FEDERATION_COMMUNITY_URL || 'http://localhost',
FEDERATION_COMMUNITY_API_PORT: process.env.FEDERATION_COMMUNITY_API_PORT || '5000',
}
// Check config version

75
dht-node/src/dht_node/index.test.ts
View File

@ -44,10 +44,11 @@ const lookupResultMock = {
token: Buffer.from(TEST_TOPIC),
from: {
id: Buffer.from('somone'),
foreign: true,
host: '188.95.53.5',
port: 63561,
},
to: { id: null, host: '83.53.31.27', port: 55723 },
to: { id: null, foreign: true, host: '83.53.31.27', port: 55723 },
peers: [
{
publicKey: Buffer.from('some-public-key'),
@ -116,6 +117,7 @@ describe('federation', () => {
beforeEach(async () => {
DHT.mockClear()
jest.clearAllMocks()
await cleanDB()
await startDHT(TEST_TOPIC)
})
@ -234,18 +236,18 @@ describe('federation', () => {
beforeEach(async () => {
jest.clearAllMocks()
jsonArray = [
{ api: 'v1_0', url: 'too much versions at the same time test' },
{ api: 'v1_0', url: 'url2' },
{ api: 'v1_0', url: 'url3' },
{ api: 'v1_0', url: 'url4' },
{ api: 'v1_0', url: 'url5' },
{ api: '1_0', url: 'too much versions at the same time test' },
{ api: '1_0', url: 'url2' },
{ api: '1_0', url: 'url3' },
{ api: '1_0', url: 'url4' },
{ api: '1_0', url: 'url5' },
]
await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
})
it('logs the received data', () => {
expect(logger.info).toBeCalledWith(
'data: [{"api":"v1_0","url":"too much versions at the same time test"},{"api":"v1_0","url":"url2"},{"api":"v1_0","url":"url3"},{"api":"v1_0","url":"url4"},{"api":"v1_0","url":"url5"}]',
'data: [{"api":"1_0","url":"too much versions at the same time test"},{"api":"1_0","url":"url2"},{"api":"1_0","url":"url3"},{"api":"1_0","url":"url4"},{"api":"1_0","url":"url5"}]',
)
})
@ -266,17 +268,17 @@ describe('federation', () => {
jsonArray = [
{
wrong: 'wrong but tolerated property test',
api: 'v1_0',
api: '1_0',
url: 'url1',
},
{
api: 'v2_0',
api: '2_0',
url: 'url2',
wrong: 'wrong but tolerated property test',
},
]
await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
result = await DbCommunity.find()
result = await DbCommunity.find({ foreign: true })
})
afterAll(async () => {
@ -287,13 +289,14 @@ describe('federation', () => {
expect(result).toHaveLength(2)
})
it('has an entry for api version v1_0', () => {
it('has an entry for api version 1_0', () => {
expect(result).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: expect.any(Number),
foreign: true,
publicKey: expect.any(Buffer),
apiVersion: 'v1_0',
apiVersion: '1_0',
endPoint: 'url1',
lastAnnouncedAt: expect.any(Date),
createdAt: expect.any(Date),
@ -303,13 +306,14 @@ describe('federation', () => {
)
})
it('has an entry for api version v2_0', () => {
it('has an entry for api version 2_0', () => {
expect(result).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: expect.any(Number),
foreign: true,
publicKey: expect.any(Buffer),
apiVersion: 'v2_0',
apiVersion: '2_0',
endPoint: 'url2',
lastAnnouncedAt: expect.any(Date),
createdAt: expect.any(Date),
@ -535,7 +539,7 @@ describe('federation', () => {
{ api: 'toolong api', url: 'some valid url' },
]
await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
result = await DbCommunity.find()
result = await DbCommunity.find({ foreign: true })
})
afterAll(async () => {
@ -551,6 +555,7 @@ describe('federation', () => {
expect.arrayContaining([
expect.objectContaining({
id: expect.any(Number),
foreign: true,
publicKey: expect.any(Buffer),
apiVersion: 'valid api',
endPoint:
@ -588,7 +593,7 @@ describe('federation', () => {
},
]
await socketEventMocks.data(Buffer.from(JSON.stringify(jsonArray)))
result = await DbCommunity.find()
result = await DbCommunity.find({ foreign: true })
})
afterAll(async () => {
@ -604,6 +609,7 @@ describe('federation', () => {
expect.arrayContaining([
expect.objectContaining({
id: expect.any(Number),
foreign: true,
publicKey: expect.any(Buffer),
apiVersion: 'valid api1',
endPoint:
@ -621,6 +627,7 @@ describe('federation', () => {
expect.arrayContaining([
expect.objectContaining({
id: expect.any(Number),
foreign: true,
publicKey: expect.any(Buffer),
apiVersion: 'valid api2',
endPoint:
@ -638,6 +645,7 @@ describe('federation', () => {
expect.arrayContaining([
expect.objectContaining({
id: expect.any(Number),
foreign: true,
publicKey: expect.any(Buffer),
apiVersion: 'valid api3',
endPoint:
@ -655,6 +663,7 @@ describe('federation', () => {
expect.arrayContaining([
expect.objectContaining({
id: expect.any(Number),
foreign: true,
publicKey: expect.any(Buffer),
apiVersion: 'valid api4',
endPoint:
@ -710,17 +719,17 @@ describe('federation', () => {
Buffer.from(
JSON.stringify([
{
api: 'v1_0',
url: 'http://localhost:4000/api/v1_0',
api: '1_0',
url: 'http://localhost:5001/api/',
},
{
api: 'v2_0',
url: 'http://localhost:4000/api/v2_0',
api: '2_0',
url: 'http://localhost:5002/api/',
},
]),
),
)
result = await DbCommunity.find()
result = await DbCommunity.find({ foreign: true })
})
afterAll(async () => {
@ -736,9 +745,10 @@ describe('federation', () => {
expect.arrayContaining([
expect.objectContaining({
id: expect.any(Number),
foreign: true,
publicKey: expect.any(Buffer),
apiVersion: 'v1_0',
endPoint: 'http://localhost:4000/api/v1_0',
apiVersion: '1_0',
endPoint: 'http://localhost:5001/api/',
lastAnnouncedAt: expect.any(Date),
createdAt: expect.any(Date),
updatedAt: null,
@ -747,14 +757,15 @@ describe('federation', () => {
)
})
it('has an entry for api version v2_0', () => {
it('has an entry for api version 2_0', () => {
expect(result).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: expect.any(Number),
foreign: true,
publicKey: expect.any(Buffer),
apiVersion: 'v2_0',
endPoint: 'http://localhost:4000/api/v2_0',
apiVersion: '2_0',
endPoint: 'http://localhost:5002/api/',
lastAnnouncedAt: expect.any(Date),
createdAt: expect.any(Date),
updatedAt: null,
@ -775,16 +786,16 @@ describe('federation', () => {
Buffer.from(
JSON.stringify([
{
api: 'v1_0',
url: 'http://localhost:4000/api/v1_0',
api: '1_0',
url: 'http://localhost:5001/api/',
},
{
api: 'v1_1',
url: 'http://localhost:4000/api/v1_1',
api: '1_1',
url: 'http://localhost:5002/api/',
},
{
api: 'v2_0',
url: 'http://localhost:4000/api/v2_0',
api: '2_0',
url: 'http://localhost:5003/api/',
},
]),
),

48
dht-node/src/dht_node/index.ts
View File

@ -15,9 +15,9 @@ const ERRORTIME = 240000
const ANNOUNCETIME = 30000
enum ApiVersionType {
V1_0 = 'v1_0',
V1_1 = 'v1_1',
V2_0 = 'v2_0',
V1_0 = '1_0',
V1_1 = '1_1',
V2_0 = '2_0',
}
type CommunityApi = {
api: string
@ -31,13 +31,16 @@ export const startDHT = async (topic: string): Promise<void> => {
logger.info(`keyPairDHT: publicKey=${keyPair.publicKey.toString('hex')}`)
logger.debug(`keyPairDHT: secretKey=${keyPair.secretKey.toString('hex')}`)
const ownApiVersions = writeHomeCommunityEnries(keyPair.publicKey)
/*
const ownApiVersions = Object.values(ApiVersionType).map(function (apiEnum) {
const comApi: CommunityApi = {
api: apiEnum,
url: CONFIG.FEDERATION_COMMUNITY_URL + apiEnum,
url: CONFIG.FEDERATION_COMMUNITY_URL,
}
return comApi
})
*/
logger.debug(`ApiList: ${JSON.stringify(ownApiVersions)}`)
const node = new DHT({ keyPair })
@ -184,3 +187,40 @@ export const startDHT = async (topic: string): Promise<void> => {
logger.error('DHT unexpected error:', err)
}
}
async function writeHomeCommunityEnries(pubKey: any): Promise<CommunityApi[]> {
const homeApiVersions: CommunityApi[] = Object.values(ApiVersionType).map(function (
apiEnum,
idx,
) {
const port = Number.parseInt(CONFIG.FEDERATION_COMMUNITY_API_PORT) + idx + 1
const comApi: CommunityApi = {
api: apiEnum,
url: CONFIG.FEDERATION_COMMUNITY_URL + ':' + port.toString() + '/api/',
}
return comApi
})
try {
// first remove privious existing homeCommunity entries
const homeComs = await DbCommunity.find({ foreign: false })
if (homeComs.length > 0) {
await DbCommunity.remove(homeComs)
}
homeApiVersions.forEach(async function (homeApi) {
const homeCom = new DbCommunity()
homeCom.foreign = false
homeCom.apiVersion = homeApi.api
homeCom.endPoint = homeApi.url
homeCom.publicKey = pubKey.toString('hex')
// this will NOT update the updatedAt column, to distingue between a normal update and the last announcement
await DbCommunity.insert(homeCom)
logger.info(`federation home-community inserted successfully: ${JSON.stringify(homeCom)}`)
})
} catch (err) {
throw new Error(`Federation: Error writing HomeCommunity-Entries: ${err}`)
}
return homeApiVersions
}