setup gradido node

dlt-connector/.gitignore

@@ -7,3 +7,4 @@ package-json.lock
 coverage
 # emacs
 *~
+gradido_node

dlt-connector/bun.lock

@@ -13,6 +13,7 @@
         "@sinclair/typemap": "^0.10.1",
         "@types/bun": "^1.2.17",
         "@types/uuid": "^8.3.4",
+        "async-mutex": "^0.5.0",
         "dotenv": "^10.0.0",
         "elysia": "1.3.8",
         "graphql-request": "^7.2.0",
@@ -306,6 +307,8 @@
 
     "async-limiter": ["async-limiter@1.0.1", "", {}, "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ=="],
 
+    "async-mutex": ["async-mutex@0.5.0", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA=="],
+
     "asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="],
 
     "atomic-sleep": ["atomic-sleep@1.0.0", "", {}, "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="],

dlt-connector/package.json

@@ -26,6 +26,7 @@
     "@sinclair/typemap": "^0.10.1",
     "@types/bun": "^1.2.17",
     "@types/uuid": "^8.3.4",
+    "async-mutex": "^0.5.0",
     "dotenv": "^10.0.0",
     "elysia": "1.3.8",
     "graphql-request": "^7.2.0",

dlt-connector/src/bootstrap/init.ts

@@ -8,6 +8,7 @@ import { SendToHieroContext } from '../interactions/sendToHiero/SendToHiero.cont
 import { Community, communitySchema } from '../schemas/transaction.schema'
 import { isPortOpenRetry } from '../utils/network'
 import { type AppContext, type AppContextClients } from './appContext'
+import { initGradidoNode } from './initGradidoNode'
 
 export function loadConfig(): Logger {
   // configure log4js
@@ -74,6 +75,9 @@ export async function checkGradidoNode(
   logger: Logger,
   homeCommunity: Community,
 ): Promise<void> {
+  // check if gradido node is running, if not setup and start it
+  await initGradidoNode(clients)
+
   // ask gradido node if community blockchain was created
   try {
     if (

dlt-connector/src/bootstrap/initGradidoNode.ts

@@ -0,0 +1,89 @@
+import { execSync } from 'node:child_process'
+import fs from 'node:fs'
+import path from 'node:path'
+import { gunzipSync } from 'node:zlib'
+import { getLogger } from 'log4js'
+import { exportCommunities } from '../client/GradidoNode/communities'
+import { GradidoNodeProcess } from '../client/GradidoNode/GradidoNodeProcess'
+import { HieroClient } from '../client/hiero/HieroClient'
+import { CONFIG } from '../config'
+import {
+  GRADIDO_NODE_HOME_FOLDER_NAME,
+  GRADIDO_NODE_RUNTIME_PATH,
+  LOG4JS_BASE_CATEGORY,
+} from '../config/const'
+import { checkFileExist, checkPathExist } from '../utils/filesystem'
+import { isPortOpen } from '../utils/network'
+import { AppContextClients } from './appContext'
+
+const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.bootstrap.initGradidoNode`)
+
+export async function initGradidoNode(clients: AppContextClients): Promise<void> {
+  const url = `http://localhost:${CONFIG.DLT_NODE_SERVER_PORT}`
+  const isOpen = await isPortOpen(url)
+  if (isOpen) {
+    logger.info(`GradidoNode is already running on ${url}`)
+    return
+  }
+
+  const gradidoNodeHomeFolder = path.join(
+    CONFIG.DLT_GRADIDO_NODE_SERVER_HOME_FOLDER,
+    GRADIDO_NODE_HOME_FOLDER_NAME,
+  )
+  // check folder, create when missing
+  checkPathExist(gradidoNodeHomeFolder, true)
+
+  await Promise.all([
+    // write Hedera Address Book
+    exportHederaAddressbooks(gradidoNodeHomeFolder, clients.hiero),
+    // check GradidoNode Runtime, download when missing
+    ensureGradidoNodeRuntimeAvailable(GRADIDO_NODE_RUNTIME_PATH),
+    // export communities to GradidoNode Folder
+    exportCommunities(gradidoNodeHomeFolder, clients.backend),
+  ])
+  GradidoNodeProcess.getInstance().start()
+}
+
+async function exportHederaAddressbooks(
+  homeFolder: string,
+  hieroClient: HieroClient,
+): Promise<void> {
+  const networkName = CONFIG.HIERO_HEDERA_NETWORK
+  const addressBook = await hieroClient.downloadAddressBook()
+  const addressBookPath = path.join(homeFolder, 'addressbook', `${networkName}.pb`)
+  checkPathExist(path.dirname(addressBookPath), true)
+  fs.writeFileSync(addressBookPath, addressBook.toBytes())
+}
+
+async function ensureGradidoNodeRuntimeAvailable(runtimeFileName: string): Promise<void> {
+  const runtimeFolder = path.dirname(runtimeFileName)
+  checkPathExist(runtimeFolder, true)
+  logger.debug(`GradidoNode Runtime: ${runtimeFileName}`)
+  if (!checkFileExist(runtimeFileName)) {
+    const runtimeArchiveFilename = createGradidoNodeRuntimeArchiveFilename()
+    const downloadUrl = new URL(
+      `https://github.com/gradido/gradido_node/releases/download/v${CONFIG.DLT_GRADIDO_NODE_SERVER_VERSION}/${runtimeArchiveFilename}`,
+    )
+    logger.debug(`download GradidoNode Runtime from ${downloadUrl}`)
+    const archive = await fetch(downloadUrl)
+    if (!archive.ok) {
+      throw new Error(`Failed to download GradidoNode Runtime: ${archive.statusText}`)
+    }
+    const compressedBuffer = await archive.arrayBuffer()
+    if (process.platform === 'win32') {
+      fs.writeFileSync(runtimeFileName, gunzipSync(Buffer.from(compressedBuffer)))
+    } else {
+      const archivePath = path.join(runtimeFolder, runtimeArchiveFilename)
+      logger.debug(`GradidoNode Runtime Archive: ${archivePath}`)
+      fs.writeFileSync(archivePath, Buffer.from(compressedBuffer))
+      execSync(`tar -xzf ${archivePath}`, { cwd: runtimeFolder })
+    }
+  }
+}
+
+function createGradidoNodeRuntimeArchiveFilename(): string {
+  const version = CONFIG.DLT_GRADIDO_NODE_SERVER_VERSION
+  const platform: string = process.platform
+  const fileEnding = platform === 'win32' ? 'zip' : 'tar.gz'
+  return `gradido_node-v${version}-${platform}-${process.arch}.${fileEnding}`
+}

dlt-connector/src/client/GradidoNode/GradidoNodeClient.ts

@@ -40,7 +40,7 @@ export class GradidoNodeClient {
 
   private constructor() {
     this.logger = getLogger(`${LOG4JS_BASE_CATEGORY}.client.GradidoNodeClient`)
-    this.urlValue = `http://localhost:${CONFIG.DLT_NODE_SERVER_PORT}`
+    this.urlValue = `http://localhost:${CONFIG.DLT_NODE_SERVER_PORT}/api`
     this.logger.addContext('url', this.urlValue)
     this.client = new JsonRpcClient({
       url: this.urlValue,

dlt-connector/src/client/GradidoNode/GradidoNodeProcess.ts

@@ -52,12 +52,13 @@ export class GradidoNodeProcess {
       env: {
         CLIENTS_HIERO_NETWORKTYPE: CONFIG.HIERO_HEDERA_NETWORK,
         SERVER_JSON_RPC_PORT: CONFIG.DLT_NODE_SERVER_PORT.toString(),
+        HOME: CONFIG.DLT_GRADIDO_NODE_SERVER_HOME_FOLDER,
       },
       onExit(proc, exitCode, signalCode, error) {
         logger.warn(`GradidoNodeProcess exited with code ${exitCode} and signalCode ${signalCode}`)
         if (error) {
           logger.error(`GradidoNodeProcess exit error: ${error}`)
-          if (logger.isDebugEnabled() && proc.stderr) {
+          /*if (logger.isDebugEnabled() && proc.stderr) {
             // print error messages from GradidoNode in our own log if debug is enabled
             proc.stderr
               .getReader()
@@ -65,9 +66,9 @@ export class GradidoNodeProcess {
               .then((chunk) => {
                 logger.debug(chunk.value?.toString())
               })
-          }
+          }*/
         }
-        logger.debug(`ressource usage: ${proc?.resourceUsage()}`)
+        logger.debug(`ressource usage: ${JSON.stringify(proc?.resourceUsage(), null, 2)}`)
         const gradidoNodeProcess = GradidoNodeProcess.getInstance()
         gradidoNodeProcess.proc = null
         if (
@@ -80,8 +81,10 @@ export class GradidoNodeProcess {
           gradidoNodeProcess.start()
         }
       },
-      stdout: 'ignore',
-      stderr: logger.isDebugEnabled() ? 'pipe' : 'ignore',
+      /*stdout: 'ignore',
+      stderr: logger.isDebugEnabled() ? 'pipe' : 'ignore',*/
+      stdout: 'inherit',
+      stderr: 'inherit',
     })
   }
 

dlt-connector/src/client/GradidoNode/communities.ts

@@ -0,0 +1,84 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import { Mutex } from 'async-mutex'
+import { getLogger } from 'log4js'
+import { CONFIG } from '../../config'
+import { GRADIDO_NODE_HOME_FOLDER_NAME, LOG4JS_BASE_CATEGORY } from '../../config/const'
+import { HieroId } from '../../schemas/typeGuard.schema'
+import { checkFileExist, checkPathExist } from '../../utils/filesystem'
+import { BackendClient } from '../backend/BackendClient'
+import { GradidoNodeProcess } from './GradidoNodeProcess'
+
+const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.client.GradidoNode.communities`)
+const ensureCommunitiesAvailableMutex: Mutex = new Mutex()
+
+// prototype, later add api call to gradido dlt node server for adding/updating communities
+type CommunityForDltNodeServer = {
+  communityId: string
+  hieroTopicId: string
+  alias: string
+  folder: string
+}
+
+export async function ensureCommunitiesAvailable(communityTopicIds: HieroId[]): Promise<void> {
+  const release = await ensureCommunitiesAvailableMutex.acquire()
+  try {
+    const homeFolder = path.join(
+      CONFIG.DLT_GRADIDO_NODE_SERVER_HOME_FOLDER,
+      GRADIDO_NODE_HOME_FOLDER_NAME,
+    )
+    if (!checkCommunityAvailable(communityTopicIds, homeFolder)) {
+      await exportCommunities(homeFolder, BackendClient.getInstance())
+      return GradidoNodeProcess.getInstance().restart()
+    }
+  } finally {
+    release()
+  }
+}
+
+export async function exportCommunities(homeFolder: string, client: BackendClient): Promise<void> {
+  const communities = await client.getReachableCommunities()
+  const communitiesPath = path.join(homeFolder, 'communities.json')
+  checkPathExist(path.dirname(communitiesPath), true)
+  // make sure communityName is unique
+  const communityName = new Set<string>()
+  const communitiesForDltNodeServer: CommunityForDltNodeServer[] = []
+  for (const com of communities) {
+    if (!com.uuid || !com.hieroTopicId) {
+      continue
+    }
+    // use name as alias if not empty and unique, otherwise use uuid
+    let alias = com.name
+    if (!alias || communityName.has(alias)) {
+      alias = com.uuid
+    }
+    communityName.add(alias)
+    communitiesForDltNodeServer.push({
+      communityId: com.uuid,
+      hieroTopicId: com.hieroTopicId,
+      alias,
+      // use only alpha-numeric chars for folder name
+      folder: alias.replace(/[^a-zA-Z0-9]/g, '_'),
+    })
+  }
+  fs.writeFileSync(communitiesPath, JSON.stringify(communitiesForDltNodeServer, null, 2))
+  logger.info(`exported ${communitiesForDltNodeServer.length} communities to ${communitiesPath}`)
+}
+
+export function checkCommunityAvailable(communityTopicIds: HieroId[], homeFolder: string): boolean {
+  const communitiesPath = path.join(homeFolder, 'communities.json')
+  if (!checkFileExist(communitiesPath)) {
+    return false
+  }
+  const communities = JSON.parse(fs.readFileSync(communitiesPath, 'utf-8'))
+  let foundCount = 0
+  for (const community of communities) {
+    if (communityTopicIds.includes(community.hieroTopicId)) {
+      foundCount++
+      if (foundCount >= communityTopicIds.length) {
+        return true
+      }
+    }
+  }
+  return false
+}

dlt-connector/src/client/backend/BackendClient.ts

@@ -5,7 +5,11 @@ import * as v from 'valibot'
 import { CONFIG } from '../../config'
 import { LOG4JS_BASE_CATEGORY } from '../../config/const'
 import { HieroId, Uuidv4 } from '../../schemas/typeGuard.schema'
-import { homeCommunityGraphqlQuery, setHomeCommunityTopicId } from './graphql'
+import {
+  getReachableCommunities,
+  homeCommunityGraphqlQuery,
+  setHomeCommunityTopicId,
+} from './graphql'
 import { type Community, communitySchema } from './output.schema'
 
 // Source: https://refactoring.guru/design-patterns/singleton/typescript/example
@@ -42,7 +46,7 @@ export class BackendClient {
   }
 
   public get url(): string {
-    return this.url
+    return this.urlValue
   }
 
   /**
@@ -84,6 +88,19 @@ export class BackendClient {
     return v.parse(communitySchema, data.updateHomeCommunity)
   }
 
+  public async getReachableCommunities(): Promise<Community[]> {
+    this.logger.info('get reachable communities on backend')
+    const { data, errors } = await this.client.rawRequest<{ reachableCommunities: Community[] }>(
+      getReachableCommunities,
+      {},
+      await this.getRequestHeader(),
+    )
+    if (errors) {
+      throw errors[0]
+    }
+    return v.parse(v.array(communitySchema), data.reachableCommunities)
+  }
+
   private async getRequestHeader(): Promise<{
     authorization: string
   }> {

dlt-connector/src/client/backend/graphql.ts

@@ -4,27 +4,40 @@ import { gql } from 'graphql-request'
  * Schema Definitions for graphql requests
  */
 
+const communityFragment = gql`
+  fragment Community_common on Community {
+    uuid
+    name
+    hieroTopicId
+    foreign
+    creationDate
+  }
+`
+
 // graphql query for getting home community in tune with community schema
 export const homeCommunityGraphqlQuery = gql`
   query {
     homeCommunity {
-      uuid
-      name
-      hieroTopicId
-      foreign
-      creationDate
+      ...Community_common
     }
   }
+  ${communityFragment}
 `
 
 export const setHomeCommunityTopicId = gql`
   mutation ($uuid: String!, $hieroTopicId: String){
     updateHomeCommunity(uuid: $uuid, hieroTopicId: $hieroTopicId) {
-      uuid
-      name
-      hieroTopicId
-      foreign
-      creationDate
+      ...Community_common
     }
   }
+  ${communityFragment}
+`
+
+export const getReachableCommunities = gql`
+  query {
+    reachableCommunities {
+    ...Community_common
+    }
+  }
+  ${communityFragment}
 `

dlt-connector/src/client/hiero/HieroClient.ts

@@ -1,8 +1,11 @@
 import {
   AccountBalance,
   AccountBalanceQuery,
+  AddressBookQuery,
   Client,
+  FileId,
   LocalProvider,
+  NodeAddressBook,
   PrivateKey,
   TopicCreateTransaction,
   TopicId,
@@ -166,6 +169,16 @@ export class HieroClient {
     return v.parse(hieroIdSchema, createReceipt.topicId?.toString())
   }
 
+  public async downloadAddressBook(): Promise<NodeAddressBook> {
+    const query = new AddressBookQuery().setFileId(FileId.ADDRESS_BOOK)
+    try {
+      return await query.execute(this.client)
+    } catch (e) {
+      this.logger.error(e)
+      throw e
+    }
+  }
+
   public async updateTopic(topicId: HieroId): Promise<void> {
     this.logger.addContext('topicId', topicId.toString())
     let transaction = new TopicUpdateTransaction()

dlt-connector/src/config/const.ts

@@ -6,7 +6,16 @@ export const MIN_TOPIC_EXPIRE_MILLISECONDS_FOR_UPDATE = 1000 * 60 * 60 * 24 * 7
 // 10 minutes
 export const MIN_TOPIC_EXPIRE_MILLISECONDS_FOR_SEND_MESSAGE = 1000 * 60 * 10
 
-export const GRADIDO_NODE_RUNTIME_PATH = path.join(__dirname, 'gradido_node', 'bin', 'GradidoNode')
+export const GRADIDO_NODE_RUNTIME_PATH = path.join(
+  __dirname,
+  '..',
+  '..',
+  'gradido_node',
+  'bin',
+  'GradidoNode',
+)
 // if last start was less than this time, do not restart
 export const GRADIDO_NODE_MIN_RUNTIME_BEFORE_RESTART_MILLISECONDS = 1000 * 30
 export const GRADIDO_NODE_KILL_TIMEOUT_MILLISECONDS = 1000
+// currently hard coded in gradido node, update in future
+export const GRADIDO_NODE_HOME_FOLDER_NAME = '.gradido'

dlt-connector/src/config/schema.ts

@@ -1,3 +1,4 @@
+import path from 'node:path'
 import { MemoryBlock } from 'gradido-blockchain-js'
 import * as v from 'valibot'
 
@@ -78,7 +79,14 @@ export const configSchema = v.object({
     ),
     '8340',
   ),
-  DLT_NODE_SERVER_VERSION: v.optional(v.string('The version of the DLT node server'), '0.9.0'),
+  DLT_GRADIDO_NODE_SERVER_VERSION: v.optional(
+    v.string('The version of the DLT node server'),
+    '0.9.0',
+  ),
+  DLT_GRADIDO_NODE_SERVER_HOME_FOLDER: v.optional(
+    v.string('The home folder for the gradido dlt node server'),
+    path.join(__dirname, '..', '..', 'gradido_node'),
+  ),
   PORT: v.optional(
     v.pipe(
       v.string('A valid port on which the backend server is running'),

dlt-connector/src/interactions/sendToHiero/SendToHiero.context.ts

@@ -7,6 +7,7 @@ import {
 } from 'gradido-blockchain-js'
 import { getLogger } from 'log4js'
 import * as v from 'valibot'
+import { ensureCommunitiesAvailable } from '../../client/GradidoNode/communities'
 import { HieroClient } from '../../client/hiero/HieroClient'
 import { LOG4JS_BASE_CATEGORY } from '../../config/const'
 import { InputTransactionType } from '../../data/InputTransactionType.enum'
@@ -40,7 +41,7 @@ const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.interactions.sendToHiero.SendT
 export async function SendToHieroContext(
   input: TransactionInput | CommunityInput,
 ): Promise<HieroTransactionIdString> {
-  const role = chooseCorrectRole(input)
+  const role = await chooseCorrectRole(input)
   const builder = await role.getGradidoTransactionBuilder()
   if (builder.isCrossCommunityTransaction()) {
     // build cross group transaction
@@ -107,9 +108,13 @@ async function sendViaHiero(
 }
 
 // choose correct role based on transaction type and input type
-function chooseCorrectRole(input: TransactionInput | CommunityInput): AbstractTransactionRole {
+async function chooseCorrectRole(
+  input: TransactionInput | CommunityInput,
+): Promise<AbstractTransactionRole> {
   const communityParsingResult = v.safeParse(communitySchema, input)
   if (communityParsingResult.success) {
+    // make sure gradido node knows community
+    await ensureCommunitiesAvailable([communityParsingResult.output.hieroTopicId])
     return new CommunityRootTransactionRole(communityParsingResult.output)
   }
 
@@ -121,6 +126,12 @@ function chooseCorrectRole(input: TransactionInput | CommunityInput): AbstractTr
     })
     throw new Error('invalid input')
   }
+  // make sure gradido node knows communities
+  const communityTopicIds = [
+    transactionParsingResult.output.user.communityTopicId,
+    transactionParsingResult.output.linkedUser?.communityTopicId,
+  ].filter((id): id is HieroId => id !== undefined)
+  await ensureCommunitiesAvailable(communityTopicIds)
 
   const transaction = transactionParsingResult.output
   switch (transaction.type) {

dlt-connector/src/server/index.test.ts

@@ -25,6 +25,12 @@ mock.module('../KeyPairCacheManager', () => {
   }
 })
 
+mock.module('../client/GradidoNode/communities', () => ({
+  ensureCommunitiesAvailable: () => {
+    return Promise.resolve()
+  },
+}))
+
 mock.module('../client/hiero/HieroClient', () => ({
   HieroClient: {
     getInstance: () => ({

dlt-connector/src/server/index.ts

@@ -3,6 +3,7 @@ import { Elysia, status, t } from 'elysia'
 import { AddressType_NONE } from 'gradido-blockchain-js'
 import { getLogger } from 'log4js'
 import * as v from 'valibot'
+import { ensureCommunitiesAvailable } from '../client/GradidoNode/communities'
 import { GradidoNodeClient } from '../client/GradidoNode/GradidoNodeClient'
 import { LOG4JS_BASE_CATEGORY } from '../config/const'
 import { KeyPairIdentifierLogic } from '../data/KeyPairIdentifier.logic'
@@ -125,6 +126,8 @@ async function isAccountExist(identifierAccount: IdentifierAccountInput): Promis
   // check and prepare input
   const startTime = Date.now()
   const identifierAccountParsed = v.parse(identifierAccountSchema, identifierAccount)
+  // make sure gradido node knows community
+  await ensureCommunitiesAvailable([identifierAccountParsed.communityTopicId])
   const accountKeyPair = await ResolveKeyPair(new KeyPairIdentifierLogic(identifierAccountParsed))
   const publicKey = accountKeyPair.getPublicKey()
   if (!publicKey) {

dlt-connector/src/utils/filesystem.ts

@@ -0,0 +1,30 @@
+import fs from 'node:fs'
+import { getLogger } from 'log4js'
+import { LOG4JS_BASE_CATEGORY } from '../config/const'
+
+const logger = getLogger(`${LOG4JS_BASE_CATEGORY}.utils.filesystem`)
+
+export function checkFileExist(filePath: string): boolean {
+  try {
+    fs.accessSync(filePath, fs.constants.R_OK | fs.constants.W_OK)
+    return true
+  } catch (err) {
+    logger.debug(`file ${filePath} does not exist: ${err}`)
+    return false
+  }
+}
+
+export function checkPathExist(path: string, createIfMissing: boolean = false): boolean {
+  const exists = checkFileExist(path)
+  if (exists) {
+    return true
+  }
+  if (createIfMissing) {
+    logger.info(`create folder ${path}`)
+    fs.mkdirSync(path, { recursive: true })
+    if (!checkPathExist(path)) {
+      throw new Error(`Failed to create path ${path}`)
+    }
+  }
+  return false
+}