Merge remote-tracking branch 'origin/master' into

2420-feature-federation-implement-exchange-of-api-versions-persist-in-table

CHANGELOG.md

@@ -4,8 +4,21 @@ All notable changes to this project will be documented in this file. Dates are d
 
 Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 
+#### [1.15.0](https://github.com/gradido/gradido/compare/1.14.1...1.15.0)
+
+- fix(database): wrong balance and decay values [`#2423`](https://github.com/gradido/gradido/pull/2423)
+- fix(backend): wrong balance after transaction receive [`#2422`](https://github.com/gradido/gradido/pull/2422)
+- feat(other): feature gradido roadmap [`#2301`](https://github.com/gradido/gradido/pull/2301)
+- refactor(backend): new password encryption implementation [`#2353`](https://github.com/gradido/gradido/pull/2353)
+- refactor(admin): statistics in a table and on separate page in admin area [`#2399`](https://github.com/gradido/gradido/pull/2399)
+- feat(backend): 🍰 Email Templates [`#2163`](https://github.com/gradido/gradido/pull/2163)
+- fix(backend): timezone problems [`#2393`](https://github.com/gradido/gradido/pull/2393)
+
 #### [1.14.1](https://github.com/gradido/gradido/compare/1.14.0...1.14.1)
 
+> 14 November 2022
+
+- chore(release): version 1.14.1 - hotfix [`#2391`](https://github.com/gradido/gradido/pull/2391)
 - fix(frontend): load contributionMessages is fixed [`#2390`](https://github.com/gradido/gradido/pull/2390)
 
 #### [1.14.0](https://github.com/gradido/gradido/compare/1.13.3...1.14.0)

admin/package.json

@@ -3,7 +3,7 @@
   "description": "Administraion Interface for Gradido",
   "main": "index.js",
   "author": "Moriz Wahl",
-  "version": "1.14.1",
+  "version": "1.15.0",
   "license": "Apache-2.0",
   "private": false,
   "scripts": {

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gradido-backend",
-  "version": "1.14.1",
+  "version": "1.15.0",
   "description": "Gradido unified backend providing an API-Service for Gradido Transactions",
   "main": "src/index.ts",
   "repository": "https://github.com/gradido/gradido/backend",
@@ -19,13 +19,7 @@
   },
   "dependencies": {
     "@hyperswarm/dht": "^6.2.0",
-    "@types/email-templates": "^10.0.1",
-    "@types/i18n": "^0.13.4",
-    "@types/jest": "^27.0.2",
-    "@types/lodash.clonedeep": "^4.5.6",
-    "@types/uuid": "^8.3.4",
     "apollo-server-express": "^2.25.2",
-    "apollo-server-testing": "^2.25.2",
     "axios": "^0.21.1",
     "class-validator": "^0.13.1",
     "cors": "^2.8.5",
@@ -46,18 +40,23 @@
     "random-bigint": "^0.0.1",
     "reflect-metadata": "^0.1.13",
     "sodium-native": "^3.3.0",
-    "ts-jest": "^27.0.5",
     "type-graphql": "^1.1.1",
     "uuid": "^8.3.2"
   },
   "devDependencies": {
+    "@types/email-templates": "^10.0.1",
     "@types/express": "^4.17.12",
     "@types/faker": "^5.5.9",
+    "@types/i18n": "^0.13.4",
+    "@types/jest": "^27.0.2",
     "@types/jsonwebtoken": "^8.5.2",
+    "@types/lodash.clonedeep": "^4.5.6",
     "@types/node": "^16.10.3",
     "@types/nodemailer": "^6.4.4",
+    "@types/uuid": "^8.3.4",
     "@typescript-eslint/eslint-plugin": "^4.28.0",
     "@typescript-eslint/parser": "^4.28.0",
+    "apollo-server-testing": "^2.25.2",
     "eslint": "^7.29.0",
     "eslint-config-prettier": "^8.3.0",
     "eslint-config-standard": "^16.0.3",
@@ -66,8 +65,10 @@
     "eslint-plugin-prettier": "^3.4.0",
     "eslint-plugin-promise": "^5.1.0",
     "faker": "^5.5.3",
+    "jest": "^27.2.4",
     "nodemon": "^2.0.7",
     "prettier": "^2.3.1",
+    "ts-jest": "^27.0.5",
     "ts-node": "^10.0.0",
     "tsconfig-paths": "^3.14.0",
     "typescript": "^4.3.4"

backend/src/config/index.ts

@@ -10,7 +10,7 @@ Decimal.set({
 })
 
 const constants = {
-  DB_VERSION: '0054-add_communities_table',
+  DB_VERSION: '0055-add_communities_table',
   DECAY_START_TIME: new Date('2021-05-13 17:46:31-0000'), // GMT+0
   LOG4JS_CONFIG: 'log4js-config.json',
   // default log level on production should be info

backend/src/graphql/resolver/TransactionLinkResolver.ts

@@ -74,7 +74,10 @@ export class TransactionLinkResolver {
     const holdAvailableAmount = amount.minus(calculateDecay(amount, createdDate, validUntil).decay)
 
     // validate amount
-    await calculateBalance(user.id, holdAvailableAmount, createdDate)
+    const sendBalance = await calculateBalance(user.id, holdAvailableAmount.mul(-1), createdDate)
+    if (!sendBalance) {
+      throw new Error("user hasn't enough GDD or amount is < 0")
+    }
 
     const transactionLink = dbTransactionLink.create()
     transactionLink.userId = user.id

backend/src/graphql/resolver/TransactionResolver.test.ts

@@ -16,7 +16,7 @@ import { stephenHawking } from '@/seeds/users/stephen-hawking'
 import { EventProtocol } from '@entity/EventProtocol'
 import { Transaction } from '@entity/Transaction'
 import { User } from '@entity/User'
-import { cleanDB, resetToken, testEnvironment } from '@test/helpers'
+import { cleanDB, testEnvironment } from '@test/helpers'
 import { logger } from '@test/testSetup'
 import { GraphQLError } from 'graphql'
 import { findUserByEmail } from './UserResolver'
@@ -253,50 +253,21 @@ describe('send coins', () => {
           }),
         ).toEqual(
           expect.objectContaining({
-            errors: [new GraphQLError(`User has not received any GDD yet`)],
+            errors: [new GraphQLError(`user hasn't enough GDD or amount is < 0`)],
           }),
         )
       })
 
       it('logs the error thrown', () => {
         expect(logger.error).toBeCalledWith(
-          `No prior transaction found for user with id: ${user[1].id}`,
+          `user hasn't enough GDD or amount is < 0 : balance=null`,
         )
       })
     })
-
-    describe('sending negative amount', () => {
-      it('throws an error', async () => {
-        jest.clearAllMocks()
-        expect(
-          await mutate({
-            mutation: sendCoins,
-            variables: {
-              email: 'peter@lustig.de',
-              amount: -50,
-              memo: 'testing negative',
-            },
-          }),
-        ).toEqual(
-          expect.objectContaining({
-            errors: [new GraphQLError('Transaction amount must be greater than 0')],
-          }),
-        )
-      })
-
-      it('logs the error thrown', () => {
-        expect(logger.error).toBeCalledWith('Transaction amount must be greater than 0: -50')
-      })
-    })
   })
 
   describe('user has some GDD', () => {
     beforeAll(async () => {
-      resetToken()
-
-      // login as bob again
-      await query({ mutation: login, variables: bobData })
-
       // create contribution as user bob
       const contribution = await mutate({
         mutation: createContribution,
@@ -316,6 +287,37 @@ describe('send coins', () => {
       await query({ mutation: login, variables: bobData })
     })
 
+    afterAll(async () => {
+      await cleanDB()
+    })
+
+    /*
+    describe('trying to send negative amount', () => {
+      it('throws an error', async () => {
+        expect(
+          await mutate({
+            mutation: sendCoins,
+            variables: {
+              email: 'peter@lustig.de',
+              amount: -50,
+              memo: 'testing negative',
+            },
+          }),
+        ).toEqual(
+          expect.objectContaining({
+            errors: [new GraphQLError(`user hasn't enough GDD or amount is < 0`)],
+          }),
+        )
+      })
+
+      it('logs the error thrown', () => {
+        expect(logger.error).toBeCalledWith(
+          `user hasn't enough GDD or amount is < 0 : balance=null`,
+        )
+      })
+    })
+    */
+
     describe('good transaction', () => {
       it('sends the coins', async () => {
         expect(

backend/src/graphql/resolver/TransactionResolver.ts

@@ -39,7 +39,6 @@ import { findUserByEmail } from './UserResolver'
 import { sendTransactionLinkRedeemedEmail } from '@/mailer/sendTransactionLinkRedeemed'
 import { Event, EventTransactionReceive, EventTransactionSend } from '@/event/Event'
 import { eventProtocol } from '@/event/EventProtocolEmitter'
-import { Decay } from '../model/Decay'
 
 export const executeTransaction = async (
   amount: Decimal,
@@ -69,8 +68,17 @@ export const executeTransaction = async (
 
   // validate amount
   const receivedCallDate = new Date()
-
-  const sendBalance = await calculateBalance(sender.id, amount, receivedCallDate, transactionLink)
+  const sendBalance = await calculateBalance(
+    sender.id,
+    amount.mul(-1),
+    receivedCallDate,
+    transactionLink,
+  )
+  logger.debug(`calculated Balance=${sendBalance}`)
+  if (!sendBalance) {
+    logger.error(`user hasn't enough GDD or amount is < 0 : balance=${sendBalance}`)
+    throw new Error("user hasn't enough GDD or amount is < 0")
+  }
 
   const queryRunner = getConnection().createQueryRunner()
   await queryRunner.connect()
@@ -100,24 +108,7 @@ export const executeTransaction = async (
     transactionReceive.userId = recipient.id
     transactionReceive.linkedUserId = sender.id
     transactionReceive.amount = amount
-
-    // state received balance
-    let receiveBalance: {
-      balance: Decimal
-      decay: Decay
-      lastTransactionId: number
-    } | null
-
-    // try received balance
-    try {
-      receiveBalance = await calculateBalance(recipient.id, amount, receivedCallDate)
-    } catch (e) {
-      logger.info(
-        `User with no transactions sent: ${recipient.id}, has received a transaction of ${amount} GDD from user: ${sender.id}`,
-      )
-      receiveBalance = null
-    }
-
+    const receiveBalance = await calculateBalance(recipient.id, amount, receivedCallDate)
     transactionReceive.balance = receiveBalance ? receiveBalance.balance : amount
     transactionReceive.balanceDate = receivedCallDate
     transactionReceive.decay = receiveBalance ? receiveBalance.decay.decay : new Decimal(0)

backend/src/util/utilities.ts

@@ -1,17 +1,5 @@
-import Decimal from 'decimal.js-light'
-
 export const objectValuesToArray = (obj: { [x: string]: string }): Array<string> => {
   return Object.keys(obj).map(function (key) {
     return obj[key]
   })
 }
-
-// to improve code readability, as String is needed, it is handled inside this utility function
-export const decimalAddition = (a: Decimal, b: Decimal): Decimal => {
-  return a.add(b.toString())
-}
-
-// to improve code readability, as String is needed, it is handled inside this utility function
-export const decimalSubtraction = (a: Decimal, b: Decimal): Decimal => {
-  return a.minus(b.toString())
-}

backend/src/util/validate.ts

@@ -5,8 +5,6 @@ import { Decay } from '@model/Decay'
 import { getCustomRepository } from '@dbTools/typeorm'
 import { TransactionLinkRepository } from '@repository/TransactionLink'
 import { TransactionLink as dbTransactionLink } from '@entity/TransactionLink'
-import { decimalSubtraction, decimalAddition } from './utilities'
-import { backendLogger as logger } from '@/server/logger'
 
 function isStringBoolean(value: string): boolean {
   const lowerValue = value.toLowerCase()
@@ -25,26 +23,13 @@ async function calculateBalance(
   amount: Decimal,
   time: Date,
   transactionLink?: dbTransactionLink | null,
-): Promise<{ balance: Decimal; decay: Decay; lastTransactionId: number }> {
-  // negative or empty amount should not be allowed
-  if (amount.lessThanOrEqualTo(0)) {
-    logger.error(`Transaction amount must be greater than 0: ${amount}`)
-    throw new Error('Transaction amount must be greater than 0')
-  }
-
-  // check if user has prior transactions
+): Promise<{ balance: Decimal; decay: Decay; lastTransactionId: number } | null> {
   const lastTransaction = await Transaction.findOne({ userId }, { order: { balanceDate: 'DESC' } })
-
-  if (!lastTransaction) {
-    logger.error(`No prior transaction found for user with id: ${userId}`)
-    throw new Error('User has not received any GDD yet')
-  }
+  if (!lastTransaction) return null
 
   const decay = calculateDecay(lastTransaction.balance, lastTransaction.balanceDate, time)
 
-  // new balance is the old balance minus the amount used
-  const balance = decimalSubtraction(decay.balance, amount)
-
+  const balance = decay.balance.add(amount.toString())
   const transactionLinkRepository = getCustomRepository(TransactionLinkRepository)
   const { sumHoldAvailableAmount } = await transactionLinkRepository.summary(userId, time)
 
@@ -52,16 +37,11 @@ async function calculateBalance(
   // else we cannot redeem links which are more or equal to half of what an account actually owns
   const releasedLinkAmount = transactionLink ? transactionLink.holdAvailableAmount : new Decimal(0)
 
-  const availableBalance = decimalSubtraction(balance, sumHoldAvailableAmount)
-
-  if (decimalAddition(availableBalance, releasedLinkAmount).lessThan(0)) {
-    logger.error(
-      `Not enough funds for a transaction of ${amount} GDD, user with id: ${userId} has only ${balance} GDD available`,
-    )
-    throw new Error('Not enough funds for transaction')
+  if (
+    balance.minus(sumHoldAvailableAmount.toString()).plus(releasedLinkAmount.toString()).lessThan(0)
+  ) {
+    return null
   }
-
-  logger.debug(`calculated Balance=${balance}`)
   return { balance, lastTransactionId: lastTransaction.id, decay }
 }
 

database/Dockerfile

@@ -100,6 +100,8 @@ COPY --from=build ${DOCKER_WORKDIR}/node_modules ./node_modules
 COPY --from=build ${DOCKER_WORKDIR}/package.json ./package.json
 # Copy Mnemonic files
 COPY --from=build ${DOCKER_WORKDIR}/src/config/*.txt ./src/config/
+# Copy log folder
+COPY --from=build ${DOCKER_WORKDIR}/log ./log
 # Copy run scripts run/
 # COPY --from=build ${DOCKER_WORKDIR}/run ./run
 

database/log/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

database/migrations/0054-recalculate_balance_and_decay.ts

@@ -0,0 +1,160 @@
+/* MIGRATION TO FIX WRONG BALANCE
+ *
+ * Due to a bug in the code
+ * the amount of a receive balance is substracted
+ * from the previous balance instead of added.
+ *
+ * Therefore all balance and decay fields must
+ * be recalculated
+ *
+ * WARNING: This Migration must be run in TZ=UTC
+ */
+
+/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import fs from 'fs'
+import Decimal from 'decimal.js-light'
+
+// Set precision value
+Decimal.set({
+  precision: 25,
+  rounding: Decimal.ROUND_HALF_UP,
+})
+
+const DECAY_START_TIME = new Date('2021-05-13 17:46:31') // GMT+0
+
+interface Decay {
+  balance: Decimal
+  decay: Decimal | null
+  start: Date | null
+  end: Date | null
+  duration: number | null
+}
+
+export enum TransactionTypeId {
+  CREATION = 1,
+  SEND = 2,
+  RECEIVE = 3,
+}
+
+function decayFormula(value: Decimal, seconds: number): Decimal {
+  return value.mul(new Decimal('0.99999997803504048973201202316767079413460520837376').pow(seconds))
+}
+
+function calculateDecay(
+  amount: Decimal,
+  from: Date,
+  to: Date,
+  startBlock: Date = DECAY_START_TIME,
+): Decay {
+  const fromMs = from.getTime()
+  const toMs = to.getTime()
+  const startBlockMs = startBlock.getTime()
+
+  if (toMs < fromMs) {
+    throw new Error('to < from, reverse decay calculation is invalid')
+  }
+
+  // Initialize with no decay
+  const decay: Decay = {
+    balance: amount,
+    decay: null,
+    start: null,
+    end: null,
+    duration: null,
+  }
+
+  // decay started after end date; no decay
+  if (startBlockMs > toMs) {
+    return decay
+  }
+  // decay started before start date; decay for full duration
+  if (startBlockMs < fromMs) {
+    decay.start = from
+    decay.duration = (toMs - fromMs) / 1000
+  }
+  // decay started between start and end date; decay from decay start till end date
+  else {
+    decay.start = startBlock
+    decay.duration = (toMs - startBlockMs) / 1000
+  }
+
+  decay.end = to
+  decay.balance = decayFormula(amount, decay.duration)
+  decay.decay = decay.balance.minus(amount)
+  return decay
+}
+
+export async function upgrade(queryFn: (query: string, values?: any[]) => Promise<Array<any>>) {
+  // Write log file
+  const logFile = 'log/0054-recalculate_balance_and_decay.log.csv'
+  await fs.writeFile(
+    logFile,
+    `email;first_name;last_name;affected_transactions;new_balance;new_decay;old_balance;old_decay;delta;\n`,
+    (err) => {
+      if (err) throw err
+    },
+  )
+
+  // Find all users & loop over them
+  const users = await queryFn('SELECT user_id FROM transactions GROUP BY user_id;')
+  for (let u = 0; u < users.length; u++) {
+    const userId = users[u].user_id
+    // find all transactions for a user
+    const transactions = await queryFn(
+      `SELECT *, CONVERT(balance, CHAR) as dec_balance, CONVERT(decay, CHAR) as dec_decay FROM transactions WHERE user_id = ${userId} ORDER BY balance_date ASC;`,
+    )
+
+    let previous = null
+    let affectedTransactions = 0
+    let balance = new Decimal(0)
+    for (let t = 0; t < transactions.length; t++) {
+      const transaction = transactions[t]
+      const decayStartDate = previous ? previous.balance_date : transaction.balance_date
+      const amount = new Decimal(transaction.amount)
+      const decay = calculateDecay(balance, decayStartDate, transaction.balance_date)
+      balance = decay.balance.add(amount)
+
+      const userContact = await queryFn(
+        `SELECT email, first_name, last_name FROM users LEFT JOIN user_contacts ON users.email_id = user_contacts.id WHERE users.id = ${userId}`,
+      )
+      const userEmail = userContact.length === 1 ? userContact[0].email : userId
+      const userFirstName = userContact.length === 1 ? userContact[0].first_name : ''
+      const userLastName = userContact.length === 1 ? userContact[0].last_name : ''
+
+      // Update if needed
+      if (!balance.eq(transaction.dec_balance)) {
+        await queryFn(`
+          UPDATE transactions SET
+            balance = ${balance},
+            decay = ${decay.decay ? decay.decay : 0}
+          WHERE id = ${transaction.id};
+        `)
+        affectedTransactions++
+
+        // Log on last entry
+        if (t === transactions.length - 1) {
+          fs.appendFile(
+            logFile,
+            `${userEmail};${userFirstName};${userLastName};${affectedTransactions};${balance};${
+              decay.decay ? decay.decay : 0
+            };${transaction.dec_balance};${transaction.dec_decay};${balance.sub(
+              transaction.dec_balance,
+            )};\n`,
+            (err) => {
+              if (err) throw err
+            },
+          )
+        }
+      }
+
+      // previous
+      previous = transaction
+    }
+  }
+}
+
+/* eslint-disable @typescript-eslint/no-empty-function */
+/* eslint-disable-next-line @typescript-eslint/no-unused-vars */
+export async function downgrade(queryFn: (query: string, values?: any[]) => Promise<Array<any>>) {}

database/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gradido-database",
-  "version": "1.14.1",
+  "version": "1.15.0",
   "description": "Gradido Database Tool to execute database migrations",
   "main": "src/index.ts",
   "repository": "https://github.com/gradido/gradido/database",

deployment/bare_metal/.env.dist

@@ -62,7 +62,7 @@ EVENT_PROTOCOL_DISABLED=false
 # Federation
 # if you set the value of FEDERATION_DHT_TOPIC, the DHT hyperswarm will start to announce and listen
 # on an hash created from this topic
-FEDERATION_DHT_TOPIC=GRADIDO_HUB
+# FEDERATION_DHT_TOPIC=GRADIDO_HUB
 # FEDERATION_DHT_SEED=64ebcb0e3ad547848fef4197c6e2332f 
 
 # database

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bootstrap-vue-gradido-wallet",
-  "version": "1.14.1",
+  "version": "1.15.0",
   "private": true,
   "scripts": {
     "start": "node run/server.js",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "gradido",
-  "version": "1.14.1",
+  "version": "1.15.0",
   "description": "Gradido",
   "main": "index.js",
   "repository": "git@github.com:gradido/gradido.git",