From 885099aa75ffd2eb3f57f292f0a85838b19f88c4 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Sat, 12 Nov 2022 19:57:53 +0100 Subject: [PATCH 01/10] moved all jest related packages in the `devDependencies` section, since thats where they belong --- backend/package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/package.json b/backend/package.json index 1db683b2a..ac8022268 100644 --- a/backend/package.json +++ b/backend/package.json @@ -19,11 +19,9 @@ }, "dependencies": { "@hyperswarm/dht": "^6.2.0", - "@types/jest": "^27.0.2", "@types/lodash.clonedeep": "^4.5.6", "@types/uuid": "^8.3.4", "apollo-server-express": "^2.25.2", - "apollo-server-testing": "^2.25.2", "axios": "^0.21.1", "class-validator": "^0.13.1", "cors": "^2.8.5", @@ -32,7 +30,6 @@ "dotenv": "^10.0.0", "express": "^4.17.1", "graphql": "^15.5.1", - "jest": "^27.2.4", "jsonwebtoken": "^8.5.1", "lodash.clonedeep": "^4.5.0", "log4js": "^6.4.6", @@ -41,18 +38,19 @@ "random-bigint": "^0.0.1", "reflect-metadata": "^0.1.13", "sodium-native": "^3.3.0", - "ts-jest": "^27.0.5", "type-graphql": "^1.1.1", "uuid": "^8.3.2" }, "devDependencies": { "@types/express": "^4.17.12", "@types/faker": "^5.5.9", + "@types/jest": "^27.0.2", "@types/jsonwebtoken": "^8.5.2", "@types/node": "^16.10.3", "@types/nodemailer": "^6.4.4", "@typescript-eslint/eslint-plugin": "^4.28.0", "@typescript-eslint/parser": "^4.28.0", + "apollo-server-testing": "^2.25.2", "eslint": "^7.29.0", "eslint-config-prettier": "^8.3.0", "eslint-config-standard": "^16.0.3", @@ -60,10 +58,12 @@ "eslint-plugin-node": "^11.1.0", "eslint-plugin-prettier": "^3.4.0", "eslint-plugin-promise": "^5.1.0", + "jest": "^27.2.4", "faker": "^5.5.3", "nodemon": "^2.0.7", "prettier": "^2.3.1", "ts-node": "^10.0.0", + "ts-jest": "^27.0.5", "tsconfig-paths": "^3.14.0", "typescript": "^4.3.4" } From 48f66dc7d2f601e49ce5f5773727ccf852057711 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Sat, 12 Nov 2022 20:02:29 +0100 Subject: [PATCH 02/10] use proper order (alphabetical) --- backend/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/package.json b/backend/package.json index ac8022268..5c0de34f3 100644 --- a/backend/package.json +++ b/backend/package.json @@ -62,8 +62,8 @@ "faker": "^5.5.3", "nodemon": "^2.0.7", "prettier": "^2.3.1", - "ts-node": "^10.0.0", "ts-jest": "^27.0.5", + "ts-node": "^10.0.0", "tsconfig-paths": "^3.14.0", "typescript": "^4.3.4" } From 3dfffbdfea14502b0b5744364ddbd8db67c7807b Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Sat, 12 Nov 2022 20:03:45 +0100 Subject: [PATCH 03/10] more order fixes --- backend/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/package.json b/backend/package.json index 5c0de34f3..b061ce0d3 100644 --- a/backend/package.json +++ b/backend/package.json @@ -58,8 +58,8 @@ "eslint-plugin-node": "^11.1.0", "eslint-plugin-prettier": "^3.4.0", "eslint-plugin-promise": "^5.1.0", - "jest": "^27.2.4", "faker": "^5.5.3", + "jest": "^27.2.4", "nodemon": "^2.0.7", "prettier": "^2.3.1", "ts-jest": "^27.0.5", From 78451f0d9f5aab5fad98e41faae9958ca7c4da0c Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Sat, 12 Nov 2022 20:05:09 +0100 Subject: [PATCH 04/10] move type definitions into `devDependencies` --- backend/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/package.json b/backend/package.json index b061ce0d3..652650d56 100644 --- a/backend/package.json +++ b/backend/package.json @@ -19,8 +19,6 @@ }, "dependencies": { "@hyperswarm/dht": "^6.2.0", - "@types/lodash.clonedeep": "^4.5.6", - "@types/uuid": "^8.3.4", "apollo-server-express": "^2.25.2", "axios": "^0.21.1", "class-validator": "^0.13.1", @@ -46,8 +44,10 @@ "@types/faker": "^5.5.9", "@types/jest": "^27.0.2", "@types/jsonwebtoken": "^8.5.2", + "@types/lodash.clonedeep": "^4.5.6", "@types/node": "^16.10.3", "@types/nodemailer": "^6.4.4", + "@types/uuid": "^8.3.4", "@typescript-eslint/eslint-plugin": "^4.28.0", "@typescript-eslint/parser": "^4.28.0", "apollo-server-testing": "^2.25.2", From 63e22eeab80b41ca4204f6604dfa5b6d30898b29 Mon Sep 17 00:00:00 2001 From: Moriz Wahl Date: Thu, 24 Nov 2022 16:30:55 +0100 Subject: [PATCH 05/10] fix(database): wrong balance and decay values --- .../0054-recalculate-balance-and-decay.ts | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 database/migrations/0054-recalculate-balance-and-decay.ts diff --git a/database/migrations/0054-recalculate-balance-and-decay.ts b/database/migrations/0054-recalculate-balance-and-decay.ts new file mode 100644 index 000000000..fe9896db1 --- /dev/null +++ b/database/migrations/0054-recalculate-balance-and-decay.ts @@ -0,0 +1,122 @@ +/* MIGRATION TO FIX WRONG BALANCE + * + * Due to a bug in the code + * the amount of a receive balance is substracted + * from the previous balance instead of added. + * + * Therefore all balance and decay fields must + * be recalculated + * + * WARNING: This Migration must be run in TZ=UTC + */ + +/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import Decimal from 'decimal.js-light' + +// Set precision value +Decimal.set({ + precision: 25, + rounding: Decimal.ROUND_HALF_UP, +}) + +const DECAY_START_TIME = new Date('2021-05-13 17:46:31') // GMT+0 + +interface Decay { + balance: Decimal + decay: Decimal | null + start: Date | null + end: Date | null + duration: number | null +} + +export enum TransactionTypeId { + CREATION = 1, + SEND = 2, + RECEIVE = 3, +} + +function decayFormula(value: Decimal, seconds: number): Decimal { + return value.mul(new Decimal('0.99999997803504048973201202316767079413460520837376').pow(seconds)) +} + +function calculateDecay( + amount: Decimal, + from: Date, + to: Date, + startBlock: Date = DECAY_START_TIME, +): Decay { + const fromMs = from.getTime() + const toMs = to.getTime() + const startBlockMs = startBlock.getTime() + + if (toMs < fromMs) { + throw new Error('to < from, reverse decay calculation is invalid') + } + + // Initialize with no decay + const decay: Decay = { + balance: amount, + decay: null, + start: null, + end: null, + duration: null, + } + + // decay started after end date; no decay + if (startBlockMs > toMs) { + return decay + } + // decay started before start date; decay for full duration + if (startBlockMs < fromMs) { + decay.start = from + decay.duration = (toMs - fromMs) / 1000 + } + // decay started between start and end date; decay from decay start till end date + else { + decay.start = startBlock + decay.duration = (toMs - startBlockMs) / 1000 + } + + decay.end = to + decay.balance = decayFormula(amount, decay.duration) + decay.decay = decay.balance.minus(amount) + return decay +} + +export async function upgrade(queryFn: (query: string, values?: any[]) => Promise>) { + // Find all users & loop over them + const users = await queryFn('SELECT user_id FROM transactions GROUP BY user_id;') + for (let u = 0; u < users.length; u++) { + // find all transactions for a user + const transactions = await queryFn( + `SELECT * FROM transactions WHERE user_id = ${users[u].user_id} ORDER BY balance_date ASC;`, + ) + let previous = null + let balance = new Decimal(0) + for (let t = 0; t < transactions.length; t++) { + const transaction = transactions[t] + + const decayStartDate = previous ? previous.balance_date : transaction.balance_date + const amount = new Decimal(transaction.amount) + const decay = calculateDecay(balance, decayStartDate, transaction.balance_date) + balance = decay.balance.add(amount) + + // Update + await queryFn(` + UPDATE transactions SET + balance = ${balance}, + decay = ${decay.decay ? decay.decay : 0} + WHERE id = ${transaction.id}; + `) + + // previous + previous = transaction + } + } +} + +/* eslint-disable @typescript-eslint/no-empty-function */ +/* eslint-disable-next-line @typescript-eslint/no-unused-vars */ +export async function downgrade(queryFn: (query: string, values?: any[]) => Promise>) {} From f2e88469325d1a188548d576367e923dbfa47d07 Mon Sep 17 00:00:00 2001 From: Moriz Wahl Date: Thu, 24 Nov 2022 16:37:32 +0100 Subject: [PATCH 06/10] rename migration file, set new db version in backend --- backend/src/config/index.ts | 2 +- ...lance-and-decay.ts => 0054-recalculate_balance_and_decay.ts} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename database/migrations/{0054-recalculate-balance-and-decay.ts => 0054-recalculate_balance_and_decay.ts} (100%) diff --git a/backend/src/config/index.ts b/backend/src/config/index.ts index 26227b90d..a66ed9765 100644 --- a/backend/src/config/index.ts +++ b/backend/src/config/index.ts @@ -10,7 +10,7 @@ Decimal.set({ }) const constants = { - DB_VERSION: '0053-change_password_encryption', + DB_VERSION: '0054-recalculate_balance_and_decay', DECAY_START_TIME: new Date('2021-05-13 17:46:31-0000'), // GMT+0 LOG4JS_CONFIG: 'log4js-config.json', // default log level on production should be info diff --git a/database/migrations/0054-recalculate-balance-and-decay.ts b/database/migrations/0054-recalculate_balance_and_decay.ts similarity index 100% rename from database/migrations/0054-recalculate-balance-and-decay.ts rename to database/migrations/0054-recalculate_balance_and_decay.ts From 0f71a486a5e86d0be0dcbdd25c6f0c49c758e9c4 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Fri, 25 Nov 2022 14:13:22 +0100 Subject: [PATCH 07/10] log affected accounts & some fixes --- database/log/.gitignore | 2 + .../0054-recalculate_balance_and_decay.ts | 52 +++++++++++++++---- 2 files changed, 45 insertions(+), 9 deletions(-) create mode 100644 database/log/.gitignore diff --git a/database/log/.gitignore b/database/log/.gitignore new file mode 100644 index 000000000..c96a04f00 --- /dev/null +++ b/database/log/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/database/migrations/0054-recalculate_balance_and_decay.ts b/database/migrations/0054-recalculate_balance_and_decay.ts index fe9896db1..6614b4a52 100644 --- a/database/migrations/0054-recalculate_balance_and_decay.ts +++ b/database/migrations/0054-recalculate_balance_and_decay.ts @@ -13,6 +13,7 @@ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ /* eslint-disable @typescript-eslint/no-explicit-any */ +import fs from 'fs' import Decimal from 'decimal.js-light' // Set precision value @@ -86,30 +87,63 @@ function calculateDecay( } export async function upgrade(queryFn: (query: string, values?: any[]) => Promise>) { + // Write log file + const logFile = 'log/0054-recalculate_balance_and_decay.log.csv' + await fs.writeFile( + logFile, + `email;first_name;last_name;affected_transactions;new_balance;new_decay;old_balance;old_decay;delta;\n`, + (err) => { + if (err) throw err + }, + ) + // Find all users & loop over them const users = await queryFn('SELECT user_id FROM transactions GROUP BY user_id;') for (let u = 0; u < users.length; u++) { + const userId = users[u].user_id // find all transactions for a user const transactions = await queryFn( - `SELECT * FROM transactions WHERE user_id = ${users[u].user_id} ORDER BY balance_date ASC;`, + `SELECT *, CONVERT(balance, CHAR) as dec_balance, CONVERT(decay, CHAR) as dec_decay FROM transactions WHERE user_id = ${userId} ORDER BY balance_date ASC;`, ) + let previous = null + let affectedTransactions = 0 let balance = new Decimal(0) for (let t = 0; t < transactions.length; t++) { const transaction = transactions[t] - const decayStartDate = previous ? previous.balance_date : transaction.balance_date const amount = new Decimal(transaction.amount) const decay = calculateDecay(balance, decayStartDate, transaction.balance_date) balance = decay.balance.add(amount) - // Update - await queryFn(` - UPDATE transactions SET - balance = ${balance}, - decay = ${decay.decay ? decay.decay : 0} - WHERE id = ${transaction.id}; - `) + const userContact = await queryFn( + `SELECT email, first_name, last_name FROM users LEFT JOIN user_contacts ON users.email_id = user_contacts.id WHERE users.id = ${userId}`, + ) + const userEmail = userContact.length === 1 ? userContact[0].email : userId + const userFirstName = userContact.length === 1 ? userContact[0].first_name : '' + const userLastName = userContact.length === 1 ? userContact[0].last_name : '' + + // Update if needed + if (!balance.eq(transaction.dec_balance)) { + await queryFn(` + UPDATE transactions SET + balance = ${balance}, + decay = ${decay.decay ? decay.decay : 0} + WHERE id = ${transaction.id}; + `) + affectedTransactions++ + + // Log on last entry + if (t === transactions.length - 1) { + fs.appendFile( + logFile, + `${userEmail};${userFirstName};${userLastName};${affectedTransactions};${balance};${decay.decay ? decay.decay : 0};${transaction.dec_balance};${transaction.dec_decay};${balance.sub(transaction.dec_balance)};\n`, + (err) => { + if (err) throw err + }, + ) + } + } // previous previous = transaction From 1ea8eb1815b8da079da4b750e8c6519e4438c73b Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Fri, 25 Nov 2022 14:23:47 +0100 Subject: [PATCH 08/10] create log folder --- database/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/database/Dockerfile b/database/Dockerfile index 4069ffcd8..03c7d9a3b 100644 --- a/database/Dockerfile +++ b/database/Dockerfile @@ -100,6 +100,8 @@ COPY --from=build ${DOCKER_WORKDIR}/node_modules ./node_modules COPY --from=build ${DOCKER_WORKDIR}/package.json ./package.json # Copy Mnemonic files COPY --from=build ${DOCKER_WORKDIR}/src/config/*.txt ./src/config/ +# Copy log folder +COPY --from=build ${DOCKER_WORKDIR}/log ./log # Copy run scripts run/ # COPY --from=build ${DOCKER_WORKDIR}/run ./run From bc0cbfe2f73d100807eaa6cc2b899a4c0f2eb891 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Fri, 25 Nov 2022 14:24:47 +0100 Subject: [PATCH 09/10] lint fix --- database/migrations/0054-recalculate_balance_and_decay.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/database/migrations/0054-recalculate_balance_and_decay.ts b/database/migrations/0054-recalculate_balance_and_decay.ts index 6614b4a52..516d0d1e3 100644 --- a/database/migrations/0054-recalculate_balance_and_decay.ts +++ b/database/migrations/0054-recalculate_balance_and_decay.ts @@ -105,7 +105,7 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis const transactions = await queryFn( `SELECT *, CONVERT(balance, CHAR) as dec_balance, CONVERT(decay, CHAR) as dec_decay FROM transactions WHERE user_id = ${userId} ORDER BY balance_date ASC;`, ) - + let previous = null let affectedTransactions = 0 let balance = new Decimal(0) @@ -137,7 +137,11 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis if (t === transactions.length - 1) { fs.appendFile( logFile, - `${userEmail};${userFirstName};${userLastName};${affectedTransactions};${balance};${decay.decay ? decay.decay : 0};${transaction.dec_balance};${transaction.dec_decay};${balance.sub(transaction.dec_balance)};\n`, + `${userEmail};${userFirstName};${userLastName};${affectedTransactions};${balance};${ + decay.decay ? decay.decay : 0 + };${transaction.dec_balance};${transaction.dec_decay};${balance.sub( + transaction.dec_balance, + )};\n`, (err) => { if (err) throw err }, From 2caf0ee664c57bf57fdf1957fdb00fb1b6ba2436 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Sat, 26 Nov 2022 01:14:24 +0100 Subject: [PATCH 10/10] v1.15.0 --- CHANGELOG.md | 13 +++++++++++++ admin/package.json | 2 +- backend/package.json | 2 +- database/package.json | 2 +- frontend/package.json | 2 +- package.json | 2 +- 6 files changed, 18 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ce354b1e..26b71ea03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,21 @@ All notable changes to this project will be documented in this file. Dates are d Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). +#### [1.15.0](https://github.com/gradido/gradido/compare/1.14.1...1.15.0) + +- fix(database): wrong balance and decay values [`#2423`](https://github.com/gradido/gradido/pull/2423) +- fix(backend): wrong balance after transaction receive [`#2422`](https://github.com/gradido/gradido/pull/2422) +- feat(other): feature gradido roadmap [`#2301`](https://github.com/gradido/gradido/pull/2301) +- refactor(backend): new password encryption implementation [`#2353`](https://github.com/gradido/gradido/pull/2353) +- refactor(admin): statistics in a table and on separate page in admin area [`#2399`](https://github.com/gradido/gradido/pull/2399) +- feat(backend): 🍰 Email Templates [`#2163`](https://github.com/gradido/gradido/pull/2163) +- fix(backend): timezone problems [`#2393`](https://github.com/gradido/gradido/pull/2393) + #### [1.14.1](https://github.com/gradido/gradido/compare/1.14.0...1.14.1) +> 14 November 2022 + +- chore(release): version 1.14.1 - hotfix [`#2391`](https://github.com/gradido/gradido/pull/2391) - fix(frontend): load contributionMessages is fixed [`#2390`](https://github.com/gradido/gradido/pull/2390) #### [1.14.0](https://github.com/gradido/gradido/compare/1.13.3...1.14.0) diff --git a/admin/package.json b/admin/package.json index 7f0e7ffd5..75800a526 100644 --- a/admin/package.json +++ b/admin/package.json @@ -3,7 +3,7 @@ "description": "Administraion Interface for Gradido", "main": "index.js", "author": "Moriz Wahl", - "version": "1.14.1", + "version": "1.15.0", "license": "Apache-2.0", "private": false, "scripts": { diff --git a/backend/package.json b/backend/package.json index 3e26225bf..25774fc7d 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,6 +1,6 @@ { "name": "gradido-backend", - "version": "1.14.1", + "version": "1.15.0", "description": "Gradido unified backend providing an API-Service for Gradido Transactions", "main": "src/index.ts", "repository": "https://github.com/gradido/gradido/backend", diff --git a/database/package.json b/database/package.json index 6216a25fb..abc7789c4 100644 --- a/database/package.json +++ b/database/package.json @@ -1,6 +1,6 @@ { "name": "gradido-database", - "version": "1.14.1", + "version": "1.15.0", "description": "Gradido Database Tool to execute database migrations", "main": "src/index.ts", "repository": "https://github.com/gradido/gradido/database", diff --git a/frontend/package.json b/frontend/package.json index cfc12630e..6f1474521 100755 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "bootstrap-vue-gradido-wallet", - "version": "1.14.1", + "version": "1.15.0", "private": true, "scripts": { "start": "node run/server.js", diff --git a/package.json b/package.json index 72efee984..22f444155 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gradido", - "version": "1.14.1", + "version": "1.15.0", "description": "Gradido", "main": "index.js", "repository": "git@github.com:gradido/gradido.git",