add dlt-database as clone from database without the entities and migrations

This commit is contained in:
einhorn_b 2023-08-22 15:42:54 +02:00
parent fab817d55f
commit b4ab690760
22 changed files with 3679 additions and 0 deletions

6
dlt-database/.env.dist Normal file
View File

@ -0,0 +1,6 @@
DB_HOST=localhost
DB_PORT=3306
DB_USER=root
DB_PASSWORD=
DB_DATABASE=gradido_dlt
MIGRATIONS_TABLE=migrations

View File

@ -0,0 +1,8 @@
CONFIG_VERSION=$DATABASE_CONFIG_VERSION
DB_HOST=localhost
DB_PORT=3306
DB_USER=$DB_USER
DB_PASSWORD=$DB_PASSWORD
DB_DATABASE=gradido_dlt
MIGRATIONS_TABLE=migrations

View File

@ -0,0 +1,3 @@
node_modules
**/*.min.js
build

206
dlt-database/.eslintrc.js Normal file
View File

@ -0,0 +1,206 @@
// eslint-disable-next-line import/no-commonjs, import/unambiguous
module.exports = {
root: true,
env: {
node: true,
},
parser: '@typescript-eslint/parser',
plugins: ['prettier', '@typescript-eslint', 'import', 'n', 'promise'],
extends: [
'standard',
'eslint:recommended',
'plugin:prettier/recommended',
'plugin:import/recommended',
'plugin:import/typescript',
// 'plugin:security/recommended',
'plugin:@eslint-community/eslint-comments/recommended',
],
settings: {
'import/parsers': {
'@typescript-eslint/parser': ['.ts', '.tsx'],
},
'import/resolver': {
typescript: {
project: ['./tsconfig.json'],
},
node: true,
},
},
rules: {
'no-console': 'error',
camelcase: 'error',
'no-debugger': 'error',
'prettier/prettier': [
'error',
{
htmlWhitespaceSensitivity: 'ignore',
},
],
// import
'import/export': 'error',
'import/no-deprecated': 'error',
'import/no-empty-named-blocks': 'error',
// 'import/no-extraneous-dependencies': 'error',
'import/no-mutable-exports': 'error',
'import/no-unused-modules': 'error',
'import/no-named-as-default': 'error',
'import/no-named-as-default-member': 'error',
'import/no-amd': 'error',
'import/no-commonjs': 'error',
'import/no-import-module-exports': 'error',
'import/no-nodejs-modules': 'off',
'import/unambiguous': 'error',
'import/default': 'error',
'import/named': 'error',
'import/namespace': 'error',
'import/no-absolute-path': 'error',
// 'import/no-cycle': 'error',
'import/no-dynamic-require': 'error',
'import/no-internal-modules': 'off',
'import/no-relative-packages': 'error',
// 'import/no-relative-parent-imports': ['error', { ignore: ['@/*'] }],
'import/no-self-import': 'error',
'import/no-unresolved': 'error',
'import/no-useless-path-segments': 'error',
'import/no-webpack-loader-syntax': 'error',
'import/consistent-type-specifier-style': 'error',
'import/exports-last': 'off',
'import/extensions': 'error',
'import/first': 'error',
'import/group-exports': 'off',
'import/newline-after-import': 'error',
'import/no-anonymous-default-export': 'error',
'import/no-default-export': 'error',
'import/no-duplicates': 'error',
'import/no-named-default': 'error',
'import/no-namespace': 'error',
'import/no-unassigned-import': 'error',
// 'import/order': [
// 'error',
// {
// groups: ['builtin', 'external', 'internal', 'parent', 'sibling', 'index', 'object', 'type'],
// 'newlines-between': 'always',
// pathGroups: [
// {
// pattern: '@?*/**',
// group: 'external',
// position: 'after',
// },
// {
// pattern: '@/**',
// group: 'external',
// position: 'after',
// },
// ],
// alphabetize: {
// order: 'asc' /* sort in ascending order. Options: ['ignore', 'asc', 'desc'] */,
// caseInsensitive: true /* ignore case. Options: [true, false] */,
// },
// distinctGroup: true,
// },
// ],
'import/prefer-default-export': 'off',
// n
'n/handle-callback-err': 'error',
'n/no-callback-literal': 'error',
'n/no-exports-assign': 'error',
// 'n/no-extraneous-import': 'error',
'n/no-extraneous-require': 'error',
'n/no-hide-core-modules': 'error',
'n/no-missing-import': 'off', // not compatible with typescript
'n/no-missing-require': 'error',
'n/no-new-require': 'error',
'n/no-path-concat': 'error',
// 'n/no-process-exit': 'error',
'n/no-unpublished-bin': 'error',
'n/no-unpublished-import': 'off', // TODO need to exclude seeds
'n/no-unpublished-require': 'error',
'n/no-unsupported-features': ['error', { ignores: ['modules'] }],
'n/no-unsupported-features/es-builtins': 'error',
'n/no-unsupported-features/es-syntax': 'error',
'n/no-unsupported-features/node-builtins': 'error',
'n/process-exit-as-throw': 'error',
'n/shebang': 'error',
'n/callback-return': 'error',
'n/exports-style': 'error',
'n/file-extension-in-import': 'off',
'n/global-require': 'error',
'n/no-mixed-requires': 'error',
'n/no-process-env': 'error',
'n/no-restricted-import': 'error',
'n/no-restricted-require': 'error',
// 'n/no-sync': 'error',
'n/prefer-global/buffer': 'error',
'n/prefer-global/console': 'error',
'n/prefer-global/process': 'error',
'n/prefer-global/text-decoder': 'error',
'n/prefer-global/text-encoder': 'error',
'n/prefer-global/url': 'error',
'n/prefer-global/url-search-params': 'error',
'n/prefer-promises/dns': 'error',
// 'n/prefer-promises/fs': 'error',
// promise
// 'promise/catch-or-return': 'error',
// 'promise/no-return-wrap': 'error',
// 'promise/param-names': 'error',
// 'promise/always-return': 'error',
// 'promise/no-native': 'off',
// 'promise/no-nesting': 'warn',
// 'promise/no-promise-in-callback': 'warn',
// 'promise/no-callback-in-promise': 'warn',
// 'promise/avoid-new': 'warn',
// 'promise/no-new-statics': 'error',
// 'promise/no-return-in-finally': 'warn',
// 'promise/valid-params': 'warn',
// 'promise/prefer-await-to-callbacks': 'error',
// 'promise/no-multiple-resolved': 'error',
// eslint comments
'@eslint-community/eslint-comments/disable-enable-pair': ['error', { allowWholeFile: true }],
'@eslint-community/eslint-comments/no-restricted-disable': 'error',
'@eslint-community/eslint-comments/no-use': 'off',
'@eslint-community/eslint-comments/require-description': 'off',
},
overrides: [
// only for ts files
{
files: ['*.ts', '*.tsx'],
extends: [
// 'plugin:@typescript-eslint/recommended',
// 'plugin:@typescript-eslint/recommended-requiring-type-checking',
// 'plugin:@typescript-eslint/strict',
],
rules: {
// allow explicitly defined dangling promises
// '@typescript-eslint/no-floating-promises': ['error', { ignoreVoid: true }],
'no-void': ['error', { allowAsStatement: true }],
// ignore prefer-regexp-exec rule to allow string.match(regex)
'@typescript-eslint/prefer-regexp-exec': 'off',
// this should not run on ts files: https://github.com/import-js/eslint-plugin-import/issues/2215#issuecomment-911245486
'import/unambiguous': 'off',
// this is not compatible with typeorm, due to joined tables can be null, but are not defined as nullable
'@typescript-eslint/no-unnecessary-condition': 'off',
},
parserOptions: {
tsconfigRootDir: __dirname,
project: ['./tsconfig.json'],
// this is to properly reference the referenced project database without requirement of compiling it
// eslint-disable-next-line camelcase
EXPERIMENTAL_useSourceOfProjectReferenceRedirect: true,
},
},
// we do not have testing on the database
// {
// files: ['*.test.ts'],
// plugins: ['jest'],
// rules: {
// 'jest/no-disabled-tests': 'error',
// 'jest/no-focused-tests': 'error',
// 'jest/no-identical-title': 'error',
// 'jest/prefer-to-have-length': 'error',
// 'jest/valid-expect': 'error',
// '@typescript-eslint/unbound-method': 'off',
// 'jest/unbound-method': 'error',
// },
// },
],
}

27
dlt-database/.gitignore vendored Normal file
View File

@ -0,0 +1,27 @@
.DS_Store
node_modules/
build/
.cache/
npm-debug.log*
yarn-debug.log*
yarn-error.log*
test/unit/coverage
package-lock.json
/.env
/.env.bak
.env.development.local
.env.production.local
# Editor directories and files
.idea
*.suo
*.ntvs*
*.njsproj
*.sln
# coverage folder
coverage/
*~

View File

@ -0,0 +1,9 @@
module.exports = {
semi: false,
printWidth: 100,
singleQuote: true,
trailingComma: "all",
tabWidth: 2,
bracketSpacing: true,
endOfLine: "auto",
};

130
dlt-database/Dockerfile Normal file
View File

@ -0,0 +1,130 @@
##################################################################################
# BASE ###########################################################################
##################################################################################
FROM node:18.7.0-alpine3.16 as base
# ENVs (available in production aswell, can be overwritten by commandline or env file)
## DOCKER_WORKDIR would be a classical ARG, but that is not multi layer persistent - shame
ENV DOCKER_WORKDIR="/app"
## We Cannot do `$(date -u +'%Y-%m-%dT%H:%M:%SZ')` here so we use unix timestamp=0
ENV BUILD_DATE="1970-01-01T00:00:00.00Z"
## We cannot do $(npm run version).${BUILD_NUMBER} here so we default to 0.0.0.0
ENV BUILD_VERSION="0.0.0.0"
## We cannot do `$(git rev-parse --short HEAD)` here so we default to 0000000
ENV BUILD_COMMIT="0000000"
## SET NODE_ENV
ENV NODE_ENV="production"
# Labels
LABEL org.label-schema.build-date="${BUILD_DATE}"
LABEL org.label-schema.name="gradido:database"
LABEL org.label-schema.description="Gradido Database Migration Service"
LABEL org.label-schema.usage="https://github.com/gradido/gradido/blob/master/README.md"
LABEL org.label-schema.url="https://gradido.net"
LABEL org.label-schema.vcs-url="https://github.com/gradido/gradido/tree/master/database"
LABEL org.label-schema.vcs-ref="${BUILD_COMMIT}"
LABEL org.label-schema.vendor="Gradido Community"
LABEL org.label-schema.version="${BUILD_VERSION}"
LABEL org.label-schema.schema-version="1.0"
LABEL maintainer="support@gradido.net"
# Install Additional Software
## install: git
#RUN apk --no-cache add git
## Workdir
RUN mkdir -p ${DOCKER_WORKDIR}
WORKDIR ${DOCKER_WORKDIR}
##################################################################################
# DEVELOPMENT (Connected to the local environment, to reload on demand) ##########
##################################################################################
FROM base as development
# We don't need to copy or build anything since we gonna bind to the
# local filesystem which will need a rebuild anyway
# Run command
# (for development we need to execute npm install since the
# node_modules are on another volume and need updating)
CMD /bin/sh -c "yarn install"
##################################################################################
# BUILD (Does contain all files and is therefore bloated) ########################
##################################################################################
FROM base as build
# Copy everything
COPY . .
# npm install
RUN yarn install --production=false --frozen-lockfile --non-interactive
# npm build
RUN yarn run build
##################################################################################
# TEST UP ########################################################################
##################################################################################
FROM build as test_up
# Run command
CMD /bin/sh -c "yarn install && yarn run dev_up"
##################################################################################
# TEST RESET #####################################################################
##################################################################################
FROM build as test_reset
# Run command
CMD /bin/sh -c "yarn install && yarn run dev_reset"
##################################################################################
# TEST DOWN ######################################################################
##################################################################################
FROM build as test_down
# Run command
CMD /bin/sh -c "yarn install && yarn run dev_down"
##################################################################################
# PRODUCTION (Does contain only "binary"- and static-files to reduce image size) #
##################################################################################
FROM base as production
# Copy "binary"-files from build image
COPY --from=build ${DOCKER_WORKDIR}/build ./build
# We also copy the node_modules express and serve-static for the run script
COPY --from=build ${DOCKER_WORKDIR}/node_modules ./node_modules
# Copy static files
# COPY --from=build ${DOCKER_WORKDIR}/public ./public
# Copy package.json for script definitions (lock file should not be needed)
COPY --from=build ${DOCKER_WORKDIR}/package.json ./package.json
# Copy Mnemonic files
COPY --from=build ${DOCKER_WORKDIR}/src/config/*.txt ./src/config/
# Copy log folder
COPY --from=build ${DOCKER_WORKDIR}/log ./log
# Copy run scripts run/
# COPY --from=build ${DOCKER_WORKDIR}/run ./run
##################################################################################
# PRODUCTION UP ##################################################################
##################################################################################
FROM production as production_up
# Run command
CMD /bin/sh -c "yarn run up"
##################################################################################
# PRODUCTION RESET ###############################################################
##################################################################################
FROM production as production_reset
# Run command
CMD /bin/sh -c "yarn run reset"
##################################################################################
# PRODUCTION DOWN ################################################################
##################################################################################
FROM production as production_down
# Run command
CMD /bin/sh -c "yarn run down"

39
dlt-database/README.md Normal file
View File

@ -0,0 +1,39 @@
# database
## Project setup
```bash
yarn install
```
## Upgrade migrations production
```bash
yarn up
```
## Upgrade migrations development
```bash
yarn dev_up
```
## Downgrade migrations production
```bash
yarn down
```
## Downgrade migrations development
```bash
yarn dev_down
```
## Reset database
```bash
yarn dev_reset
```
Runs all down migrations and after this all up migrations.

2
dlt-database/log/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
*
!.gitignore

View File

@ -0,0 +1,345 @@
/* FIRST MIGRATION
*
* This migration is special since it takes into account that
* the database can be setup already but also may not be.
* Therefore you will find all `CREATE TABLE` statements with
* a `IF NOT EXISTS`, all `INSERT` with an `IGNORE` and in the
* downgrade function all `DROP TABLE` with a `IF EXISTS`.
* This ensures compatibility for existing or non-existing
* databases.
*/
/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
/* eslint-disable @typescript-eslint/no-explicit-any */
export async function upgrade(queryFn: (query: string, values?: any[]) => Promise<Array<any>>) {
// write upgrade logic as parameter of queryFn
await queryFn(`
CREATE TABLE IF NOT EXISTS \`address_types\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`name\` varchar(45) COLLATE utf8mb4_unicode_ci NOT NULL,
\`text\` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`)
await queryFn(`
INSERT IGNORE INTO \`address_types\` (\`id\`, \`name\`, \`text\`) VALUES
(1, 'user main', 'user main address');`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`admin_errors\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`state_user_id\` int(11) NOT NULL,
\`controller\` varchar(255) NOT NULL,
\`action\` varchar(255) NOT NULL,
\`state\` varchar(255) NOT NULL,
\`msg\` varchar(255) NOT NULL,
\`details\` varchar(255) DEFAULT NULL,
\`created\` datetime NOT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`blockchain_types\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`name\` varchar(45) NOT NULL,
\`text\` varchar(255) NULL,
\`symbol\` varchar(10) NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
INSERT IGNORE INTO \`blockchain_types\` (\`id\`, \`name\`, \`text\`, \`symbol\`) VALUES
(1, 'mysql', 'use mysql db as blockchain, work only with single community-server', NULL),
(2, 'hedera', 'use hedera for transactions', 'HBAR');`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`community_profiles\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`state_user_id\` int(10) unsigned NOT NULL,
\`profile_img\` longblob,
\`profile_desc\` varchar(2000) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
PRIMARY KEY (\`id\`),
KEY \`state_user_id\` (\`state_user_id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`operator_types\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`name\` varchar(25) COLLATE utf8mb4_unicode_ci NOT NULL,
\`text\` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`operators\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`username\` varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL,
\`user_pubkey\` binary(32) NOT NULL,
\`data_base64\` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
\`modified\` datetime NOT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`pending_transactions\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`transactionID\` varchar(25) COLLATE utf8mb4_unicode_ci NOT NULL,
\`service\` varchar(20) COLLATE utf8mb4_unicode_ci NOT NULL,
\`method\` varchar(20) COLLATE utf8mb4_unicode_ci NOT NULL,
\`h_server_id\` int(11) NOT NULL,
\`timeout\` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (\`id\`),
UNIQUE KEY \`transactionID\` (\`transactionID\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`roles\` (
\`id\` int UNSIGNED NOT NULL AUTO_INCREMENT,
\`title\` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`server_users\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`username\` varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL,
\`password\` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
\`email\` varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL,
\`role\` varchar(20) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'admin',
\`activated\` tinyint(4) NOT NULL DEFAULT '0',
\`last_login\` datetime DEFAULT NULL,
\`created\` datetime NOT NULL,
\`modified\` datetime NOT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`state_balances\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`state_user_id\` int(10) unsigned NOT NULL,
\`modified\` datetime NOT NULL,
\`record_date\`datetime NULL,
\`amount\` bigint(20) NOT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`state_created\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`transaction_id\` int(10) unsigned NOT NULL,
\`month\` tinyint(3) unsigned NOT NULL,
\`year\` smallint(5) unsigned NOT NULL,
\`state_user_id\` int(10) unsigned NOT NULL,
\`created\` datetime NOT NULL,
\`short_ident_hash\` int(10) unsigned NOT NULL,
PRIMARY KEY (\`id\`),
KEY \`short_ident_hash\` (\`short_ident_hash\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`state_errors\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`state_user_id\` int(10) unsigned NOT NULL,
\`transaction_type_id\` int(10) unsigned NOT NULL,
\`created\` datetime NOT NULL,
\`message_json\` text COLLATE utf8mb4_unicode_ci NOT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`state_group_addresses\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`group_id\` int(10) unsigned NOT NULL,
\`public_key\` binary(32) NOT NULL,
\`address_type_id\` int(10) unsigned NOT NULL,
PRIMARY KEY (\`id\`),
UNIQUE(\`public_key\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`state_group_relationships\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`group1_id\` int(10) unsigned NOT NULL,
\`group2_id\` int(10) unsigned NOT NULL,
\`state_relationship_id\` int(10) unsigned NOT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`state_groups\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`index_id\` varbinary(64) NOT NULL,
\`name\` varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL,
\`root_public_key\` binary(32) NOT NULL,
\`user_count\` smallint(5) unsigned NOT NULL DEFAULT '0',
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`state_relationship_types\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`name\` varchar(25) COLLATE utf8mb4_unicode_ci NOT NULL,
\`text\` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`state_user_roles\` (
\`id\` int UNSIGNED NOT NULL AUTO_INCREMENT,
\`state_user_id\` int(11) NOT NULL,
\`role_id\` int(11) NOT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`state_user_transactions\` (
\`id\` int UNSIGNED NOT NULL AUTO_INCREMENT,
\`state_user_id\` int UNSIGNED NOT NULL,
\`transaction_id\` int UNSIGNED NOT NULL,
\`transaction_type_id\` int UNSIGNED NOT NULL,
\`balance\` bigint(20) DEFAULT 0,
\`balance_date\` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`state_users\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`index_id\` smallint(6) NOT NULL DEFAULT '0',
\`group_id\` int(10) unsigned NOT NULL DEFAULT '0',
\`public_key\` binary(32) NOT NULL,
\`email\` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
\`first_name\` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
\`last_name\` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
\`username\` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
\`disabled\` tinyint(4) DEFAULT '0',
PRIMARY KEY (\`id\`),
UNIQUE KEY \`public_key\` (\`public_key\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`transaction_creations\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`transaction_id\` int(10) unsigned NOT NULL,
\`state_user_id\` int(10) unsigned NOT NULL,
\`amount\` bigint(20) NOT NULL,
\`ident_hash\` binary(32) NULL,
\`target_date\` timestamp NULL DEFAULT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`transaction_group_addaddress\` (
\`id\` int unsigned NOT NULL AUTO_INCREMENT,
\`transaction_id\` int unsigned NOT NULL,
\`address_type_id\` int unsigned NOT NULL,
\`remove_from_group\` BOOLEAN DEFAULT FALSE,
\`public_key\` binary(32) NOT NULL,
\`state_user_id\` int unsigned NOT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`transaction_group_allowtrades\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`transaction_id\` int(10) unsigned NOT NULL,
\`remote_group_id\` varbinary(64) NOT NULL,
\`allow\` tinyint(4) NOT NULL DEFAULT '0',
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`transaction_group_creates\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`transaction_id\` int(10) unsigned NOT NULL,
\`group_public_key\` binary(32) NOT NULL,
\`group_id\` varchar(64) COLLATE utf8mb4_unicode_ci NOT NULL,
\`name\` varchar(64) COLLATE utf8mb4_unicode_ci NOT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`transaction_send_coins\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`transaction_id\` int(10) unsigned NOT NULL,
\`sender_public_key\` binary(32) NOT NULL,
\`state_user_id\` int(10) unsigned DEFAULT 0,
\`receiver_public_key\` binary(32) NOT NULL,
\`receiver_user_id\` int(10) unsigned DEFAULT 0,
\`amount\` bigint(20) NOT NULL,
\`sender_final_balance\` bigint(20) NOT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`transaction_signatures\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`transaction_id\` int(10) unsigned NOT NULL,
\`signature\` binary(64) NOT NULL,
\`pubkey\` binary(32) NOT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`transaction_types\` (
\`id\` int(10) unsigned NOT NULL AUTO_INCREMENT,
\`name\` varchar(90) COLLATE utf8mb4_unicode_ci NOT NULL,
\`text\` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
await queryFn(`
INSERT IGNORE INTO \`transaction_types\` (\`id\`, \`name\`, \`text\`) VALUES
(1, 'creation', 'create new gradidos for member and also for group (in development)'),
(2, 'transfer', 'send gradidos from one member to another, also cross group transfer'),
(3, 'group create', 'create a new group, trigger creation of new hedera topic and new blockchain on node server'),
(4, 'group add member', 'add user to a group or move if he was already in a group'),
(5, 'group remove member', 'remove user from group, maybe he was moved elsewhere'),
(6, 'hedera topic create', 'create new topic on hedera'),
(7, 'hedera topic send message', 'send consensus message over hedera topic'),
(8, 'hedera account create', 'create new account on hedera for holding some founds with unencrypted keys'),
(9, 'decay start', 'signalize the starting point for decay calculation, allowed only once per chain');`)
await queryFn(`
CREATE TABLE IF NOT EXISTS \`transactions\` (
\`id\` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
\`state_group_id\` int(10) unsigned DEFAULT NULL,
\`transaction_type_id\` int(10) unsigned NOT NULL,
\`tx_hash\` binary(48) DEFAULT NULL,
\`memo\` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
\`received\` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
\`blockchain_type_id\` bigint(20) unsigned NOT NULL DEFAULT 1,
PRIMARY KEY (\`id\`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;`)
}
export async function downgrade(queryFn: (query: string, values?: any[]) => Promise<Array<any>>) {
// write downgrade logic as parameter of queryFn
await queryFn(`DROP TABLE IF EXISTS \`address_types\`;`)
await queryFn(`DROP TABLE IF EXISTS \`admin_errors\`;`)
await queryFn(`DROP TABLE IF EXISTS \`blockchain_types\`;`)
await queryFn(`DROP TABLE IF EXISTS \`community_profiles\`;`)
await queryFn(`DROP TABLE IF EXISTS \`operator_types\`;`)
await queryFn(`DROP TABLE IF EXISTS \`operators\`;`)
await queryFn(`DROP TABLE IF EXISTS \`pending_transactions\`;`)
await queryFn(`DROP TABLE IF EXISTS \`roles\`;`)
await queryFn(`DROP TABLE IF EXISTS \`server_users\`;`)
await queryFn(`DROP TABLE IF EXISTS \`state_balances\`;`)
await queryFn(`DROP TABLE IF EXISTS \`state_created\`;`)
await queryFn(`DROP TABLE IF EXISTS \`state_errors\`;`)
await queryFn(`DROP TABLE IF EXISTS \`state_group_addresses\`;`)
await queryFn(`DROP TABLE IF EXISTS \`state_group_relationships\`;`)
await queryFn(`DROP TABLE IF EXISTS \`state_groups\`;`)
await queryFn(`DROP TABLE IF EXISTS \`state_relationship_types\`;`)
await queryFn(`DROP TABLE IF EXISTS \`state_user_roles\`;`)
await queryFn(`DROP TABLE IF EXISTS \`state_user_transactions\`;`)
await queryFn(`DROP TABLE IF EXISTS \`state_users\`;`)
await queryFn(`DROP TABLE IF EXISTS \`transaction_creations\`;`)
await queryFn(`DROP TABLE IF EXISTS \`transaction_group_addaddress\`;`)
await queryFn(`DROP TABLE IF EXISTS \`transaction_group_allowtrades\`;`)
await queryFn(`DROP TABLE IF EXISTS \`transaction_group_creates\`;`)
await queryFn(`DROP TABLE IF EXISTS \`transaction_send_coins\`;`)
await queryFn(`DROP TABLE IF EXISTS \`transaction_signatures\`;`)
await queryFn(`DROP TABLE IF EXISTS \`transaction_types\`;`)
await queryFn(`DROP TABLE IF EXISTS \`transactions\`;`)
}

55
dlt-database/package.json Normal file
View File

@ -0,0 +1,55 @@
{
"name": "gradido-database",
"version": "1.23.2",
"description": "Gradido Database Tool to execute database migrations",
"main": "src/index.ts",
"repository": "https://github.com/gradido/gradido/database",
"author": "Ulf Gebhardt",
"license": "Apache-2.0",
"private": false,
"scripts": {
"build": "mkdir -p build/src/config/ && cp src/config/*.txt build/src/config/ && tsc --build",
"clean": "tsc --build --clean",
"up": "cross-env TZ=UTC node build/src/index.js up",
"down": "cross-env TZ=UTC node build/src/index.js down",
"reset": "cross-env TZ=UTC node build/src/index.js reset",
"dev_up": "cross-env TZ=UTC ts-node src/index.ts up",
"dev_down": "cross-env TZ=UTC ts-node src/index.ts down",
"dev_reset": "cross-env TZ=UTC ts-node src/index.ts reset",
"lint": "eslint --max-warnings=0 --ext .js,.ts ."
},
"devDependencies": {
"@eslint-community/eslint-plugin-eslint-comments": "^3.2.1",
"@types/faker": "^5.5.9",
"@types/node": "^16.10.3",
"@typescript-eslint/eslint-plugin": "^5.57.1",
"@typescript-eslint/parser": "^5.57.1",
"eslint": "^8.37.0",
"eslint-config-prettier": "^8.8.0",
"eslint-config-standard": "^17.0.0",
"eslint-import-resolver-typescript": "^3.5.4",
"eslint-plugin-import": "^2.27.5",
"eslint-plugin-n": "^15.7.0",
"eslint-plugin-prettier": "^4.2.1",
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-security": "^1.7.1",
"prettier": "^2.8.7",
"ts-node": "^10.2.1",
"typescript": "^4.3.5"
},
"dependencies": {
"@types/uuid": "^8.3.4",
"cross-env": "^7.0.3",
"crypto": "^1.0.1",
"decimal.js-light": "^2.5.1",
"dotenv": "^10.0.0",
"mysql2": "^2.3.0",
"reflect-metadata": "^0.1.13",
"ts-mysql-migrate": "^1.0.2",
"typeorm": "^0.3.16",
"uuid": "^8.3.2"
},
"engines": {
"node": ">=14"
}
}

View File

@ -0,0 +1,39 @@
/* eslint-disable n/no-process-env */
import dotenv from 'dotenv'
dotenv.config()
const constants = {
CONFIG_VERSION: {
DEFAULT: 'DEFAULT',
EXPECTED: 'v1.2022-08-22',
CURRENT: '',
},
}
const database = {
DB_HOST: process.env.DB_HOST || 'localhost',
DB_PORT: process.env.DB_PORT ? parseInt(process.env.DB_PORT) : 3306,
DB_USER: process.env.DB_USER || 'root',
DB_PASSWORD: process.env.DB_PASSWORD || '',
DB_DATABASE: process.env.DB_DATABASE || 'gradido_dlt',
}
const migrations = {
MIGRATIONS_TABLE: process.env.MIGRATIONS_TABLE || 'migrations',
}
// Check config version
constants.CONFIG_VERSION.CURRENT = process.env.CONFIG_VERSION || constants.CONFIG_VERSION.DEFAULT
if (
![constants.CONFIG_VERSION.EXPECTED, constants.CONFIG_VERSION.DEFAULT].includes(
constants.CONFIG_VERSION.CURRENT,
)
) {
throw new Error(
`Fatal: Config Version incorrect - expected "${constants.CONFIG_VERSION.EXPECTED}" or "${constants.CONFIG_VERSION.DEFAULT}", but found "${constants.CONFIG_VERSION.CURRENT}"`,
)
}
export const CONFIG = { ...constants, ...database, ...migrations }

56
dlt-database/src/index.ts Normal file
View File

@ -0,0 +1,56 @@
import { createDatabase } from './prepare'
import { CONFIG } from './config'
import { createPool } from 'mysql'
import { Migration } from 'ts-mysql-migrate'
import path from 'path'
const run = async (command: string) => {
// Database actions not supported by our migration library
await createDatabase()
// Initialize Migrations
const pool = createPool({
host: CONFIG.DB_HOST,
port: CONFIG.DB_PORT,
user: CONFIG.DB_USER,
password: CONFIG.DB_PASSWORD,
database: CONFIG.DB_DATABASE,
})
const migration = new Migration({
conn: pool,
tableName: CONFIG.MIGRATIONS_TABLE,
silent: true,
dir: path.join(__dirname, '..', 'migrations'),
})
await migration.initialize()
// Execute command
switch (command) {
case 'up':
await migration.up() // use for upgrade script
break
case 'down':
await migration.down() // use for downgrade script
break
case 'reset':
// TODO protect from production
await migration.reset()
break
default:
throw new Error(`Unsupported command ${command}`)
}
// Terminate connections gracefully
pool.end()
}
run(process.argv[2])
.catch((err) => {
// eslint-disable-next-line no-console
console.log(err)
process.exit(1)
})
.then(() => {
process.exit()
})

View File

@ -0,0 +1,22 @@
import { createConnection } from 'mysql2/promise'
import { CONFIG } from './config'
export const createDatabase = async (): Promise<void> => {
const con = await createConnection({
host: CONFIG.DB_HOST,
port: CONFIG.DB_PORT,
user: CONFIG.DB_USER,
password: CONFIG.DB_PASSWORD,
})
await con.connect()
// Create Database `gradido_dlt`
await con.query(`
CREATE DATABASE IF NOT EXISTS ${CONFIG.DB_DATABASE}
DEFAULT CHARACTER SET utf8mb4
DEFAULT COLLATE utf8mb4_unicode_ci;`)
await con.end()
}

View File

@ -0,0 +1 @@
export * from 'typeorm'

View File

@ -0,0 +1,19 @@
import { Decimal } from 'decimal.js-light'
import { ValueTransformer } from 'typeorm'
Decimal.set({
precision: 25,
rounding: Decimal.ROUND_HALF_UP,
})
export const DecimalTransformer: ValueTransformer = {
/**
* Used to marshal Decimal when writing to the database.
*/
to: (decimal: Decimal | null): string | null => (decimal ? decimal.toString() : null),
/**
* Used to unmarshal Decimal when reading from the database.
*/
from: (decimal: string | null): Decimal | null => (decimal ? new Decimal(decimal) : null),
}

View File

@ -0,0 +1,73 @@
{
"compilerOptions": {
/* Visit https://aka.ms/tsconfig.json to read more about this file */
/* Basic Options */
// "incremental": true, /* Enable incremental compilation */
"target": "es6", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', 'ES2021', or 'ESNEXT'. */
"module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
// "lib": [], /* Specify library files to be included in the compilation. */
// "allowJs": true, /* Allow javascript files to be compiled. */
// "checkJs": true, /* Report errors in .js files. */
// "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
"declaration": true, /* Generates corresponding '.d.ts' file. */
"declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
// "sourceMap": true, /* Generates corresponding '.map' file. */
// "outFile": "./build/outfile.js", /* Concatenate and emit output to single file. */
"outDir": "./build", /* Redirect output structure to the directory. */
// "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
"composite": true, /* Enable project compilation */
// "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */
// "removeComments": true, /* Do not emit comments to output. */
// "noEmit": true, /* Do not emit outputs. */
// "importHelpers": true, /* Import emit helpers from 'tslib'. */
// "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
// "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
/* Strict Type-Checking Options */
"strict": true, /* Enable all strict type-checking options. */
// "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */
// "strictNullChecks": true, /* Enable strict null checks. */
// "strictFunctionTypes": true, /* Enable strict checking of function types. */
// "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
"strictPropertyInitialization": false, /* Enable strict checking of property initialization in classes. */
// "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */
// "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */
/* Additional Checks */
// "noUnusedLocals": true, /* Report errors on unused locals. */
// "noUnusedParameters": true, /* Report errors on unused parameters. */
// "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */
// "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */
// "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */
// "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an 'override' modifier. */
// "noPropertyAccessFromIndexSignature": true, /* Require undeclared properties from index signatures to use element accesses. */
/* Module Resolution Options */
// "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
// "baseUrl": "./", /* Base directory to resolve non-absolute module names. */
// "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
// "rootDirs": [".", "../database"], /* List of root folders whose combined content represents the structure of the project at runtime. */
// "typeRoots": [], /* List of folders to include type definitions from. */
// "types": [], /* Type declaration files to be included in compilation. */
// "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
"esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
// "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
/* Source Map Options */
// "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
// "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */
// "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
/* Experimental Options */
"experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */
"emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */
/* Advanced Options */
"skipLibCheck": true, /* Skip type checking of declaration files. */
"forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */
},
"references": [] /* Any project that is referenced must itself have a `references` array (which may be empty). */
}

2573
dlt-database/yarn.lock Normal file

File diff suppressed because it is too large Load Diff

View File

@ -35,6 +35,12 @@ services:
########################################################
database:
platform: linux/amd64
########################################################
# DLT-DATABASE #############################################
########################################################
dlt-database:
platform: linux/amd64
#########################################################
## NGINX ################################################

View File

@ -149,6 +149,28 @@ services:
# bind the local folder to the docker to allow live reload
- ./database:/app
########################################################
# DLT-DATABASE ##############################################
########################################################
dlt-database:
# we always run on production here since else the service lingers
# feel free to change this behaviour if it seems useful
# Due to problems with the volume caching the built files
# we changed this to test build. This keeps the service running.
# name the image so that it cannot be found in a DockerHub repository, otherwise it will not be built locally from the 'dockerfile' but pulled from there
image: gradido/dlt-database:local-test_up
build:
target: test_up
environment:
- NODE_ENV="development"
volumes:
# This makes sure the docker container has its own node modules.
# Therefore it is possible to have a different node version on the host machine
- dlt-database_node_modules:/app/node_modules
- dlt-database_build:/app/build
# bind the local folder to the docker to allow live reload
- ./dlt-database:/app
#########################################################
## MARIADB ##############################################
#########################################################

View File

@ -77,6 +77,18 @@ services:
- NODE_ENV="test"
# restart: always # this is very dangerous, but worth a test for the delayed mariadb startup at first run
########################################################
# DLT-DATABASE #############################################
########################################################
dlt-database:
image: gradido/dlt-database:test_up
build:
context: ./dlt-database
target: test_up
environment:
- NODE_ENV="test"
# restart: always # this is very dangerous, but worth a test for the delayed mariadb startup at first run
#########################################################
## MARIADB ##############################################
#########################################################

View File

@ -239,6 +239,32 @@ services:
# Application only envs
#env_file:
# - ./frontend/.env
########################################################
# DLT-DATABASE #############################################
########################################################
dlt-database:
# name the image so that it cannot be found in a DockerHub repository, otherwise it will not be built locally from the 'dockerfile' but pulled from there
image: gradido/dlt-database:local-production_up
build:
context: ./dlt-database
target: production_up
depends_on:
- mariadb
networks:
- internal-net
- external-net # this is required to fetch the packages
environment:
# Envs used in Dockerfile
# - DOCKER_WORKDIR="/app"
- BUILD_DATE
- BUILD_VERSION
- BUILD_COMMIT
- NODE_ENV="production"
- DB_HOST=mariadb
# Application only envs
#env_file:
# - ./frontend/.env
#########################################################
## NGINX ################################################