From 31e82b1aa8638c9299bde182983e7c1b688e4722 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 04:06:54 +0200 Subject: [PATCH 01/19] make settings more error prone to prevent invalid migrations --- database/migrations/0002-add_settings.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/migrations/0002-add_settings.ts b/database/migrations/0002-add_settings.ts index 68c340040..d26a2b4cc 100644 --- a/database/migrations/0002-add_settings.ts +++ b/database/migrations/0002-add_settings.ts @@ -11,7 +11,7 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promise>) { await queryFn(` - CREATE TABLE IF NOT EXISTS \`user_setting\` ( + CREATE TABLE \`user_setting\` ( \`id\` int(10) unsigned NOT NULL AUTO_INCREMENT, \`userId\` int(11) NOT NULL, \`key\` varchar(255) NOT NULL, @@ -22,5 +22,5 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis export async function downgrade(queryFn: (query: string, values?: any[]) => Promise>) { // write downgrade logic as parameter of queryFn - await queryFn(`DROP TABLE IF EXISTS \`user_setting\`;`) + await queryFn(`DROP TABLE \`user_setting\`;`) } From 3cee3dbbb717d1be5608e01ab00864ec8df1988c Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 04:07:31 +0200 Subject: [PATCH 02/19] removed sql create statements from login server --- .../skeema/gradido_login/app_access_tokens.sql | 9 --------- .../skeema/gradido_login/elopage_buys.sql | 15 --------------- .../skeema/gradido_login/email_opt_in.sql | 11 ----------- .../skeema/gradido_login/email_opt_in_types.sql | 6 ------ login_server/skeema/gradido_login/groups.sql | 11 ----------- .../skeema/gradido_login/pending_tasks.sql | 13 ------------- login_server/skeema/gradido_login/roles.sql | 7 ------- .../skeema/gradido_login/user_backups.sql | 7 ------- login_server/skeema/gradido_login/user_roles.sql | 6 ------ 9 files changed, 85 deletions(-) delete mode 100644 login_server/skeema/gradido_login/app_access_tokens.sql delete mode 100644 login_server/skeema/gradido_login/elopage_buys.sql delete mode 100644 login_server/skeema/gradido_login/email_opt_in.sql delete mode 100644 login_server/skeema/gradido_login/email_opt_in_types.sql delete mode 100644 login_server/skeema/gradido_login/groups.sql delete mode 100644 login_server/skeema/gradido_login/pending_tasks.sql delete mode 100644 login_server/skeema/gradido_login/roles.sql delete mode 100644 login_server/skeema/gradido_login/user_backups.sql delete mode 100644 login_server/skeema/gradido_login/user_roles.sql diff --git a/login_server/skeema/gradido_login/app_access_tokens.sql b/login_server/skeema/gradido_login/app_access_tokens.sql deleted file mode 100644 index d320bba5c..000000000 --- a/login_server/skeema/gradido_login/app_access_tokens.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE TABLE `app_access_tokens` ( - `id` int unsigned NOT NULL AUTO_INCREMENT, - `user_id` int NOT NULL, - `access_code` bigint unsigned NOT NULL, - `created` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `updated` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `access_code` (`access_code`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/login_server/skeema/gradido_login/elopage_buys.sql b/login_server/skeema/gradido_login/elopage_buys.sql deleted file mode 100644 index ba39e8c7a..000000000 --- a/login_server/skeema/gradido_login/elopage_buys.sql +++ /dev/null @@ -1,15 +0,0 @@ -CREATE TABLE `elopage_buys` ( - `id` int unsigned NOT NULL AUTO_INCREMENT, - `elopage_user_id` int DEFAULT NULL, - `affiliate_program_id` int NOT NULL, - `publisher_id` int NOT NULL, - `order_id` int NOT NULL, - `product_id` int NOT NULL, - `product_price` int NOT NULL, - `payer_email` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, - `publisher_email` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, - `payed` tinyint NOT NULL, - `success_date` datetime NOT NULL, - `event` varchar(255) NOT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/login_server/skeema/gradido_login/email_opt_in.sql b/login_server/skeema/gradido_login/email_opt_in.sql deleted file mode 100644 index da4288475..000000000 --- a/login_server/skeema/gradido_login/email_opt_in.sql +++ /dev/null @@ -1,11 +0,0 @@ -CREATE TABLE `email_opt_in` ( - `id` int unsigned NOT NULL AUTO_INCREMENT, - `user_id` int NOT NULL, - `verification_code` bigint unsigned NOT NULL, - `email_opt_in_type_id` int NOT NULL, - `created` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `resend_count` int DEFAULT '0', - `updated` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - UNIQUE KEY `verification_code` (`verification_code`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/login_server/skeema/gradido_login/email_opt_in_types.sql b/login_server/skeema/gradido_login/email_opt_in_types.sql deleted file mode 100644 index dbcd2d272..000000000 --- a/login_server/skeema/gradido_login/email_opt_in_types.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE `email_opt_in_types` ( - `id` int unsigned NOT NULL AUTO_INCREMENT, - `name` varchar(255) NOT NULL, - `description` varchar(255) NOT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/login_server/skeema/gradido_login/groups.sql b/login_server/skeema/gradido_login/groups.sql deleted file mode 100644 index 9d57a3b2c..000000000 --- a/login_server/skeema/gradido_login/groups.sql +++ /dev/null @@ -1,11 +0,0 @@ -CREATE TABLE `groups` ( - `id` int unsigned NOT NULL AUTO_INCREMENT, - `alias` varchar(190) NOT NULL, - `name` varchar(255) NOT NULL, - `url` varchar(255) NOT NULL, - `host` varchar(255) DEFAULT "/", - `home` varchar(255) DEFAULT "/", - `description` text, - PRIMARY KEY (`id`), - UNIQUE KEY `alias` (`alias`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/login_server/skeema/gradido_login/pending_tasks.sql b/login_server/skeema/gradido_login/pending_tasks.sql deleted file mode 100644 index 6b7570c0c..000000000 --- a/login_server/skeema/gradido_login/pending_tasks.sql +++ /dev/null @@ -1,13 +0,0 @@ -CREATE TABLE `pending_tasks` ( - `id` int UNSIGNED NOT NULL AUTO_INCREMENT, - `user_id` int UNSIGNED DEFAULT 0, - `request` varbinary(2048) NOT NULL, - `created` datetime NOT NULL, - `finished` datetime DEFAULT '2000-01-01 000000', - `result_json` text DEFAULT NULL, - `param_json` text DEFAULT NULL, - `task_type_id` int UNSIGNED NOT NULL, - `child_pending_task_id` int UNSIGNED DEFAULT 0, - `parent_pending_task_id` int UNSIGNED DEFAULT 0, - PRIMARY KEY (`id`) -) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/login_server/skeema/gradido_login/roles.sql b/login_server/skeema/gradido_login/roles.sql deleted file mode 100644 index c99223961..000000000 --- a/login_server/skeema/gradido_login/roles.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE TABLE `roles` ( - `id` int unsigned NOT NULL AUTO_INCREMENT, - `name` varchar(255) NOT NULL, - `description` varchar(255) NOT NULL, - `flags` bigint NOT NULL DEFAULT '0', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/login_server/skeema/gradido_login/user_backups.sql b/login_server/skeema/gradido_login/user_backups.sql deleted file mode 100644 index 7e3b7d85a..000000000 --- a/login_server/skeema/gradido_login/user_backups.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE TABLE `user_backups` ( - `id` int unsigned NOT NULL AUTO_INCREMENT, - `user_id` int NOT NULL, - `passphrase` text NOT NULL, - `mnemonic_type` int DEFAULT '-1', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/login_server/skeema/gradido_login/user_roles.sql b/login_server/skeema/gradido_login/user_roles.sql deleted file mode 100644 index a92154ce8..000000000 --- a/login_server/skeema/gradido_login/user_roles.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE `user_roles` ( - `id` int unsigned NOT NULL AUTO_INCREMENT, - `user_id` int NOT NULL, - `role_id` int NOT NULL, - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; From c15ea2c36ec2d9d24712e2d04878f8c4e352302a Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 04:08:02 +0200 Subject: [PATCH 03/19] add login server table migrations for login server tables --- .../migrations/0003-login_server_tables.ts | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 database/migrations/0003-login_server_tables.ts diff --git a/database/migrations/0003-login_server_tables.ts b/database/migrations/0003-login_server_tables.ts new file mode 100644 index 000000000..dacc211ac --- /dev/null +++ b/database/migrations/0003-login_server_tables.ts @@ -0,0 +1,153 @@ +/* FIRST MIGRATION + * + * This migration is special since it takes into account that + * the database can be setup already but also may not be. + * Therefore you will find all `CREATE TABLE` statements with + * a `IF NOT EXISTS`, all `INSERT` with an `IGNORE` and in the + * downgrade function all `DROP TABLE` with a `IF EXISTS`. + * This ensures compatibility for existing or non-existing + * databases. + */ + +export async function upgrade(queryFn: (query: string, values?: any[]) => Promise>) { + await queryFn(` + CREATE TABLE \`login_app_access_tokens\` ( + \`id\` int unsigned NOT NULL AUTO_INCREMENT, + \`user_id\` int NOT NULL, + \`access_code\` bigint unsigned NOT NULL, + \`created\` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + \`updated\` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (\`id\`), + UNIQUE KEY \`access_code\` (\`access_code\`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + `) + await queryFn(` + CREATE TABLE \`login_elopage_buys\` ( + \`id\` int unsigned NOT NULL AUTO_INCREMENT, + \`elopage_user_id\` int DEFAULT NULL, + \`affiliate_program_id\` int NOT NULL, + \`publisher_id\` int NOT NULL, + \`order_id\` int NOT NULL, + \`product_id\` int NOT NULL, + \`product_price\` int NOT NULL, + \`payer_email\` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + \`publisher_email\` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + \`payed\` tinyint NOT NULL, + \`success_date\` datetime NOT NULL, + \`event\` varchar(255) NOT NULL, + PRIMARY KEY (\`id\`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + `) + await queryFn(` + CREATE TABLE \`login_email_opt_in_types\` ( + \`id\` int unsigned NOT NULL AUTO_INCREMENT, + \`name\` varchar(255) NOT NULL, + \`description\` varchar(255) NOT NULL, + PRIMARY KEY (\`id\`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + `) + await queryFn(` + CREATE TABLE \`login_email_opt_in\` ( + \`id\` int unsigned NOT NULL AUTO_INCREMENT, + \`user_id\` int NOT NULL, + \`verification_code\` bigint unsigned NOT NULL, + \`email_opt_in_type_id\` int NOT NULL, + \`created\` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + \`resend_count\` int DEFAULT '0', + \`updated\` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (\`id\`), + UNIQUE KEY \`verification_code\` (\`verification_code\`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + `) + await queryFn(` + CREATE TABLE \`login_groups\` ( + \`id\` int unsigned NOT NULL AUTO_INCREMENT, + \`alias\` varchar(190) NOT NULL, + \`name\` varchar(255) NOT NULL, + \`url\` varchar(255) NOT NULL, + \`host\` varchar(255) DEFAULT "/", + \`home\` varchar(255) DEFAULT "/", + \`description\` text, + PRIMARY KEY (\`id\`), + UNIQUE KEY \`alias\` (\`alias\`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + `) + await queryFn(` + CREATE TABLE \`login_pending_tasks\` ( + \`id\` int UNSIGNED NOT NULL AUTO_INCREMENT, + \`user_id\` int UNSIGNED DEFAULT 0, + \`request\` varbinary(2048) NOT NULL, + \`created\` datetime NOT NULL, + \`finished\` datetime DEFAULT '2000-01-01 000000', + \`result_json\` text DEFAULT NULL, + \`param_json\` text DEFAULT NULL, + \`task_type_id\` int UNSIGNED NOT NULL, + \`child_pending_task_id\` int UNSIGNED DEFAULT 0, + \`parent_pending_task_id\` int UNSIGNED DEFAULT 0, + PRIMARY KEY (\`id\`) + ) ENGINE = InnoDB DEFAULT CHARSET=utf8mb4; + `) + await queryFn(` + CREATE TABLE \`login_roles\` ( + \`id\` int unsigned NOT NULL AUTO_INCREMENT, + \`name\` varchar(255) NOT NULL, + \`description\` varchar(255) NOT NULL, + \`flags\` bigint NOT NULL DEFAULT '0', + PRIMARY KEY (\`id\`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + `) + await queryFn(` + CREATE TABLE \`login_user_backups\` ( + \`id\` int unsigned NOT NULL AUTO_INCREMENT, + \`user_id\` int NOT NULL, + \`passphrase\` text NOT NULL, + \`mnemonic_type\` int DEFAULT '-1', + PRIMARY KEY (\`id\`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + `) + await queryFn(` + CREATE TABLE \`login_user_roles\` ( + \`id\` int unsigned NOT NULL AUTO_INCREMENT, + \`user_id\` int NOT NULL, + \`role_id\` int NOT NULL, + PRIMARY KEY (\`id\`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + `) + await queryFn(` + CREATE TABLE \`login_users\` ( + \`id\` int unsigned NOT NULL AUTO_INCREMENT, + \`email\` varchar(191) NOT NULL, + \`first_name\` varchar(150) NOT NULL, + \`last_name\` varchar(255) DEFAULT '', + \`username\` varchar(255) DEFAULT '', + \`description\` text DEFAULT '', + \`password\` bigint unsigned DEFAULT '0', + \`pubkey\` binary(32) DEFAULT NULL, + \`privkey\` binary(80) DEFAULT NULL, + \`email_hash\` binary(32) DEFAULT NULL, + \`created\` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + \`email_checked\` tinyint NOT NULL DEFAULT '0', + \`passphrase_shown\` tinyint NOT NULL DEFAULT '0', + \`language\` varchar(4) NOT NULL DEFAULT 'de', + \`disabled\` tinyint DEFAULT '0', + \`group_id\` int unsigned DEFAULT 0, + \`publisher_id\` int DEFAULT 0, + PRIMARY KEY (\`id\`), + UNIQUE KEY \`email\` (\`email\`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + `) +} + +export async function downgrade(queryFn: (query: string, values?: any[]) => Promise>) { + // write downgrade logic as parameter of queryFn + await queryFn(`DROP TABLE \`login_app_access_tokens\`;`) + await queryFn(`DROP TABLE \`login_elopage_buys\`;`) + await queryFn(`DROP TABLE \`login_email_opt_in_types\`;`) + await queryFn(`DROP TABLE \`login_email_opt_in\`;`) + await queryFn(`DROP TABLE \`login_groups\`;`) + await queryFn(`DROP TABLE \`login_pending_tasks\`;`) + await queryFn(`DROP TABLE \`login_roles\`;`) + await queryFn(`DROP TABLE \`login_user_backups\`;`) + await queryFn(`DROP TABLE \`login_user_roles\`;`) + await queryFn(`DROP TABLE \`login_users\`;`) +} From 2b87f2eaf29be9bdaa78c37265e9d6aca56afc9a Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 04:28:55 +0200 Subject: [PATCH 04/19] move all data from login_server database to community database --- database/migrations/0004-login_server_data.ts | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 database/migrations/0004-login_server_data.ts diff --git a/database/migrations/0004-login_server_data.ts b/database/migrations/0004-login_server_data.ts new file mode 100644 index 000000000..34cba5b6f --- /dev/null +++ b/database/migrations/0004-login_server_data.ts @@ -0,0 +1,59 @@ +/* FIRST MIGRATION + * + * This migration is special since it takes into account that + * the database can be setup already but also may not be. + * Therefore you will find all `CREATE TABLE` statements with + * a `IF NOT EXISTS`, all `INSERT` with an `IGNORE` and in the + * downgrade function all `DROP TABLE` with a `IF EXISTS`. + * This ensures compatibility for existing or non-existing + * databases. + */ + +const LOGIN_SERVER_DB = '`gradido_login`' + +export async function upgrade(queryFn: (query: string, values?: any[]) => Promise>) { + await queryFn(` + INSERT INTO \`login_app_access_tokens\` SELECT * FROM ${LOGIN_SERVER_DB}.\`app_access_tokens\`; + `) + await queryFn(` + INSERT INTO \`login_elopage_buys\` SELECT * FROM ${LOGIN_SERVER_DB}.\`elopage_buys\`; + `) + await queryFn(` + INSERT INTO \`login_email_opt_in_types\` SELECT * FROM ${LOGIN_SERVER_DB}.\`email_opt_in_types\`; + `) + await queryFn(` + INSERT INTO \`login_email_opt_in\` SELECT * FROM ${LOGIN_SERVER_DB}.\`email_opt_in\`; + `) + await queryFn(` + INSERT INTO \`login_groups\` SELECT * FROM ${LOGIN_SERVER_DB}.\`groups\`; + `) + await queryFn(` + INSERT INTO \`login_pending_tasks\` SELECT * FROM ${LOGIN_SERVER_DB}.\`pending_tasks\`; + `) + await queryFn(` + INSERT INTO \`login_roles\` SELECT * FROM ${LOGIN_SERVER_DB}.\`roles\`; + `) + await queryFn(` + INSERT INTO \`login_user_backups\` SELECT * FROM ${LOGIN_SERVER_DB}.\`user_backups\`; + `) + await queryFn(` + INSERT INTO \`login_user_roles\` SELECT * FROM ${LOGIN_SERVER_DB}.\`user_roles\`; + `) + await queryFn(` + INSERT INTO \`login_users\` SELECT * FROM ${LOGIN_SERVER_DB}.\`users\`; + `) +} + +export async function downgrade(queryFn: (query: string, values?: any[]) => Promise>) { + // write downgrade logic as parameter of queryFn + await queryFn(`DELETE FROM \`login_app_access_tokens\`;`) + await queryFn(`DELETE FROM \`login_elopage_buys\`;`) + await queryFn(`DELETE FROM \`login_email_opt_in_types\`;`) + await queryFn(`DELETE FROM \`login_email_opt_in\`;`) + await queryFn(`DELETE FROM \`login_groups\`;`) + await queryFn(`DELETE FROM \`login_pending_tasks\`;`) + await queryFn(`DELETE FROM \`login_roles\`;`) + await queryFn(`DELETE FROM \`login_user_backups\`;`) + await queryFn(`DELETE FROM \`login_user_roles\`;`) + await queryFn(`DELETE FROM \`login_users\`;`) +} From e64019d6c3c1cadb37009a0474f9b3ee26522574 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 04:31:42 +0200 Subject: [PATCH 05/19] also remove user.sql file --- login_server/skeema/gradido_login/users.sql | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 login_server/skeema/gradido_login/users.sql diff --git a/login_server/skeema/gradido_login/users.sql b/login_server/skeema/gradido_login/users.sql deleted file mode 100644 index 95b6acec1..000000000 --- a/login_server/skeema/gradido_login/users.sql +++ /dev/null @@ -1,21 +0,0 @@ -CREATE TABLE `users` ( - `id` int unsigned NOT NULL AUTO_INCREMENT, - `email` varchar(191) NOT NULL, - `first_name` varchar(150) NOT NULL, - `last_name` varchar(255) DEFAULT '', - `username` varchar(255) DEFAULT '', - `description` text DEFAULT '', - `password` bigint unsigned DEFAULT '0', - `pubkey` binary(32) DEFAULT NULL, - `privkey` binary(80) DEFAULT NULL, - `email_hash` binary(32) DEFAULT NULL, - `created` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `email_checked` tinyint NOT NULL DEFAULT '0', - `passphrase_shown` tinyint NOT NULL DEFAULT '0', - `language` varchar(4) NOT NULL DEFAULT 'de', - `disabled` tinyint DEFAULT '0', - `group_id` int unsigned DEFAULT 0, - `publisher_id` int DEFAULT 0, - PRIMARY KEY (`id`), - UNIQUE KEY `email` (`email`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; From c65ac49594e739b1214637b19473ededbaded6f1 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 04:49:19 +0200 Subject: [PATCH 06/19] - removed skeema - simplified mariadb docker container & reduced context(build time) - datapoint for docker group into migration - removed unused node docker definitions from docker-compose --- .github/workflows/test.yml | 2 +- .gitignore | 1 - .../setup_db_tables/setup_docker_group.sql | 6 -- database/migrations/0004-login_server_data.ts | 6 ++ deployment/bare_metal/update_all.sh | 2 - deployment/bare_metal/update_db_schemas.sh | 7 --- docker-compose.override.yml | 17 +----- docker-compose.test.yml | 15 +---- docker-compose.yml | 40 +------------- .../Architecture/TechnicalOverview.md | 2 +- login_server/.gitignore | 1 - login_server/README.md | 3 +- mariadb/.skeema | 5 -- mariadb/.skeema.login | 3 - mariadb/Dockerfile | 28 +--------- mariadb/setup_dbs.sql | 7 --- mariadb/setup_dbs_different_user.sh | 55 ------------------- mariadb/setup_test_dbs.sql | 10 ---- skeema/.skeema | 5 -- skeema/Dockerfile | 39 ------------- 20 files changed, 16 insertions(+), 238 deletions(-) delete mode 100644 configs/login_server/setup_db_tables/setup_docker_group.sql delete mode 100755 deployment/bare_metal/update_db_schemas.sh delete mode 100644 mariadb/.skeema delete mode 100644 mariadb/.skeema.login delete mode 100644 mariadb/setup_dbs.sql delete mode 100755 mariadb/setup_dbs_different_user.sh delete mode 100644 mariadb/setup_test_dbs.sql delete mode 100644 skeema/.skeema delete mode 100644 skeema/Dockerfile diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b6fd6c31d..acd1ac7cc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -147,7 +147,7 @@ jobs: ########################################################################## - name: mariadb | Build `test` image run: | - docker build --target mariadb_server_test -t "gradido/mariadb:test" -f ./mariadb/Dockerfile ./ + docker build --target mariadb_server -t "gradido/mariadb:test" -f ./mariadb/Dockerfile ./ docker save "gradido/mariadb:test" > /tmp/mariadb.tar - name: Upload Artifact uses: actions/upload-artifact@v2 diff --git a/.gitignore b/.gitignore index caed320af..c09811443 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ /node_modules/* .vscode messages.pot -.skeema nbproject .metadata /.env diff --git a/configs/login_server/setup_db_tables/setup_docker_group.sql b/configs/login_server/setup_db_tables/setup_docker_group.sql deleted file mode 100644 index 5f1b64c3f..000000000 --- a/configs/login_server/setup_db_tables/setup_docker_group.sql +++ /dev/null @@ -1,6 +0,0 @@ -INSERT INTO `groups` (`id`, `alias`, `name`, `url`, `host`, `home`, `description`) VALUES -(1, 'docker', 'docker gradido group', 'localhost', 'nginx', '/', 'gradido test group for docker and stage2 with blockchain db'); - - - - diff --git a/database/migrations/0004-login_server_data.ts b/database/migrations/0004-login_server_data.ts index 34cba5b6f..6a6668bd9 100644 --- a/database/migrations/0004-login_server_data.ts +++ b/database/migrations/0004-login_server_data.ts @@ -42,6 +42,12 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis await queryFn(` INSERT INTO \`login_users\` SELECT * FROM ${LOGIN_SERVER_DB}.\`users\`; `) + + // TODO clarify if we need this on non docker environment? + await queryFn(` + INSERT IGNORE INTO \`login_groups\` (\`id\`, \`alias\`, \`name\`, \`url\`, \`host\`, \`home\`, \`description\`) VALUES + (1, 'docker', 'docker gradido group', 'localhost', 'nginx', '/', 'gradido test group for docker and stage2 with blockchain db'); + `) } export async function downgrade(queryFn: (query: string, values?: any[]) => Promise>) { diff --git a/deployment/bare_metal/update_all.sh b/deployment/bare_metal/update_all.sh index b065a16d4..e045d3f39 100755 --- a/deployment/bare_metal/update_all.sh +++ b/deployment/bare_metal/update_all.sh @@ -36,8 +36,6 @@ cd $PROJECT_PATH # git checkout -f master git pull cd deployment/bare_metal -echo 'update schemas' >> $UPDATE_HTML -./update_db_schemas.sh echo 'starting with rebuilding login-server
' >> $UPDATE_HTML ./build_and_start_login_server.sh echo 'starting with rebuilding frontend
' >> $UPDATE_HTML diff --git a/deployment/bare_metal/update_db_schemas.sh b/deployment/bare_metal/update_db_schemas.sh deleted file mode 100755 index 80628f35c..000000000 --- a/deployment/bare_metal/update_db_schemas.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# For that to work skeema needed to be installed on system -# in login_server/skeema skeema configuration files need to be there also in the subfolders -# Update DB Schemas (only the schemas, not the data) - -cd ../../login_server/skeema -skeema push --allow-unsafe diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 1fa6c7e23..5783b9fe7 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -136,22 +136,7 @@ services: - external-net volumes: - /sessions - - ######################################################### - ## skeema for updating dbs if changes happend ########### - ######################################################### - skeema: - build: - context: . - dockerfile: ./skeema/Dockerfile - target: skeema_dev_run - depends_on: - - mariadb - networks: - - internal-net - volumes: - - ./login_server/skeema/gradido_login:/skeema/gradido_login - + volumes: frontend_node_modules: backend_node_modules: diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 457faf950..73da574d2 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -9,7 +9,7 @@ services: build: context: . dockerfile: ./mariadb/Dockerfile - target: mariadb_server_test + target: mariadb_server environment: - MARIADB_ALLOW_EMPTY_PASSWORD=1 - MARIADB_USER=root @@ -75,19 +75,6 @@ services: - external-net volumes: - /sessions - - ######################################################### - ## skeema for updating dbs if changes happend ########### - ######################################################### - skeema: - build: - context: . - dockerfile: ./skeema/Dockerfile - target: skeema_run - depends_on: - - mariadb - networks: - - internal-net networks: external-net: diff --git a/docker-compose.yml b/docker-compose.yml index b85d155fe..3ce88715d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,9 +35,8 @@ services: ######################################################### mariadb: build: - context: . - dockerfile: ./mariadb/Dockerfile - target: mariadb_server_test + context: ./mariadb + target: mariadb_server environment: - MARIADB_ALLOW_EMPTY_PASSWORD=1 - MARIADB_USER=root @@ -158,41 +157,6 @@ services: - internal-net volumes: - ./community_server/config/php-fpm/php-ini-overrides.ini:/etc/php/7.4/fpm/conf.d/99-overrides.ini - - ######################################################### - ## skeema for updating dbs if changes happend ########### - ######################################################### - skeema: - build: - context: . - dockerfile: ./skeema/Dockerfile - target: skeema_run - depends_on: - - mariadb - networks: - - internal-net - - ######################################################### - ## GRADIDO NODE v1 ###################################### - ######################################################### - # gradido-node: - # build: - # context: . - # dockerfile: ./gn/docker/deprecated-hedera-node/Dockerfile - # volumes: - # - ${GN_INSTANCE_FOLDER}:/opt/instance - # container_name: ${GN_CONTAINER_NAME} - - ######################################################### - ## GRADIDO NODE test ################################### - ######################################################### - # gradido-node-test: - # build: - # context: . - # dockerfile: ./gn/docker/deprecated-hedera-node/Dockerfile - # container_name: gn-test - # working_dir: /opt/gn/build - # command: ["./unit_tests"] networks: external-net: diff --git a/docu/Concepts/TechnicalRequirements/Architecture/TechnicalOverview.md b/docu/Concepts/TechnicalRequirements/Architecture/TechnicalOverview.md index f0fe1e52a..73f74dbae 100644 --- a/docu/Concepts/TechnicalRequirements/Architecture/TechnicalOverview.md +++ b/docu/Concepts/TechnicalRequirements/Architecture/TechnicalOverview.md @@ -15,7 +15,7 @@ This document describes the technical overview for the Gradido infrastructur. Be ![CommunityServerAPI](../image/CommunityServerAPI.png) -### Database Skeema +### Database Skeema (outdated) ![CommunityDBSkeema](../image/CommunityDBSkeema.png) diff --git a/login_server/.gitignore b/login_server/.gitignore index 1d623d885..ca7f5f440 100644 --- a/login_server/.gitignore +++ b/login_server/.gitignore @@ -5,6 +5,5 @@ src/cpsp/*.h src/cpsp/*.cpp src/cpp/proto/ build*/ -/skeema/gradido_login/insert/crypto_key.sql src/LOCALE/messages.pot diff --git a/login_server/README.md b/login_server/README.md index 73f5deea0..118d0d53c 100644 --- a/login_server/README.md +++ b/login_server/README.md @@ -56,8 +56,7 @@ To update messages.pot run This will be also called by ./scripts/build_debug.sh ## database -Login-Server needs a db to run, it is tested with mariadb -table definitions are found in folder ./skeema/gradido_login +Login-Server needs a db to run, it is tested with mariadb. Currently at least one group must be present in table groups. For example: ```sql diff --git a/mariadb/.skeema b/mariadb/.skeema deleted file mode 100644 index e5525e7b5..000000000 --- a/mariadb/.skeema +++ /dev/null @@ -1,5 +0,0 @@ -[production] -flavor=mariadb:10.5 -host=127.0.0.1 -port=3306 -user=root \ No newline at end of file diff --git a/mariadb/.skeema.login b/mariadb/.skeema.login deleted file mode 100644 index c6431dd8c..000000000 --- a/mariadb/.skeema.login +++ /dev/null @@ -1,3 +0,0 @@ -default-character-set=utf8mb4 -default-collation=utf8mb4_unicode_ci -schema=gradido_login \ No newline at end of file diff --git a/mariadb/Dockerfile b/mariadb/Dockerfile index 6b6b1e9aa..07d2ba368 100644 --- a/mariadb/Dockerfile +++ b/mariadb/Dockerfile @@ -3,29 +3,7 @@ ######################################################################################################### FROM mariadb/server:10.5 as mariadb_server -ENV DOCKER_WORKDIR="/docker-entrypoint-initdb.d" +# ENV DOCKER_WORKDIR="/docker-entrypoint-initdb.d" -RUN mkdir -p ${DOCKER_WORKDIR} -WORKDIR ${DOCKER_WORKDIR} - -# create databases -COPY ./mariadb/setup_dbs.sql a1_setup_dbs.sql -# login server db -COPY ./login_server/skeema/ . -RUN cd ./gradido_login/ && for f in *.sql; do cp -- "$f" "../b1_$f"; sed -i '1i use gradido_login;' "../b1_$f"; done -COPY ./configs/login_server/setup_db_tables ./gradido_login/insert -RUN cd ./gradido_login/insert && for f in *.sql; do cp -- "$f" "../../c1_$f"; sed -i '1i use gradido_login;' "../../c1_$f"; done - -######################################################################################################### -# mariadb server with test dbs -######################################################################################################### -FROM mariadb_server as mariadb_server_test - -# create test databases -COPY ./mariadb/setup_test_dbs.sql a2_setup_dbs.sql - -# login server test db -COPY ./login_server/skeema/ . -RUN cd ./gradido_login/ && for f in *.sql; do cp -- "$f" "../b2_$f"; sed -i '1i use gradido_login_test;' "../b2_$f"; done -COPY ./configs/login_server/setup_db_tables ./gradido_login/insert -RUN cd ./gradido_login/insert && for f in *.sql; do cp -- "$f" "../../c2_$f"; sed -i '1i use gradido_login_test;' "../../c2_$f"; done +# RUN mkdir -p ${DOCKER_WORKDIR} +# WORKDIR ${DOCKER_WORKDIR} diff --git a/mariadb/setup_dbs.sql b/mariadb/setup_dbs.sql deleted file mode 100644 index bf44c2ae6..000000000 --- a/mariadb/setup_dbs.sql +++ /dev/null @@ -1,7 +0,0 @@ -create database gradido_login - DEFAULT CHARACTER SET utf8mb4 - DEFAULT COLLATE utf8mb4_unicode_ci; -create database IF NOT EXISTS _skeema_tmp - DEFAULT CHARACTER SET utf8mb4 - DEFAULT COLLATE utf8mb4_unicode_ci; -FLUSH PRIVILEGES; \ No newline at end of file diff --git a/mariadb/setup_dbs_different_user.sh b/mariadb/setup_dbs_different_user.sh deleted file mode 100755 index 139337cbb..000000000 --- a/mariadb/setup_dbs_different_user.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -COLOR_GREEN="\033[0;32m" -COLOR_YELLOW="\e[33m" -COLOR_NONE="\033[0m" - -LOGIN_DB_USER=gradido_login_live -LOGIN_DB_NAME=gradido_login_live -LOGIN_DB_PASSWD=$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo); - -COMMUNITY_DB_USER=gradido_community_live -COMMUNITY_DB_NAME=gradido_community_live -COMMUNITY_DB_PASSWD=$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo); - -# create table -mysql < .skeema -[production] -flavor=mariadb:10.3.25 -host=127.0.0.1 -port=3306 -user=$LOGIN_DB_USER -EOF -cd gradido_login -sudo cat << EOF > .skeema -default-character-set=utf8mb4 -default-collation=utf8mb4_unicode_ci -schema=$LOGIN_DB_NAME -EOF - -source $HOME/.gvm/scripts/gvm -gvm use go1.14.4 -skeema push -p$LOGIN_DB_PASSWD - -echo -e "${COLOR_YELLOW}Login-Server db password: $LOGIN_DB_PASSWD${COLOR_NONE}" diff --git a/mariadb/setup_test_dbs.sql b/mariadb/setup_test_dbs.sql deleted file mode 100644 index 10326d002..000000000 --- a/mariadb/setup_test_dbs.sql +++ /dev/null @@ -1,10 +0,0 @@ -create database gradido_login_test - DEFAULT CHARACTER SET utf8mb4 - DEFAULT COLLATE utf8mb4_unicode_ci; -create database gradido_community_test - DEFAULT CHARACTER SET utf8mb4 - DEFAULT COLLATE utf8mb4_unicode_ci; -create database IF NOT EXISTS _skeema_tmp - DEFAULT CHARACTER SET utf8mb4 - DEFAULT COLLATE utf8mb4_unicode_ci; -FLUSH PRIVILEGES; \ No newline at end of file diff --git a/skeema/.skeema b/skeema/.skeema deleted file mode 100644 index dc87968d6..000000000 --- a/skeema/.skeema +++ /dev/null @@ -1,5 +0,0 @@ -[production] -flavor=mariadb:10.5 -host=mariadb -port=3306 -user=root \ No newline at end of file diff --git a/skeema/Dockerfile b/skeema/Dockerfile deleted file mode 100644 index bbc95f2d0..000000000 --- a/skeema/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -######################################################################################################### -# Build skeema -######################################################################################################### -FROM golang:1.17.1 as skeema_build -RUN go get -d -v github.com/skeema/skeema -WORKDIR /go/src/github.com/skeema/skeema -RUN go install github.com/skeema/skeema@v1.5.3 - -######################################################################################################### -# Run skeema for dev (dynamic) -######################################################################################################### -FROM skeema_build as skeema_dev_run - -ENV DOCKER_WORKDIR="/skeema" - -RUN mkdir -p ${DOCKER_WORKDIR} -WORKDIR ${DOCKER_WORKDIR} - -COPY ./skeema/.skeema . -COPY ./mariadb/.skeema.login . - -CMD cp .skeema.login ./gradido_login/.skeema && skeema push --allow-unsafe && rm ./gradido_login/.skeema - -######################################################################################################### -# Run skeema -######################################################################################################### -FROM skeema_build as skeema_run - -ENV DOCKER_WORKDIR="/skeema" - -RUN mkdir -p ${DOCKER_WORKDIR} -WORKDIR ${DOCKER_WORKDIR} - -COPY ./skeema/.skeema . -COPY ./login_server/skeema/ . -COPY ./mariadb/.skeema.login ./gradido_login/.skeema - -CMD skeema push --allow-unsafe - From 3e726fbe82538425e029d1b47be8e6290ff073f1 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 05:10:45 +0200 Subject: [PATCH 07/19] - changed login server database to community database - adjusted table names in login server queries - removed roles model code since its is unused/empty --- configs/login_server/grd_login.properties | 2 +- .../src/cpp/model/table/AppAccessToken.h | 2 +- login_server/src/cpp/model/table/ElopageBuy.h | 2 +- login_server/src/cpp/model/table/EmailOptIn.h | 2 +- login_server/src/cpp/model/table/Group.h | 2 +- .../src/cpp/model/table/PendingTask.h | 2 +- login_server/src/cpp/model/table/Roles.cpp | 0 login_server/src/cpp/model/table/Roles.h | 60 ------------------- login_server/src/cpp/model/table/User.cpp | 14 +++-- login_server/src/cpp/model/table/User.h | 2 +- login_server/src/cpp/model/table/UserBackup.h | 2 +- login_server/src/cpp/model/table/UserRole.h | 3 +- 12 files changed, 17 insertions(+), 76 deletions(-) delete mode 100644 login_server/src/cpp/model/table/Roles.cpp delete mode 100644 login_server/src/cpp/model/table/Roles.h diff --git a/configs/login_server/grd_login.properties b/configs/login_server/grd_login.properties index 96b0ea928..74424328e 100644 --- a/configs/login_server/grd_login.properties +++ b/configs/login_server/grd_login.properties @@ -17,7 +17,7 @@ phpServer.host = nginx loginServer.path = http://localhost/account loginServer.default_locale = de loginServer.db.host = mariadb -loginServer.db.name = gradido_login +loginServer.db.name = gradido_community loginServer.db.user = root loginServer.db.password = loginServer.db.port = 3306 diff --git a/login_server/src/cpp/model/table/AppAccessToken.h b/login_server/src/cpp/model/table/AppAccessToken.h index bbee0568d..37c92a921 100644 --- a/login_server/src/cpp/model/table/AppAccessToken.h +++ b/login_server/src/cpp/model/table/AppAccessToken.h @@ -19,7 +19,7 @@ namespace model { // generic db operations - const char* getTableName() const { return "app_access_tokens"; } + const char* getTableName() const { return "login_app_access_tokens"; } std::string toString(); inline Poco::UInt64 getCode() const { return mAccessCode; } diff --git a/login_server/src/cpp/model/table/ElopageBuy.h b/login_server/src/cpp/model/table/ElopageBuy.h index 008a11a52..8f10d2dbd 100644 --- a/login_server/src/cpp/model/table/ElopageBuy.h +++ b/login_server/src/cpp/model/table/ElopageBuy.h @@ -34,7 +34,7 @@ namespace model { ElopageBuy(); // generic db operations - const char* getTableName() const { return "elopage_buys"; } + const char* getTableName() const { return "login_elopage_buys"; } std::string toString(); diff --git a/login_server/src/cpp/model/table/EmailOptIn.h b/login_server/src/cpp/model/table/EmailOptIn.h index 9e0fc335a..7430a8000 100644 --- a/login_server/src/cpp/model/table/EmailOptIn.h +++ b/login_server/src/cpp/model/table/EmailOptIn.h @@ -29,7 +29,7 @@ namespace model { // generic db operations - const char* getTableName() const { return "email_opt_in"; } + const char* getTableName() const { return "login_email_opt_in"; } std::string toString(); inline Poco::UInt64 getCode() const { return mEmailVerificationCode; } diff --git a/login_server/src/cpp/model/table/Group.h b/login_server/src/cpp/model/table/Group.h index e998cfb7c..a2a300e3b 100644 --- a/login_server/src/cpp/model/table/Group.h +++ b/login_server/src/cpp/model/table/Group.h @@ -17,7 +17,7 @@ namespace model { Group(GroupTuple userTuple); // generic db operations - const char* getTableName() const { return "groups"; } + const char* getTableName() const { return "login_groups"; } std::string toString(); inline const std::string& getAlias() const { return mAlias; } diff --git a/login_server/src/cpp/model/table/PendingTask.h b/login_server/src/cpp/model/table/PendingTask.h index 3e7a0e8d5..730efcd43 100644 --- a/login_server/src/cpp/model/table/PendingTask.h +++ b/login_server/src/cpp/model/table/PendingTask.h @@ -30,7 +30,7 @@ namespace model { // generic db operations - const char* getTableName() const { return "pending_tasks"; } + const char* getTableName() const { return "login_pending_tasks"; } std::string toString(); //! \brief update table row with current request diff --git a/login_server/src/cpp/model/table/Roles.cpp b/login_server/src/cpp/model/table/Roles.cpp deleted file mode 100644 index e69de29bb..000000000 diff --git a/login_server/src/cpp/model/table/Roles.h b/login_server/src/cpp/model/table/Roles.h deleted file mode 100644 index 01775b58c..000000000 --- a/login_server/src/cpp/model/table/Roles.h +++ /dev/null @@ -1,60 +0,0 @@ -#ifndef GRADIDO_LOGIN_SERVER_MODEL_TABLE_ROLES_INCLUDE -#define GRADIDO_LOGIN_SERVER_MODEL_TABLE_ROLES_INCLUDE - -#include "ModelBase.h" -#include "Poco/Types.h" -#include "Poco/Tuple.h" - -namespace model { - namespace table { - - enum RoleType { - ROLE_ADMIN = 1 - }; - - class Roles : public ModelBase - { - - }; - /* - typedef Poco::Tuple EmailOptInTuple; - - class EmailOptIn : public ModelBase - { - public: - EmailOptIn(const Poco::UInt64& code, int user_id, EmailOptInType type); - EmailOptIn(const Poco::UInt64& code, EmailOptInType type); - EmailOptIn(const EmailOptInTuple& tuple); - EmailOptIn(); - ~EmailOptIn(); - - // generic db operations - const char* getTableName() { return "email_opt_in"; } - std::string toString(); - - inline Poco::UInt64 getCode() const { return mEmailVerificationCode; } - inline int getUserId() const { return mUserId; } - inline EmailOptInType getType() const { return static_cast(mType); } - inline void setCode(Poco::UInt64 code) { mEmailVerificationCode = code; } - inline void setUserId(int user_Id) { mUserId = user_Id; } - - static const char* typeToString(EmailOptInType type); - protected: - Poco::Data::Statement _loadFromDB(Poco::Data::Session session, const std::string& fieldName); - Poco::Data::Statement _loadIdFromDB(Poco::Data::Session session); - Poco::Data::Statement _loadMultipleFromDB(Poco::Data::Session session, const std::string& fieldName); - Poco::Data::Statement _loadFromDB(Poco::Data::Session session, const std::vector& fieldNames, MysqlConditionType conditionType = MYSQL_CONDITION_AND); - Poco::Data::Statement _insertIntoDB(Poco::Data::Session session); - - int mUserId; - // data type must be a multiple of 4 - Poco::UInt64 mEmailVerificationCode; - int mType; - - }; - */ - } -} - - -#endif //GRADIDO_LOGIN_SERVER_MODEL_TABLE_ROLES_INCLUDE \ No newline at end of file diff --git a/login_server/src/cpp/model/table/User.cpp b/login_server/src/cpp/model/table/User.cpp index 6d76b3b9e..38f48ea18 100644 --- a/login_server/src/cpp/model/table/User.cpp +++ b/login_server/src/cpp/model/table/User.cpp @@ -83,11 +83,11 @@ namespace model { if (mPasswordHashed) { - insert << "INSERT INTO users (email, first_name, last_name, username, description, password, email_hash, language, group_id, publisher_id) VALUES(?,?,?,?,?,?,?,?,?,?);", + insert << "INSERT INTO " << getTableName() << " (email, first_name, last_name, username, description, password, email_hash, language, group_id, publisher_id) VALUES(?,?,?,?,?,?,?,?,?,?);", use(mEmail), use(mFirstName), use(mLastName), use(mUsername), use(mDescription), bind(mPasswordHashed), use(mEmailHash), use(mLanguageKey), use(mGroupId), use(mPublisherId); } else { - insert << "INSERT INTO users (email, first_name, last_name, username, description, email_hash, language, group_id, publisher_id) VALUES(?,?,?,?,?,?,?,?,?);", + insert << "INSERT INTO " << getTableName() << " (email, first_name, last_name, username, description, email_hash, language, group_id, publisher_id) VALUES(?,?,?,?,?,?,?,?,?);", use(mEmail), use(mFirstName), use(mLastName), use(mUsername), use(mDescription), use(mEmailHash), use(mLanguageKey), use(mGroupId), use(mPublisherId); } @@ -103,9 +103,11 @@ namespace model { } Poco::Data::Statement select(session); + std::string table_name_user_roles = "login_user_roles" + select << "SELECT " << getTableName() << ".id, email, first_name, last_name, username, description, password, pubkey, privkey, email_hash, created, email_checked, language, disabled, group_id, publisher_id, user_roles.role_id " << " FROM " << getTableName() - << " LEFT JOIN user_roles ON " << getTableName() << ".id = user_roles.user_id " + << " LEFT JOIN " << table_name_user_roles << " ON " << getTableName() << ".id = " << table_name_user_roles << ".user_id " << " WHERE " << _fieldName << " = ?" , into(mID), into(mEmail), into(mFirstName), into(mLastName), into(mUsername), into(mDescription), into(mPasswordHashed), into(mPublicKey), into(mPrivateKey), into(mEmailHash), into(mCreated), into(mEmailChecked), @@ -194,7 +196,7 @@ namespace model { Poco::Data::Statement update(session); - update << "UPDATE users SET password = ?, privkey = ? where id = ?;", + update << "UPDATE " << getTableName() << " SET password = ?, privkey = ? where id = ?;", bind(mPasswordHashed), use(mPrivateKey), use(mID); @@ -221,7 +223,7 @@ namespace model { Poco::Data::Statement update(session); - update << "UPDATE users SET pubkey = ?, privkey = ? where id = ?;", + update << "UPDATE " << getTableName() << " SET pubkey = ?, privkey = ? where id = ?;", use(mPublicKey), use(mPrivateKey), use(mID); @@ -246,7 +248,7 @@ namespace model { auto session = cm->getConnection(CONNECTION_MYSQL_LOGIN_SERVER); Poco::Data::Statement update(session); - update << "UPDATE users SET first_name = ?, last_name = ?, username = ?, description = ?, disabled = ?, language = ?, publisher_id = ? where id = ?;", + update << "UPDATE " << getTableName() << " SET first_name = ?, last_name = ?, username = ?, description = ?, disabled = ?, language = ?, publisher_id = ? where id = ?;", use(mFirstName), use(mLastName), use(mUsername), use(mDescription), use(mDisabled), use(mLanguageKey), use(mPublisherId), use(mID); try { diff --git a/login_server/src/cpp/model/table/User.h b/login_server/src/cpp/model/table/User.h index ac756434b..8d3acf0e2 100644 --- a/login_server/src/cpp/model/table/User.h +++ b/login_server/src/cpp/model/table/User.h @@ -54,7 +54,7 @@ namespace model { // generic db operations - const char* getTableName() const { return "users"; } + const char* getTableName() const { return "login_users"; } std::string toString(); std::string toHTMLString(); diff --git a/login_server/src/cpp/model/table/UserBackup.h b/login_server/src/cpp/model/table/UserBackup.h index a2f407fbd..1a8ec41ad 100644 --- a/login_server/src/cpp/model/table/UserBackup.h +++ b/login_server/src/cpp/model/table/UserBackup.h @@ -17,7 +17,7 @@ namespace model { UserBackup(); // generic db operations - const char* getTableName() const { return "user_backups"; } + const char* getTableName() const { return "login_user_backups"; } std::string toString(); inline int getUserId() const { return mUserId; } diff --git a/login_server/src/cpp/model/table/UserRole.h b/login_server/src/cpp/model/table/UserRole.h index 1369dbe86..30547b4b9 100644 --- a/login_server/src/cpp/model/table/UserRole.h +++ b/login_server/src/cpp/model/table/UserRole.h @@ -4,7 +4,6 @@ #include "ModelBase.h" #include "Poco/Types.h" #include "Poco/Tuple.h" -//#include "Roles.h" namespace model { namespace table { @@ -25,7 +24,7 @@ namespace model { UserRole(); // generic db operations - const char* getTableName() const { return "user_roles"; } + const char* getTableName() const { return "login_user_roles"; } std::string toString(); inline int getUserId() const { return mUserId; } From 02068fa66242794ba42ab961efb410c24051853d Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 05:10:58 +0200 Subject: [PATCH 08/19] fixed typo --- login_server/src/cpp/SingletonManager/EmailManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/login_server/src/cpp/SingletonManager/EmailManager.cpp b/login_server/src/cpp/SingletonManager/EmailManager.cpp index e460d129e..8a48fc8fa 100644 --- a/login_server/src/cpp/SingletonManager/EmailManager.cpp +++ b/login_server/src/cpp/SingletonManager/EmailManager.cpp @@ -58,7 +58,7 @@ bool EmailManager::init(const Poco::Util::LayeredConfiguration& cfg) void EmailManager::addEmail(model::Email* email) { if (mDisableEmail) { std::string dateTimeString = Poco::DateTimeFormatter::format(Poco::DateTime(), "%d.%m.%y %H:%M:%S"); - std::string log_message = dateTimeString + " Email should be sended to: "; + std::string log_message = dateTimeString + " Email should have been sent to: "; auto email_user = email->getUser(); Poco::AutoPtr email_model; if (email_user) { From 7729f44ba13dbe232b9e5ab1094867bdbf525f7c Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 05:13:45 +0200 Subject: [PATCH 09/19] updated backend required database version --- backend/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index 9dd75dce5..65f899b26 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -23,7 +23,7 @@ import schema from './graphql/schema' // TODO implement // import queryComplexity, { simpleEstimator, fieldConfigEstimator } from "graphql-query-complexity"; -const DB_VERSION = '0002-add_settings' +const DB_VERSION = '0004-login_server_data' async function main() { // open mysql connection From 95ab5036262c9633e113427a7cc509d1dc48c835 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 05:25:37 +0200 Subject: [PATCH 10/19] - missing semicolon - missing table name parameter --- login_server/src/cpp/model/table/User.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/login_server/src/cpp/model/table/User.cpp b/login_server/src/cpp/model/table/User.cpp index 38f48ea18..3fca20edd 100644 --- a/login_server/src/cpp/model/table/User.cpp +++ b/login_server/src/cpp/model/table/User.cpp @@ -103,12 +103,13 @@ namespace model { } Poco::Data::Statement select(session); - std::string table_name_user_roles = "login_user_roles" + std::string table_name_user_roles = "login_user_roles"; - select << "SELECT " << getTableName() << ".id, email, first_name, last_name, username, description, password, pubkey, privkey, email_hash, created, email_checked, language, disabled, group_id, publisher_id, user_roles.role_id " + select << "SELECT " << getTableName() << ".id, email, first_name, last_name, username, description, password, pubkey, privkey, email_hash, created, email_checked, language, disabled, group_id, publisher_id, " << table_name_user_roles << ".role_id " << " FROM " << getTableName() - << " LEFT JOIN " << table_name_user_roles << " ON " << getTableName() << ".id = " << table_name_user_roles << ".user_id " - << " WHERE " << _fieldName << " = ?" , + << " LEFT JOIN " << table_name_user_roles + << " ON " << getTableName() << ".id = " << table_name_user_roles << ".user_id " + << " WHERE " << _fieldName << " = ?; " , into(mID), into(mEmail), into(mFirstName), into(mLastName), into(mUsername), into(mDescription), into(mPasswordHashed), into(mPublicKey), into(mPrivateKey), into(mEmailHash), into(mCreated), into(mEmailChecked), into(mLanguageKey), into(mDisabled), into(mGroupId), into(mPublisherId), into(mRole); From dbae29f47dc11abf550e77fe6be4c863b3077ca4 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 05:33:49 +0200 Subject: [PATCH 11/19] more stray queries --- login_server/src/cpp/controller/User.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/login_server/src/cpp/controller/User.cpp b/login_server/src/cpp/controller/User.cpp index 576b2b8c8..e03916bc9 100644 --- a/login_server/src/cpp/controller/User.cpp +++ b/login_server/src/cpp/controller/User.cpp @@ -446,8 +446,9 @@ namespace controller { std::vector> results; int email_checked = 0; int resend_count = 1; - select << "select u.id, v.created from users as u " - << "LEFT JOIN email_opt_in as v ON(u.id = v.user_id) " + std::string table_name_email_opt_in = "login_email_opt_in"; + select << "select u.id, v.created from " << db->getTableName() << " as u " + << "LEFT JOIN " << table_name_email_opt_in << " as v ON(u.id = v.user_id) " << "where u.email_checked = ? " << "AND v.resend_count <= ? " << "ORDER BY u.id, v.created " , @@ -525,8 +526,8 @@ namespace controller { Poco::Data::Statement select(session); std::vector> results; - select << "select id, email from users " - << "where email_hash IS NULL " + select << "select id, email from " << db->getTableName() + << " where email_hash IS NULL " , Poco::Data::Keywords::into(results) ; int result_count = 0; @@ -556,7 +557,7 @@ namespace controller { // update db // reuse connection, I hope it's working Poco::Data::Statement update(session); - update << "UPDATE users set email_hash = ? where id = ?" + update << "UPDATE " << db->getTableName() << " set email_hash = ? where id = ?" , Poco::Data::Keywords::use(updates); int updated_count = 0; try { From 29e56dbfb24b14b8d11b19a9875c390f652891d7 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 05:40:19 +0200 Subject: [PATCH 12/19] initialize model in order to access database name --- login_server/src/cpp/controller/User.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/login_server/src/cpp/controller/User.cpp b/login_server/src/cpp/controller/User.cpp index e03916bc9..e54832427 100644 --- a/login_server/src/cpp/controller/User.cpp +++ b/login_server/src/cpp/controller/User.cpp @@ -439,6 +439,7 @@ namespace controller { return 0; auto cm = ConnectionManager::getInstance(); auto em = ErrorManager::getInstance(); + auto db = new model::table::User(); static const char* function_name = "User::checkIfVerificationEmailsShouldBeResend"; auto session = cm->getConnection(CONNECTION_MYSQL_LOGIN_SERVER); @@ -520,6 +521,7 @@ namespace controller { { auto cm = ConnectionManager::getInstance(); auto em = ErrorManager::getInstance(); + auto db = new model::table::User(); static const char* function_name = "User::addMissingEmailHashes"; auto session = cm->getConnection(CONNECTION_MYSQL_LOGIN_SERVER); From ae3a5ec2f0a3c977754b9072548471ad6d0bf51f Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 06:09:17 +0200 Subject: [PATCH 13/19] typo --- login_server/src/cpp/SingletonManager/SessionManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/login_server/src/cpp/SingletonManager/SessionManager.cpp b/login_server/src/cpp/SingletonManager/SessionManager.cpp index c4039b10d..ea525b248 100644 --- a/login_server/src/cpp/SingletonManager/SessionManager.cpp +++ b/login_server/src/cpp/SingletonManager/SessionManager.cpp @@ -148,7 +148,7 @@ Session* SessionManager::getNewSession(int* handle) mWorkingMutex.tryLock(500); } catch (Poco::TimeoutException &ex) { - printf("[%s] exception timout mutex: %s\n", functionName, ex.displayText().data()); + printf("[%s] exception timeout mutex: %s\n", functionName, ex.displayText().data()); return nullptr; } //mWorkingMutex.lock(); From 976e8ffbedcbea06422b5a435a47533998a82909 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 20:51:20 +0200 Subject: [PATCH 14/19] only migrate login database if available --- database/migrations/0004-login_server_data.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/database/migrations/0004-login_server_data.ts b/database/migrations/0004-login_server_data.ts index 6a6668bd9..dad7d1e34 100644 --- a/database/migrations/0004-login_server_data.ts +++ b/database/migrations/0004-login_server_data.ts @@ -9,9 +9,18 @@ * databases. */ -const LOGIN_SERVER_DB = '`gradido_login`' +const LOGIN_SERVER_DB = 'gradido_login' export async function upgrade(queryFn: (query: string, values?: any[]) => Promise>) { + const loginDatabaseExists = await queryFn(` + SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = '${LOGIN_SERVER_DB}' + `) + if (loginDatabaseExists.length === 0) { + // eslint-disable-next-line no-console + console.log(`Skipping Login Server Database migration - Database ${LOGIN_SERVER_DB} not found`) + return + } + await queryFn(` INSERT INTO \`login_app_access_tokens\` SELECT * FROM ${LOGIN_SERVER_DB}.\`app_access_tokens\`; `) From 5268d6b7b4848e439cc6ed0fc545cdf65d6f8b9a Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 23:17:05 +0200 Subject: [PATCH 15/19] adjust test database name --- configs/login_server/grd_login_test.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/login_server/grd_login_test.properties b/configs/login_server/grd_login_test.properties index 8a269694a..bf1b3d35f 100644 --- a/configs/login_server/grd_login_test.properties +++ b/configs/login_server/grd_login_test.properties @@ -17,7 +17,7 @@ phpServer.host = nginx loginServer.path = http://localhost/account loginServer.default_locale = de loginServer.db.host = mariadb -loginServer.db.name = gradido_login_test +loginServer.db.name = gradido_community loginServer.db.user = root loginServer.db.password = loginServer.db.port = 3306 From 21dc23bfb1e3901744045a9176a88ed996172304 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 23:31:04 +0200 Subject: [PATCH 16/19] migrate database to create database, use test db just in case --- .github/workflows/test.yml | 16 ++++++++++++---- configs/login_server/grd_login_test.properties | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index acd1ac7cc..edcd6753f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -409,10 +409,18 @@ jobs: --health-timeout=3s --health-retries=4 steps: - - name: Debug service - run: echo "$(docker ps)" - - name: Debug container choosing script - run: echo "$(docker container ls | grep mariadb | awk '{ print $1 }')" + # - name: Debug service + # run: echo "$(docker ps)" + #- name: Debug container choosing script + # run: echo "$(docker container ls | grep mariadb | awk '{ print $1 }')" + - name: get mariadb container id + run: echo "::set-output name=id::$(docker container ls | grep mariadb | awk '{ print $1 }')" + id: mariadb_container + - name: get automatic created network + run: echo "::set-output name=id::$(docker network ls | grep github_network | awk '{ print $1 }')" + id: network + - name: Start database migration + run: docker run --network ${{ steps.network.outputs.id }} --name=database --env NODE_ENV=production --env DB_HOST=mariadb --env DB_DATABASE=gradido_community_test -d gradido/database:production_up ########################################################################## # CHECKOUT CODE ########################################################## ########################################################################## diff --git a/configs/login_server/grd_login_test.properties b/configs/login_server/grd_login_test.properties index bf1b3d35f..65aa0c5d0 100644 --- a/configs/login_server/grd_login_test.properties +++ b/configs/login_server/grd_login_test.properties @@ -17,7 +17,7 @@ phpServer.host = nginx loginServer.path = http://localhost/account loginServer.default_locale = de loginServer.db.host = mariadb -loginServer.db.name = gradido_community +loginServer.db.name = gradido_community_test loginServer.db.user = root loginServer.db.password = loginServer.db.port = 3306 From 32b64f4e6d730a8b502b1037402911fb2e304c7b Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 23:44:26 +0200 Subject: [PATCH 17/19] adjusted test seed --- login_server/src/cpp/test/main.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/login_server/src/cpp/test/main.cpp b/login_server/src/cpp/test/main.cpp index b98c86e61..f7f48cff2 100644 --- a/login_server/src/cpp/test/main.cpp +++ b/login_server/src/cpp/test/main.cpp @@ -176,9 +176,9 @@ int load(int argc, char* argv[]) { // clean up and fill db std::string tables[] = { - "groups", - "users", - "user_roles" + "login_groups", + "login_users", + "login_user_roles" }; for (int i = 0; i < 3; i++) { if (runMysql("TRUNCATE " + tables[i])) { @@ -191,7 +191,7 @@ int load(int argc, char* argv[]) { std::stringstream ss; // password = TestP4ssword&H - ss << "INSERT INTO `users` (`id`, `email`, `first_name`, `last_name`, `username`, `password`, `pubkey`, `privkey`, `created`, `email_checked`, `passphrase_shown`, `language`, `disabled`, `group_id`) VALUES " + ss << "INSERT INTO `login_users` (`id`, `email`, `first_name`, `last_name`, `username`, `password`, `pubkey`, `privkey`, `created`, `email_checked`, `passphrase_shown`, `language`, `disabled`, `group_id`) VALUES " << "(1, 'd_schultz32@gmx.de', 'DDD', 'Schultz', 'Diddel', 18242007140018938940, 0x69f2fefd6fa6947a370b9f8d3147f6617cf67416517ce25cb2d63901c666933c, 0x567f3e623a1899d1f8d69190c5799433c134ce0137c0c38cc0347874586d6234a19f2a0b484e6cc1863502e580ae6c17db1131f29a35eba45a46be29c7ee592940a3bd3ad519075fdeed6e368f0eb818, '2020-02-20 16:05:44', 1, 0, 'de', 0, 1), "; // if this isn't the same, some tests will fail, so we update the test data here. @@ -211,7 +211,7 @@ int load(int argc, char* argv[]) { } ss.str(std::string()); - ss << "INSERT INTO `user_roles` (`id`, `user_id`, `role_id`) VALUES" + ss << "INSERT INTO `login_user_roles` (`id`, `user_id`, `role_id`) VALUES" << "(1, 3, 1);"; if (runMysql(ss.str())) { @@ -219,7 +219,7 @@ int load(int argc, char* argv[]) { } ss.str(std::string()); - ss << "INSERT INTO `groups` (`id`, `alias`, `name`, `url`, `description`) VALUES" + ss << "INSERT INTO `login_groups` (`id`, `alias`, `name`, `url`, `description`) VALUES" << "(1, 'gdd1', 'Gradido1', 'gdd1.gradido.com', 'Der erste offizielle Gradido Server (zum Testen)'), " << "(2, 'gdd_test', 'Gradido Test', 'gdd1.gradido.com', 'Testgroup (zum Testen)'); "; if (runMysql(ss.str())) { From 104fd011696924880c132cb68360eb51704c4f7a Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 23:59:38 +0200 Subject: [PATCH 18/19] dont run database container from registry, build it --- .github/workflows/test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index edcd6753f..243692e6d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -419,8 +419,6 @@ jobs: - name: get automatic created network run: echo "::set-output name=id::$(docker network ls | grep github_network | awk '{ print $1 }')" id: network - - name: Start database migration - run: docker run --network ${{ steps.network.outputs.id }} --name=database --env NODE_ENV=production --env DB_HOST=mariadb --env DB_DATABASE=gradido_community_test -d gradido/database:production_up ########################################################################## # CHECKOUT CODE ########################################################## ########################################################################## @@ -428,6 +426,9 @@ jobs: uses: actions/checkout@v2 with: submodules: true + # Database migration + - name: Start database migration + run: docker run --network ${{ steps.network.outputs.id }} --name=database --env NODE_ENV=production --env DB_HOST=mariadb --env DB_DATABASE=gradido_community_test -d --target production_up ./database ########################################################################## # Build Login-Server Test Docker image ################################### ########################################################################## From 2a372f0fb70695a080ae46d87b88eb7781bcfa09 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Fri, 22 Oct 2021 00:05:57 +0200 Subject: [PATCH 19/19] docker build database test up --- .github/workflows/test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 243692e6d..889f07e8e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -428,7 +428,9 @@ jobs: submodules: true # Database migration - name: Start database migration - run: docker run --network ${{ steps.network.outputs.id }} --name=database --env NODE_ENV=production --env DB_HOST=mariadb --env DB_DATABASE=gradido_community_test -d --target production_up ./database + run: | + docker build --target production_up -t "gradido/database:production_up" database/ + docker run --network ${{ steps.network.outputs.id }} --name=database --env NODE_ENV=production --env DB_HOST=mariadb --env DB_DATABASE=gradido_community_test -d gradido/database:production_up ########################################################################## # Build Login-Server Test Docker image ################################### ##########################################################################