From c65ac49594e739b1214637b19473ededbaded6f1 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 21 Oct 2021 04:49:19 +0200 Subject: [PATCH] - removed skeema - simplified mariadb docker container & reduced context(build time) - datapoint for docker group into migration - removed unused node docker definitions from docker-compose --- .github/workflows/test.yml | 2 +- .gitignore | 1 - .../setup_db_tables/setup_docker_group.sql | 6 -- database/migrations/0004-login_server_data.ts | 6 ++ deployment/bare_metal/update_all.sh | 2 - deployment/bare_metal/update_db_schemas.sh | 7 --- docker-compose.override.yml | 17 +----- docker-compose.test.yml | 15 +---- docker-compose.yml | 40 +------------- .../Architecture/TechnicalOverview.md | 2 +- login_server/.gitignore | 1 - login_server/README.md | 3 +- mariadb/.skeema | 5 -- mariadb/.skeema.login | 3 - mariadb/Dockerfile | 28 +--------- mariadb/setup_dbs.sql | 7 --- mariadb/setup_dbs_different_user.sh | 55 ------------------- mariadb/setup_test_dbs.sql | 10 ---- skeema/.skeema | 5 -- skeema/Dockerfile | 39 ------------- 20 files changed, 16 insertions(+), 238 deletions(-) delete mode 100644 configs/login_server/setup_db_tables/setup_docker_group.sql delete mode 100755 deployment/bare_metal/update_db_schemas.sh delete mode 100644 mariadb/.skeema delete mode 100644 mariadb/.skeema.login delete mode 100644 mariadb/setup_dbs.sql delete mode 100755 mariadb/setup_dbs_different_user.sh delete mode 100644 mariadb/setup_test_dbs.sql delete mode 100644 skeema/.skeema delete mode 100644 skeema/Dockerfile diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b6fd6c31d..acd1ac7cc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -147,7 +147,7 @@ jobs: ########################################################################## - name: mariadb | Build `test` image run: | - docker build --target mariadb_server_test -t "gradido/mariadb:test" -f ./mariadb/Dockerfile ./ + docker build --target mariadb_server -t "gradido/mariadb:test" -f ./mariadb/Dockerfile ./ docker save "gradido/mariadb:test" > /tmp/mariadb.tar - name: Upload Artifact uses: actions/upload-artifact@v2 diff --git a/.gitignore b/.gitignore index caed320af..c09811443 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ /node_modules/* .vscode messages.pot -.skeema nbproject .metadata /.env diff --git a/configs/login_server/setup_db_tables/setup_docker_group.sql b/configs/login_server/setup_db_tables/setup_docker_group.sql deleted file mode 100644 index 5f1b64c3f..000000000 --- a/configs/login_server/setup_db_tables/setup_docker_group.sql +++ /dev/null @@ -1,6 +0,0 @@ -INSERT INTO `groups` (`id`, `alias`, `name`, `url`, `host`, `home`, `description`) VALUES -(1, 'docker', 'docker gradido group', 'localhost', 'nginx', '/', 'gradido test group for docker and stage2 with blockchain db'); - - - - diff --git a/database/migrations/0004-login_server_data.ts b/database/migrations/0004-login_server_data.ts index 34cba5b6f..6a6668bd9 100644 --- a/database/migrations/0004-login_server_data.ts +++ b/database/migrations/0004-login_server_data.ts @@ -42,6 +42,12 @@ export async function upgrade(queryFn: (query: string, values?: any[]) => Promis await queryFn(` INSERT INTO \`login_users\` SELECT * FROM ${LOGIN_SERVER_DB}.\`users\`; `) + + // TODO clarify if we need this on non docker environment? + await queryFn(` + INSERT IGNORE INTO \`login_groups\` (\`id\`, \`alias\`, \`name\`, \`url\`, \`host\`, \`home\`, \`description\`) VALUES + (1, 'docker', 'docker gradido group', 'localhost', 'nginx', '/', 'gradido test group for docker and stage2 with blockchain db'); + `) } export async function downgrade(queryFn: (query: string, values?: any[]) => Promise>) { diff --git a/deployment/bare_metal/update_all.sh b/deployment/bare_metal/update_all.sh index b065a16d4..e045d3f39 100755 --- a/deployment/bare_metal/update_all.sh +++ b/deployment/bare_metal/update_all.sh @@ -36,8 +36,6 @@ cd $PROJECT_PATH # git checkout -f master git pull cd deployment/bare_metal -echo 'update schemas' >> $UPDATE_HTML -./update_db_schemas.sh echo 'starting with rebuilding login-server
' >> $UPDATE_HTML ./build_and_start_login_server.sh echo 'starting with rebuilding frontend
' >> $UPDATE_HTML diff --git a/deployment/bare_metal/update_db_schemas.sh b/deployment/bare_metal/update_db_schemas.sh deleted file mode 100755 index 80628f35c..000000000 --- a/deployment/bare_metal/update_db_schemas.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# For that to work skeema needed to be installed on system -# in login_server/skeema skeema configuration files need to be there also in the subfolders -# Update DB Schemas (only the schemas, not the data) - -cd ../../login_server/skeema -skeema push --allow-unsafe diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 1fa6c7e23..5783b9fe7 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -136,22 +136,7 @@ services: - external-net volumes: - /sessions - - ######################################################### - ## skeema for updating dbs if changes happend ########### - ######################################################### - skeema: - build: - context: . - dockerfile: ./skeema/Dockerfile - target: skeema_dev_run - depends_on: - - mariadb - networks: - - internal-net - volumes: - - ./login_server/skeema/gradido_login:/skeema/gradido_login - + volumes: frontend_node_modules: backend_node_modules: diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 457faf950..73da574d2 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -9,7 +9,7 @@ services: build: context: . dockerfile: ./mariadb/Dockerfile - target: mariadb_server_test + target: mariadb_server environment: - MARIADB_ALLOW_EMPTY_PASSWORD=1 - MARIADB_USER=root @@ -75,19 +75,6 @@ services: - external-net volumes: - /sessions - - ######################################################### - ## skeema for updating dbs if changes happend ########### - ######################################################### - skeema: - build: - context: . - dockerfile: ./skeema/Dockerfile - target: skeema_run - depends_on: - - mariadb - networks: - - internal-net networks: external-net: diff --git a/docker-compose.yml b/docker-compose.yml index b85d155fe..3ce88715d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,9 +35,8 @@ services: ######################################################### mariadb: build: - context: . - dockerfile: ./mariadb/Dockerfile - target: mariadb_server_test + context: ./mariadb + target: mariadb_server environment: - MARIADB_ALLOW_EMPTY_PASSWORD=1 - MARIADB_USER=root @@ -158,41 +157,6 @@ services: - internal-net volumes: - ./community_server/config/php-fpm/php-ini-overrides.ini:/etc/php/7.4/fpm/conf.d/99-overrides.ini - - ######################################################### - ## skeema for updating dbs if changes happend ########### - ######################################################### - skeema: - build: - context: . - dockerfile: ./skeema/Dockerfile - target: skeema_run - depends_on: - - mariadb - networks: - - internal-net - - ######################################################### - ## GRADIDO NODE v1 ###################################### - ######################################################### - # gradido-node: - # build: - # context: . - # dockerfile: ./gn/docker/deprecated-hedera-node/Dockerfile - # volumes: - # - ${GN_INSTANCE_FOLDER}:/opt/instance - # container_name: ${GN_CONTAINER_NAME} - - ######################################################### - ## GRADIDO NODE test ################################### - ######################################################### - # gradido-node-test: - # build: - # context: . - # dockerfile: ./gn/docker/deprecated-hedera-node/Dockerfile - # container_name: gn-test - # working_dir: /opt/gn/build - # command: ["./unit_tests"] networks: external-net: diff --git a/docu/Concepts/TechnicalRequirements/Architecture/TechnicalOverview.md b/docu/Concepts/TechnicalRequirements/Architecture/TechnicalOverview.md index f0fe1e52a..73f74dbae 100644 --- a/docu/Concepts/TechnicalRequirements/Architecture/TechnicalOverview.md +++ b/docu/Concepts/TechnicalRequirements/Architecture/TechnicalOverview.md @@ -15,7 +15,7 @@ This document describes the technical overview for the Gradido infrastructur. Be ![CommunityServerAPI](../image/CommunityServerAPI.png) -### Database Skeema +### Database Skeema (outdated) ![CommunityDBSkeema](../image/CommunityDBSkeema.png) diff --git a/login_server/.gitignore b/login_server/.gitignore index 1d623d885..ca7f5f440 100644 --- a/login_server/.gitignore +++ b/login_server/.gitignore @@ -5,6 +5,5 @@ src/cpsp/*.h src/cpsp/*.cpp src/cpp/proto/ build*/ -/skeema/gradido_login/insert/crypto_key.sql src/LOCALE/messages.pot diff --git a/login_server/README.md b/login_server/README.md index 73f5deea0..118d0d53c 100644 --- a/login_server/README.md +++ b/login_server/README.md @@ -56,8 +56,7 @@ To update messages.pot run This will be also called by ./scripts/build_debug.sh ## database -Login-Server needs a db to run, it is tested with mariadb -table definitions are found in folder ./skeema/gradido_login +Login-Server needs a db to run, it is tested with mariadb. Currently at least one group must be present in table groups. For example: ```sql diff --git a/mariadb/.skeema b/mariadb/.skeema deleted file mode 100644 index e5525e7b5..000000000 --- a/mariadb/.skeema +++ /dev/null @@ -1,5 +0,0 @@ -[production] -flavor=mariadb:10.5 -host=127.0.0.1 -port=3306 -user=root \ No newline at end of file diff --git a/mariadb/.skeema.login b/mariadb/.skeema.login deleted file mode 100644 index c6431dd8c..000000000 --- a/mariadb/.skeema.login +++ /dev/null @@ -1,3 +0,0 @@ -default-character-set=utf8mb4 -default-collation=utf8mb4_unicode_ci -schema=gradido_login \ No newline at end of file diff --git a/mariadb/Dockerfile b/mariadb/Dockerfile index 6b6b1e9aa..07d2ba368 100644 --- a/mariadb/Dockerfile +++ b/mariadb/Dockerfile @@ -3,29 +3,7 @@ ######################################################################################################### FROM mariadb/server:10.5 as mariadb_server -ENV DOCKER_WORKDIR="/docker-entrypoint-initdb.d" +# ENV DOCKER_WORKDIR="/docker-entrypoint-initdb.d" -RUN mkdir -p ${DOCKER_WORKDIR} -WORKDIR ${DOCKER_WORKDIR} - -# create databases -COPY ./mariadb/setup_dbs.sql a1_setup_dbs.sql -# login server db -COPY ./login_server/skeema/ . -RUN cd ./gradido_login/ && for f in *.sql; do cp -- "$f" "../b1_$f"; sed -i '1i use gradido_login;' "../b1_$f"; done -COPY ./configs/login_server/setup_db_tables ./gradido_login/insert -RUN cd ./gradido_login/insert && for f in *.sql; do cp -- "$f" "../../c1_$f"; sed -i '1i use gradido_login;' "../../c1_$f"; done - -######################################################################################################### -# mariadb server with test dbs -######################################################################################################### -FROM mariadb_server as mariadb_server_test - -# create test databases -COPY ./mariadb/setup_test_dbs.sql a2_setup_dbs.sql - -# login server test db -COPY ./login_server/skeema/ . -RUN cd ./gradido_login/ && for f in *.sql; do cp -- "$f" "../b2_$f"; sed -i '1i use gradido_login_test;' "../b2_$f"; done -COPY ./configs/login_server/setup_db_tables ./gradido_login/insert -RUN cd ./gradido_login/insert && for f in *.sql; do cp -- "$f" "../../c2_$f"; sed -i '1i use gradido_login_test;' "../../c2_$f"; done +# RUN mkdir -p ${DOCKER_WORKDIR} +# WORKDIR ${DOCKER_WORKDIR} diff --git a/mariadb/setup_dbs.sql b/mariadb/setup_dbs.sql deleted file mode 100644 index bf44c2ae6..000000000 --- a/mariadb/setup_dbs.sql +++ /dev/null @@ -1,7 +0,0 @@ -create database gradido_login - DEFAULT CHARACTER SET utf8mb4 - DEFAULT COLLATE utf8mb4_unicode_ci; -create database IF NOT EXISTS _skeema_tmp - DEFAULT CHARACTER SET utf8mb4 - DEFAULT COLLATE utf8mb4_unicode_ci; -FLUSH PRIVILEGES; \ No newline at end of file diff --git a/mariadb/setup_dbs_different_user.sh b/mariadb/setup_dbs_different_user.sh deleted file mode 100755 index 139337cbb..000000000 --- a/mariadb/setup_dbs_different_user.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -COLOR_GREEN="\033[0;32m" -COLOR_YELLOW="\e[33m" -COLOR_NONE="\033[0m" - -LOGIN_DB_USER=gradido_login_live -LOGIN_DB_NAME=gradido_login_live -LOGIN_DB_PASSWD=$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo); - -COMMUNITY_DB_USER=gradido_community_live -COMMUNITY_DB_NAME=gradido_community_live -COMMUNITY_DB_PASSWD=$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo); - -# create table -mysql < .skeema -[production] -flavor=mariadb:10.3.25 -host=127.0.0.1 -port=3306 -user=$LOGIN_DB_USER -EOF -cd gradido_login -sudo cat << EOF > .skeema -default-character-set=utf8mb4 -default-collation=utf8mb4_unicode_ci -schema=$LOGIN_DB_NAME -EOF - -source $HOME/.gvm/scripts/gvm -gvm use go1.14.4 -skeema push -p$LOGIN_DB_PASSWD - -echo -e "${COLOR_YELLOW}Login-Server db password: $LOGIN_DB_PASSWD${COLOR_NONE}" diff --git a/mariadb/setup_test_dbs.sql b/mariadb/setup_test_dbs.sql deleted file mode 100644 index 10326d002..000000000 --- a/mariadb/setup_test_dbs.sql +++ /dev/null @@ -1,10 +0,0 @@ -create database gradido_login_test - DEFAULT CHARACTER SET utf8mb4 - DEFAULT COLLATE utf8mb4_unicode_ci; -create database gradido_community_test - DEFAULT CHARACTER SET utf8mb4 - DEFAULT COLLATE utf8mb4_unicode_ci; -create database IF NOT EXISTS _skeema_tmp - DEFAULT CHARACTER SET utf8mb4 - DEFAULT COLLATE utf8mb4_unicode_ci; -FLUSH PRIVILEGES; \ No newline at end of file diff --git a/skeema/.skeema b/skeema/.skeema deleted file mode 100644 index dc87968d6..000000000 --- a/skeema/.skeema +++ /dev/null @@ -1,5 +0,0 @@ -[production] -flavor=mariadb:10.5 -host=mariadb -port=3306 -user=root \ No newline at end of file diff --git a/skeema/Dockerfile b/skeema/Dockerfile deleted file mode 100644 index bbc95f2d0..000000000 --- a/skeema/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -######################################################################################################### -# Build skeema -######################################################################################################### -FROM golang:1.17.1 as skeema_build -RUN go get -d -v github.com/skeema/skeema -WORKDIR /go/src/github.com/skeema/skeema -RUN go install github.com/skeema/skeema@v1.5.3 - -######################################################################################################### -# Run skeema for dev (dynamic) -######################################################################################################### -FROM skeema_build as skeema_dev_run - -ENV DOCKER_WORKDIR="/skeema" - -RUN mkdir -p ${DOCKER_WORKDIR} -WORKDIR ${DOCKER_WORKDIR} - -COPY ./skeema/.skeema . -COPY ./mariadb/.skeema.login . - -CMD cp .skeema.login ./gradido_login/.skeema && skeema push --allow-unsafe && rm ./gradido_login/.skeema - -######################################################################################################### -# Run skeema -######################################################################################################### -FROM skeema_build as skeema_run - -ENV DOCKER_WORKDIR="/skeema" - -RUN mkdir -p ${DOCKER_WORKDIR} -WORKDIR ${DOCKER_WORKDIR} - -COPY ./skeema/.skeema . -COPY ./login_server/skeema/ . -COPY ./mariadb/.skeema.login ./gradido_login/.skeema - -CMD skeema push --allow-unsafe -