From 301e7fa60cb0879df6b35f7fa015cdf1bfda3d11 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Tue, 28 May 2019 17:03:44 +0200 Subject: [PATCH 1/3] fixed data export for alpha: - include all collections - refactored mongodb export script - renamed to export - use of .env file for (additional) configurations / configuration overrides --- .../binaries/import_legacy_db | 2 +- .../migration/mongo/export.sh | 45 +++++++++++++++++++ .../migration/mongo/import.sh | 26 ----------- 3 files changed, 46 insertions(+), 27 deletions(-) create mode 100644 deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh delete mode 100755 deployment/legacy-migration/maintenance-worker/migration/mongo/import.sh diff --git a/deployment/legacy-migration/maintenance-worker/binaries/import_legacy_db b/deployment/legacy-migration/maintenance-worker/binaries/import_legacy_db index 233798527..6ffdf8e3f 100755 --- a/deployment/legacy-migration/maintenance-worker/binaries/import_legacy_db +++ b/deployment/legacy-migration/maintenance-worker/binaries/import_legacy_db @@ -8,5 +8,5 @@ do fi done -/migration/mongo/import.sh +/migration/mongo/export.sh /migration/neo4j/import.sh diff --git a/deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh b/deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh new file mode 100644 index 000000000..257bc4c61 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -e + +# import .env config +set -o allexport +source $(dirname "$0")/.env +set +o allexport + +# Export collection function defintion +function export_collection () { + "${EXPORT_MONGOEXPORT_BIN}" --db ${MONGODB_DATABASE} --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --collection $1 --collection $1 --out "${EXPORT_PATH}$1.json" + mkdir -p ${EXPORT_PATH}splits/$1/ + split -l 1000 -a 3 ${EXPORT_PATH}$1.json ${EXPORT_PATH}splits/$1/ +} + +# Delete old export & ensure directory +rm -rf ${EXPORT_PATH}* +mkdir -p ${EXPORT_PATH} + +# Open SSH Tunnel +ssh -4 -M -S my-ctrl-socket -fnNT -L 27018:localhost:27017 -l ${SSH_USERNAME} ${SSH_HOST} + +# Export all Data from the Alpha to json and split them up +export_collection "badges" +export_collection "categories" +export_collection "comments" +export_collection "contributions" +export_collection "emotions" +export_collection "follows" +export_collection "invites" +export_collection "notifications" +export_collection "organizations" +export_collection "pages" +export_collection "projects" +export_collection "settings" +export_collection "shouts" +export_collection "status" +export_collection "systemnotifications" +export_collection "users" +export_collection "userscandos" +export_collection "usersettings" + +# Close SSH Tunnel +ssh -S my-ctrl-socket -O check -l ${SSH_USERNAME} ${SSH_HOST} +ssh -S my-ctrl-socket -O exit -l ${SSH_USERNAME} ${SSH_HOST} diff --git a/deployment/legacy-migration/maintenance-worker/migration/mongo/import.sh b/deployment/legacy-migration/maintenance-worker/migration/mongo/import.sh deleted file mode 100755 index d68a8c2a8..000000000 --- a/deployment/legacy-migration/maintenance-worker/migration/mongo/import.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -set -e - -echo "SSH_USERNAME ${SSH_USERNAME}" -echo "SSH_HOST ${SSH_HOST}" -echo "MONGODB_USERNAME ${MONGODB_USERNAME}" -echo "MONGODB_PASSWORD ${MONGODB_PASSWORD}" -echo "MONGODB_DATABASE ${MONGODB_DATABASE}" -echo "MONGODB_AUTH_DB ${MONGODB_AUTH_DB}" -echo "-------------------------------------------------" - - -rm -rf /tmp/mongo-export/* -mkdir -p /tmp/mongo-export/ - -ssh -4 -M -S my-ctrl-socket -fnNT -L 27018:localhost:27017 -l ${SSH_USERNAME} ${SSH_HOST} - -for collection in "categories" "badges" "users" "contributions" "comments" "follows" "shouts" -do - mongoexport --db ${MONGODB_DATABASE} --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --collection $collection --collection $collection --out "/tmp/mongo-export/$collection.json" - mkdir -p /tmp/mongo-export/splits/$collection/ - split -l 1000 -a 3 /tmp/mongo-export/$collection.json /tmp/mongo-export/splits/$collection/ -done - -ssh -S my-ctrl-socket -O check -l ${SSH_USERNAME} ${SSH_HOST} -ssh -S my-ctrl-socket -O exit -l ${SSH_USERNAME} ${SSH_HOST} From 4b5138880d928651761adf033917b69fb6ccf3c7 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Tue, 28 May 2019 17:07:54 +0200 Subject: [PATCH 2/3] missing default .env file --- .../maintenance-worker/migration/mongo/.env | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 deployment/legacy-migration/maintenance-worker/migration/mongo/.env diff --git a/deployment/legacy-migration/maintenance-worker/migration/mongo/.env b/deployment/legacy-migration/maintenance-worker/migration/mongo/.env new file mode 100644 index 000000000..602f51fc4 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/mongo/.env @@ -0,0 +1,16 @@ +# SSH Access +# SSH_USERNAME='username' +# SSH_HOST='example.org' + +# Mongo DB on Remote Maschine +# MONGODB_USERNAME='mongouser' +# MONGODB_PASSWORD='mongopassword' +# MONGODB_DATABASE='mongodatabase' +# MONGODB_AUTH_DB='admin' + +# Export Settings +# On Windows this resolves to C:\Users\dornhoeschen\AppData\Local\Temp\mongo-export (MinGW) +EXPORT_PATH='/tmp/mongo-export/' +EXPORT_MONGOEXPORT_BIN='mongoexport' +# On Windows use something like this +# EXPORT_MONGOEXPORT_BIN='C:\Program Files\MongoDB\Server\3.6\bin\mongoexport.exe' \ No newline at end of file From f9ac22e560611e626fcf4b3cc6f1d5f40fecbe5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Tue, 28 May 2019 18:14:21 +0200 Subject: [PATCH 3/3] Add executable UNIX permissions to export script I encourage @ulfgebhardt to run the following command once: ``` SH_USERNAME=ulf SSH_HOST=***** MONGODB_USERNAME='hc-api' MONGODB_PASSWORD=***** MONGODB_DATABASE=hc_api MONGODB_AUTH_DB=admin UPLOADS_DIRECTORY=/data/api/uploads docker-compose -f docker-compose.maintenance.yml up --build ``` Once you're done with everything. You don't have to run docker for development, but this procedure would ensure docker environment works as expected. --- .../legacy-migration/maintenance-worker/migration/mongo/export.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh diff --git a/deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh b/deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh old mode 100644 new mode 100755