Merge pull request #694 from Human-Connection/2019/kw22/alpha_data_export

🍰 2019/kw22/alpha_data_export

deployment/legacy-migration/maintenance-worker/binaries/import_legacy_db

@@ -8,5 +8,5 @@ do
   fi
 done
 
-/migration/mongo/import.sh
+/migration/mongo/export.sh
 /migration/neo4j/import.sh

deployment/legacy-migration/maintenance-worker/migration/mongo/.env

@@ -0,0 +1,16 @@
+# SSH Access
+# SSH_USERNAME='username'
+# SSH_HOST='example.org'
+
+# Mongo DB on Remote Maschine
+# MONGODB_USERNAME='mongouser'
+# MONGODB_PASSWORD='mongopassword'
+# MONGODB_DATABASE='mongodatabase'
+# MONGODB_AUTH_DB='admin'
+
+# Export Settings
+# On Windows this resolves to C:\Users\dornhoeschen\AppData\Local\Temp\mongo-export (MinGW)
+EXPORT_PATH='/tmp/mongo-export/'
+EXPORT_MONGOEXPORT_BIN='mongoexport'
+# On Windows use something like this
+# EXPORT_MONGOEXPORT_BIN='C:\Program Files\MongoDB\Server\3.6\bin\mongoexport.exe'

deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -e
+
+# import .env config
+set -o allexport
+source $(dirname "$0")/.env
+set +o allexport
+
+# Export collection function defintion
+function export_collection () {
+  "${EXPORT_MONGOEXPORT_BIN}" --db ${MONGODB_DATABASE} --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --collection $1  --collection $1 --out "${EXPORT_PATH}$1.json"
+  mkdir -p ${EXPORT_PATH}splits/$1/
+  split -l 1000 -a 3 ${EXPORT_PATH}$1.json ${EXPORT_PATH}splits/$1/
+}
+
+# Delete old export & ensure directory
+rm -rf ${EXPORT_PATH}*
+mkdir -p ${EXPORT_PATH}
+
+# Open SSH Tunnel
+ssh -4 -M -S my-ctrl-socket -fnNT -L 27018:localhost:27017 -l ${SSH_USERNAME} ${SSH_HOST}
+
+# Export all Data from the Alpha to json and split them up
+export_collection "badges"
+export_collection "categories"
+export_collection "comments"
+export_collection "contributions"
+export_collection "emotions"
+export_collection "follows"
+export_collection "invites"
+export_collection "notifications"
+export_collection "organizations"
+export_collection "pages"
+export_collection "projects"
+export_collection "settings"
+export_collection "shouts"
+export_collection "status"
+export_collection "systemnotifications"
+export_collection "users"
+export_collection "userscandos"
+export_collection "usersettings"
+
+# Close SSH Tunnel
+ssh -S my-ctrl-socket -O check -l ${SSH_USERNAME} ${SSH_HOST}
+ssh -S my-ctrl-socket -O exit  -l ${SSH_USERNAME} ${SSH_HOST}

deployment/legacy-migration/maintenance-worker/migration/mongo/import.sh

@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-echo "SSH_USERNAME             ${SSH_USERNAME}"
-echo "SSH_HOST                 ${SSH_HOST}"
-echo "MONGODB_USERNAME         ${MONGODB_USERNAME}"
-echo "MONGODB_PASSWORD         ${MONGODB_PASSWORD}"
-echo "MONGODB_DATABASE         ${MONGODB_DATABASE}"
-echo "MONGODB_AUTH_DB          ${MONGODB_AUTH_DB}"
-echo "-------------------------------------------------"
-
-
-rm -rf /tmp/mongo-export/*
-mkdir -p /tmp/mongo-export/
-
-ssh -4 -M -S my-ctrl-socket -fnNT -L 27018:localhost:27017 -l ${SSH_USERNAME} ${SSH_HOST}
-
-for collection in "categories" "badges" "users" "contributions" "comments" "follows" "shouts"
-do
-  mongoexport --db ${MONGODB_DATABASE} --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --collection $collection  --collection $collection --out "/tmp/mongo-export/$collection.json"
-  mkdir -p /tmp/mongo-export/splits/$collection/
-  split -l 1000 -a 3 /tmp/mongo-export/$collection.json /tmp/mongo-export/splits/$collection/
-done
-
-ssh -S my-ctrl-socket -O check -l ${SSH_USERNAME} ${SSH_HOST}
-ssh -S my-ctrl-socket -O exit  -l ${SSH_USERNAME} ${SSH_HOST}