From 019fd04d4448e1aee6a0b6ce2db86bf7a7c3884c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Fri, 31 May 2019 18:35:31 +0200 Subject: [PATCH 1/4] Auto-push maintenance-worker image to dockerhub --- scripts/docker_push.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/docker_push.sh b/scripts/docker_push.sh index 1f627cf1a..9aa9a8e2a 100755 --- a/scripts/docker_push.sh +++ b/scripts/docker_push.sh @@ -2,5 +2,7 @@ echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin docker build --build-arg BUILD_COMMIT=$TRAVIS_COMMIT --target production -t humanconnection/nitro-backend:latest $TRAVIS_BUILD_DIR/backend docker build --build-arg BUILD_COMMIT=$TRAVIS_COMMIT --target production -t humanconnection/nitro-web:latest $TRAVIS_BUILD_DIR/webapp +docker build -t humanconnection/nitro-maintenance-worker:latest $TRAVIS_BUILD_DIR/deployment/legacy-migration/maintenance-worker docker push humanconnection/nitro-backend:latest docker push humanconnection/nitro-web:latest +docker push humanconnection/nitro-maintenance-worker:latest From 115022fc2e9cfa9fee5fa9f66d2942fcadb52d3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Fri, 31 May 2019 18:35:58 +0200 Subject: [PATCH 2/4] Decrease split size This might take longer but will require less memory. I guess this might work even on our kubernetes cluster. --- .../maintenance-worker/migration/mongo/export.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh b/deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh index 257bc4c61..339a5bd64 100755 --- a/deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh +++ b/deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh @@ -10,7 +10,7 @@ set +o allexport function export_collection () { "${EXPORT_MONGOEXPORT_BIN}" --db ${MONGODB_DATABASE} --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --collection $1 --collection $1 --out "${EXPORT_PATH}$1.json" mkdir -p ${EXPORT_PATH}splits/$1/ - split -l 1000 -a 3 ${EXPORT_PATH}$1.json ${EXPORT_PATH}splits/$1/ + split -l 100 -a 3 ${EXPORT_PATH}$1.json ${EXPORT_PATH}splits/$1/ } # Delete old export & ensure directory From f9e9aa47b706b8969a777fda61c50a315983c10c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Fri, 31 May 2019 20:14:08 +0200 Subject: [PATCH 3/4] Disable collections without import statement Even if the import statement does literally nothing, this slows down the import. Let's avoid this until we do sth. with the data. --- .../migration/neo4j/import.sh | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh index 08c05c8cc..ac256e3f0 100755 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh @@ -83,17 +83,17 @@ import_collection "contributions" import_collection "shouts" import_collection "comments" -import_collection "emotions" -import_collection "invites" -import_collection "notifications" -import_collection "organizations" -import_collection "pages" -import_collection "projects" -import_collection "settings" -import_collection "status" -import_collection "systemnotifications" -import_collection "userscandos" -import_collection "usersettings" +# import_collection "emotions" +# import_collection "invites" +# import_collection "notifications" +# import_collection "organizations" +# import_collection "pages" +# import_collection "projects" +# import_collection "settings" +# import_collection "status" +# import_collection "systemnotifications" +# import_collection "userscandos" +# import_collection "usersettings" echo "DONE" From df70ae451ce1fdd19754506b10e5bb6854df4f9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Mon, 3 Jun 2019 13:11:12 +0200 Subject: [PATCH 4/4] Parameterize split size @ulfgebhardt that was a really good suggestion, thanks! --- .../legacy-migration/maintenance-worker/migration/mongo/.env | 3 ++- .../maintenance-worker/migration/mongo/export.sh | 2 +- docker-compose.maintenance.yml | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/deployment/legacy-migration/maintenance-worker/migration/mongo/.env b/deployment/legacy-migration/maintenance-worker/migration/mongo/.env index 602f51fc4..4c5f9e18c 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/mongo/.env +++ b/deployment/legacy-migration/maintenance-worker/migration/mongo/.env @@ -12,5 +12,6 @@ # On Windows this resolves to C:\Users\dornhoeschen\AppData\Local\Temp\mongo-export (MinGW) EXPORT_PATH='/tmp/mongo-export/' EXPORT_MONGOEXPORT_BIN='mongoexport' +MONGO_EXPORT_SPLIT_SIZE=100 # On Windows use something like this -# EXPORT_MONGOEXPORT_BIN='C:\Program Files\MongoDB\Server\3.6\bin\mongoexport.exe' \ No newline at end of file +# EXPORT_MONGOEXPORT_BIN='C:\Program Files\MongoDB\Server\3.6\bin\mongoexport.exe' diff --git a/deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh b/deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh index 339a5bd64..abed9b0f5 100755 --- a/deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh +++ b/deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh @@ -10,7 +10,7 @@ set +o allexport function export_collection () { "${EXPORT_MONGOEXPORT_BIN}" --db ${MONGODB_DATABASE} --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --collection $1 --collection $1 --out "${EXPORT_PATH}$1.json" mkdir -p ${EXPORT_PATH}splits/$1/ - split -l 100 -a 3 ${EXPORT_PATH}$1.json ${EXPORT_PATH}splits/$1/ + split -l ${MONGO_EXPORT_SPLIT_SIZE} -a 3 ${EXPORT_PATH}$1.json ${EXPORT_PATH}splits/$1/ } # Delete old export & ensure directory diff --git a/docker-compose.maintenance.yml b/docker-compose.maintenance.yml index 113b4492c..4ff307594 100644 --- a/docker-compose.maintenance.yml +++ b/docker-compose.maintenance.yml @@ -30,6 +30,7 @@ services: - "MONGODB_AUTH_DB=${MONGODB_AUTH_DB}" - "MONGODB_DATABASE=${MONGODB_DATABASE}" - "UPLOADS_DIRECTORY=${UPLOADS_DIRECTORY}" + - "MONGO_EXPORT_SPLIT_SIZE=${MONGO_EXPORT_SPLIT_SIZE}" ports: - 7687:7687 - 7474:7474