diff --git a/deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh b/deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh index abed9b0f5..8d16f42fa 100755 --- a/deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh +++ b/deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh @@ -8,11 +8,18 @@ set +o allexport # Export collection function defintion function export_collection () { - "${EXPORT_MONGOEXPORT_BIN}" --db ${MONGODB_DATABASE} --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --collection $1 --collection $1 --out "${EXPORT_PATH}$1.json" + "${EXPORT_MONGOEXPORT_BIN}" --db ${MONGODB_DATABASE} --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --collection $1 --out "${EXPORT_PATH}$1.json" mkdir -p ${EXPORT_PATH}splits/$1/ split -l ${MONGO_EXPORT_SPLIT_SIZE} -a 3 ${EXPORT_PATH}$1.json ${EXPORT_PATH}splits/$1/ } +# Export collection with query function defintion +function export_collection_query () { + "${EXPORT_MONGOEXPORT_BIN}" --db ${MONGODB_DATABASE} --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --collection $1 --out "${EXPORT_PATH}$1_$3.json" --query "$2" + mkdir -p ${EXPORT_PATH}splits/$1_$3/ + split -l ${MONGO_EXPORT_SPLIT_SIZE} -a 3 ${EXPORT_PATH}$1_$3.json ${EXPORT_PATH}splits/$1_$3/ +} + # Delete old export & ensure directory rm -rf ${EXPORT_PATH}* mkdir -p ${EXPORT_PATH} @@ -24,9 +31,12 @@ ssh -4 -M -S my-ctrl-socket -fnNT -L 27018:localhost:27017 -l ${SSH_USERNAME} ${ export_collection "badges" export_collection "categories" export_collection "comments" -export_collection "contributions" +export_collection_query "contributions" "{'type': 'DELETED'}" "DELETED" +export_collection_query "contributions" "{'type': 'post'}" "post" +export_collection_query "contributions" "{'type': 'cando'}" "cando" export_collection "emotions" -export_collection "follows" +export_collection_query "follows" "{'foreignService': 'organizations'}" "organizations" +export_collection_query "follows" "{'foreignService': 'users'}" "users" export_collection "invites" export_collection "notifications" export_collection "organizations" diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges/badges.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/badges/badges.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges/delete.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/badges_delete.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/badges/delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories/categories.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/categories/categories.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories/delete.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/categories_delete.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/categories/delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments/comments.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/comments/comments.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments/delete.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/comments_delete.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/comments/delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions/contributions.cql similarity index 96% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions/contributions.cql index 472354763..a0f1418aa 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions/contributions.cql @@ -109,8 +109,8 @@ } } }, -[?] deleted: { -[X] type: Boolean, +[?] deleted: { // THis field is not always present in the alpha-data +[?] type: Boolean, [ ] default: false, // Default value is missing in Nitro [-] index: true }, @@ -137,7 +137,7 @@ p.contentExcerpt = post.contentExcerpt, p.visibility = toLower(post.visibility), p.createdAt = post.createdAt.`$date`, p.updatedAt = post.updatedAt.`$date`, -p.deleted = post.deleted, +p.deleted = COALESCE(post.deleted,false), p.disabled = NOT post.isEnabled WITH p, post MATCH (u:User {id: post.userId}) diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions/delete.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions_delete.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions/delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/_delete_all.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/delete_all.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/_delete_all.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/delete_all.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions/delete.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions_delete.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions/delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions/emotions.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions/emotions.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows/delete.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/follows_delete.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/follows/delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows/follows.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/follows/follows.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh index ac256e3f0..8eef68c92 100755 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh @@ -9,10 +9,10 @@ set +o allexport # Delete collection function defintion function delete_collection () { # Delete from Database - echo "Delete $1" - "${IMPORT_CYPHERSHELL_BIN}" < $(dirname "$0")/$1_delete.cql > /dev/null + echo "Delete $2" + "${IMPORT_CYPHERSHELL_BIN}" < $(dirname "$0")/$1/delete.cql > /dev/null # Delete index file - rm -f "${IMPORT_PATH}splits/$1.index" + rm -f "${IMPORT_PATH}splits/$2.index" } # Import collection function defintion @@ -34,7 +34,7 @@ function import_collection () { # calculate the path of the chunk export IMPORT_CHUNK_PATH_CQL_FILE="${IMPORT_CHUNK_PATH_CQL}$1/${CHUNK_FILE_NAME}" # load the neo4j command and replace file variable with actual path - NEO4J_COMMAND="$(envsubst '${IMPORT_CHUNK_PATH_CQL_FILE}' < $(dirname "$0")/$1.cql)" + NEO4J_COMMAND="$(envsubst '${IMPORT_CHUNK_PATH_CQL_FILE}' < $(dirname "$0")/$2)" # run the import of the chunk echo "Import $1 ${CHUNK_FILE_NAME} (${chunk})" echo "${NEO4J_COMMAND}" | "${IMPORT_CYPHERSHELL_BIN}" > /dev/null @@ -52,13 +52,14 @@ SECONDS=0 # Delete all Neo4J Database content echo "Deleting Database Contents" -delete_collection "badges" -delete_collection "categories" -delete_collection "users" -delete_collection "follows" -delete_collection "contributions" -delete_collection "shouts" -delete_collection "comments" +delete_collection "badges" "badges" +delete_collection "categories" "categories" +delete_collection "users" "users" +delete_collection "follows" "follows_users" +delete_collection "contributions" "contributions_post" +delete_collection "contributions" "contributions_cando" +delete_collection "shouts" "shouts" +delete_collection "comments" "comments" #delete_collection "emotions" #delete_collection "invites" @@ -75,26 +76,33 @@ echo "DONE" # Import Data echo "Start Importing Data" -import_collection "badges" -import_collection "categories" -import_collection "users" -import_collection "follows" -import_collection "contributions" -import_collection "shouts" -import_collection "comments" +import_collection "badges" "badges/badges.cql" +import_collection "categories" "categories/categories.cql" +import_collection "users" "users/users.cql" +import_collection "follows_users" "follows/follows.cql" +#import_collection "follows_organizations" "follows/follows.cql" +import_collection "contributions_post" "contributions/contributions.cql" +import_collection "contributions_cando" "contributions/contributions.cql" +#import_collection "contributions_DELETED" "contributions/contributions.cql" +import_collection "shouts" "shouts/shouts.cql" +import_collection "comments" "comments/comments.cql" # import_collection "emotions" # import_collection "invites" # import_collection "notifications" # import_collection "organizations" # import_collection "pages" -# import_collection "projects" -# import_collection "settings" -# import_collection "status" # import_collection "systemnotifications" # import_collection "userscandos" # import_collection "usersettings" +# does only contain dummy data +# import_collection "projects" + +# does only contain alpha specifc data +# import_collection "status +# import_collection "settings"" + echo "DONE" echo "Time elapsed: $SECONDS seconds" diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/invites_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/invites/delete.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/invites_delete.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/invites/delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/invites.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/invites/invites.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/invites.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/invites/invites.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications/delete.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications_delete.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications/delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications/notifications.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications/notifications.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations/delete.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations_delete.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations/delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations/organizations.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations/organizations.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/pages_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/pages/delete.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/pages_delete.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/pages/delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/pages.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/pages/pages.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/pages.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/pages/pages.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/projects_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/projects/delete.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/projects_delete.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/projects/delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/projects.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/projects/projects.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/projects.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/projects/projects.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/settings_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/settings/delete.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/settings_delete.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/settings/delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/settings.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/settings/settings.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/settings.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/settings/settings.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts/delete.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts_delete.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts/delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts/shouts.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts/shouts.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/status_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/status/delete.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/status_delete.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/status/delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/status.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/status/status.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/status.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/status/status.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications/delete.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications_delete.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications/delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications/systemnotifications.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications/systemnotifications.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/users_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/users/delete.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/users_delete.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/users/delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/users/users.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/users/users.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/userscandos_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/userscandos/delete.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/userscandos_delete.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/userscandos/delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/userscandos.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/userscandos/userscandos.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/userscandos.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/userscandos/userscandos.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/usersettings_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/usersettings/delete.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/usersettings_delete.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/usersettings/delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/usersettings.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/usersettings/usersettings.cql similarity index 100% rename from deployment/legacy-migration/maintenance-worker/migration/neo4j/usersettings.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/usersettings/usersettings.cql