diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/.env b/deployment/legacy-migration/maintenance-worker/migration/neo4j/.env new file mode 100644 index 000000000..7691306ac --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/.env @@ -0,0 +1,16 @@ +# Neo4J Settings +# NEO4J_USERNAME='neo4j' +# NEO4J_PASSWORD='letmein' + +# Import Settings +# On Windows this resolves to C:\Users\dornhoeschen\AppData\Local\Temp\mongo-export (MinGW) +IMPORT_PATH='/tmp/mongo-export/' +IMPORT_CHUNK_PATH='/tmp/mongo-export/splits/current-chunk.json' + +IMPORT_CHUNK_PATH_CQL='/tmp/mongo-export/splits/current-chunk.json' +# On Windows this path needs to be windows style since the cypher-shell runs native - note the forward slash +# IMPORT_CHUNK_PATH_CQL='C:/Users/dornhoeschen/AppData/Local/Temp/mongo-export/splits/current-chunk.json' + +IMPORT_CYPHERSHELL_BIN='cypher-shell' +# On Windows use something like this +# IMPORT_CYPHERSHELL_BIN='C:\Program Files\neo4j-community\bin\cypher-shell.bat' \ No newline at end of file diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql index 6b6a09592..62eddf124 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as badge +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as badge MERGE(b:Badge {id: badge._id["$oid"]}) ON CREATE SET b.key = badge.key, diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql index 776811bec..81e73c1b9 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as category +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as category MERGE(c:Category {id: category._id["$oid"]}) ON CREATE SET c.name = category.title, diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql index 234d29d26..1c8eb9397 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as json +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as json MERGE (comment:Comment {id: json._id["$oid"]}) ON CREATE SET diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql index 194ba39fd..e4e148af3 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as post +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as post MERGE (p:Post {id: post._id["$oid"]}) ON CREATE SET p.title = post.title, diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql index 0cd6d9cfc..bf3837b04 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as follow +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as follow MATCH (u1:User {id: follow.userId}), (u2:User {id: follow.foreignId}) MERGE (u1)-[:FOLLOWS]->(u2) ; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh index b7de74782..12cc7ce67 100755 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh @@ -1,17 +1,48 @@ #!/usr/bin/env bash set -e -SECONDS=0 -SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +# import .env config +set -o allexport +source $(dirname "$0")/.env +set +o allexport -echo "MATCH (n) DETACH DELETE n;" | cypher-shell - -for collection in "badges" "categories" "users" "follows" "contributions" "shouts" "comments" -do - for chunk in /tmp/mongo-export/splits/$collection/* +# Import collection function defintion +function import_collection () { + for chunk in ${IMPORT_PATH}splits/$1/* do - mv $chunk /tmp/mongo-export/splits/current-chunk.json - echo "Import ${chunk}" && cypher-shell < $SCRIPT_DIRECTORY/$collection.cql + mv $chunk ${IMPORT_CHUNK_PATH} + NEO4J_COMMAND="$(envsubst '${IMPORT_CHUNK_PATH_CQL}' < $(dirname "$0")/$1.cql)" + echo "Import ${chunk}" + echo "${NEO4J_COMMAND}" | "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} done -done +} + +# Time variable +SECONDS=0 + +# Delete all Neo4J Database content +echo "Deleting Database Contents" +echo "MATCH (n) DETACH DELETE n;" | "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} + +# Import Data +echo "Start Importing Data" +import_collection "badges" +import_collection "categories" +import_collection "users" +import_collection "follows" +import_collection "contributions" +import_collection "shouts" +import_collection "comments" +#import_collection "emotions" +#import_collection "invites" +#import_collection "notifications" +#import_collection "organizations" +#import_collection "pages" +#import_collection "projects" +#import_collection "settings" +#import_collection "status" +#import_collection "systemnotifications" +#import_collection "userscandos" +#import_collection "usersettings" + echo "Time elapsed: $SECONDS seconds" diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql index 5019cdc32..a82a7a33d 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as shout +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as shout MATCH (u:User {id: shout.userId}), (p:Post {id: shout.foreignId}) MERGE (u)-[:SHOUTED]->(p) ; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql index c877f8377..693fd75b6 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as user +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as user MERGE(u:User {id: user._id["$oid"]}) ON CREATE SET u.name = user.name,