Merge pull request #697 from Human-Connection/2019/kw22/alpha_data_import

🍰 2019/kw22/alpha_data_import

deployment/legacy-migration/maintenance-worker/migration/neo4j/.env

@@ -0,0 +1,16 @@
+# Neo4J Settings
+# NEO4J_USERNAME='neo4j'
+# NEO4J_PASSWORD='letmein'
+
+# Import Settings
+# On Windows this resolves to C:\Users\dornhoeschen\AppData\Local\Temp\mongo-export (MinGW)
+IMPORT_PATH='/tmp/mongo-export/'
+IMPORT_CHUNK_PATH='/tmp/mongo-export/splits/current-chunk.json'
+
+IMPORT_CHUNK_PATH_CQL='/tmp/mongo-export/splits/current-chunk.json'
+# On Windows this path needs to be windows style since the cypher-shell runs native - note the forward slash
+# IMPORT_CHUNK_PATH_CQL='C:/Users/dornhoeschen/AppData/Local/Temp/mongo-export/splits/current-chunk.json'
+
+IMPORT_CYPHERSHELL_BIN='cypher-shell'
+# On Windows use something like this
+# IMPORT_CYPHERSHELL_BIN='C:\Program Files\neo4j-community\bin\cypher-shell.bat'

deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql

@@ -1,4 +1,4 @@
-CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as badge
+CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as badge
 MERGE(b:Badge {id: badge._id["$oid"]})
 ON CREATE SET
 b.key       = badge.key,

deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql

@@ -1,4 +1,4 @@
-CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as category
+CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as category
 MERGE(c:Category {id: category._id["$oid"]})
 ON CREATE SET
 c.name      = category.title,

deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql

@@ -1,4 +1,4 @@
-CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as json
+CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as json
 
 MERGE (comment:Comment {id: json._id["$oid"]})
 ON CREATE SET

deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql

@@ -1,4 +1,4 @@
-CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as post
+CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as post
 MERGE (p:Post {id: post._id["$oid"]})
 ON CREATE SET
 p.title          = post.title,

deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql

@@ -1,4 +1,4 @@
-CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as follow
+CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as follow
 MATCH (u1:User {id: follow.userId}), (u2:User {id: follow.foreignId})
 MERGE (u1)-[:FOLLOWS]->(u2)
 ;

deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh

@@ -1,17 +1,48 @@
 #!/usr/bin/env bash
 set -e
 
-SECONDS=0
-SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+# import .env config
+set -o allexport
+source $(dirname "$0")/.env
+set +o allexport
 
-echo "MATCH (n) DETACH DELETE n;" | cypher-shell
-
-for collection in "badges" "categories" "users" "follows" "contributions" "shouts" "comments"
-do
-  for chunk in /tmp/mongo-export/splits/$collection/*
+# Import collection function defintion
+function import_collection () {
+  for chunk in ${IMPORT_PATH}splits/$1/*
   do
-    mv $chunk /tmp/mongo-export/splits/current-chunk.json
-    echo "Import ${chunk}" && cypher-shell < $SCRIPT_DIRECTORY/$collection.cql
+    mv $chunk ${IMPORT_CHUNK_PATH}
+    NEO4J_COMMAND="$(envsubst '${IMPORT_CHUNK_PATH_CQL}' < $(dirname "$0")/$1.cql)"
+    echo "Import ${chunk}"
+    echo "${NEO4J_COMMAND}" | "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD}
   done
-done
+}
+
+# Time variable
+SECONDS=0
+
+# Delete all Neo4J Database content
+echo "Deleting Database Contents"
+echo "MATCH (n) DETACH DELETE n;" | "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD}
+
+# Import Data
+echo "Start Importing Data"
+import_collection "badges"
+import_collection "categories"
+import_collection "users"
+import_collection "follows"
+import_collection "contributions"
+import_collection "shouts"
+import_collection "comments"
+#import_collection "emotions"
+#import_collection "invites"
+#import_collection "notifications"
+#import_collection "organizations"
+#import_collection "pages"
+#import_collection "projects"
+#import_collection "settings"
+#import_collection "status"
+#import_collection "systemnotifications"
+#import_collection "userscandos"
+#import_collection "usersettings"
+
 echo "Time elapsed: $SECONDS seconds"

deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql

@@ -1,4 +1,4 @@
-CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as shout
+CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as shout
 MATCH (u:User {id: shout.userId}), (p:Post {id: shout.foreignId})
 MERGE (u)-[:SHOUTED]->(p)
 ;

deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql

@@ -1,4 +1,4 @@
-CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as user
+CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as user
 MERGE(u:User {id: user._id["$oid"]})
 ON CREATE SET
 u.name       = user.name,