Merge pull request #529 from Human-Connection/split_data_import

Performant import of production data

backend/docker-compose.prod.yml

@@ -1,9 +0,0 @@
-version: "3.7"
-
-services:
-  neo4j:
-    environment:
-      - NEO4J_PASSWORD=letmein
-  backend:
-    environment:
-      - NEO4J_PASSWORD=letmein

deployment/legacy-migration/maintenance-worker/binaries/create_private_ssh_key_from_env

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-mkdir -p ~/.ssh
-echo $SSH_PRIVATE_KEY | base64 -d > ~/.ssh/id_rsa
-chmod 600 ~/.ssh/id_rsa

deployment/legacy-migration/maintenance-worker/binaries/idle

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+tail -f /dev/null

deployment/legacy-migration/maintenance-worker/binaries/import_legacy_db

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 set -e
-for var in "SSH_USERNAME" "SSH_HOST" "MONGODB_USERNAME" "MONGODB_PASSWORD" "MONGODB_DATABASE" "MONGODB_AUTH_DB" "NEO4J_URI"
+for var in "SSH_USERNAME" "SSH_HOST" "MONGODB_USERNAME" "MONGODB_PASSWORD" "MONGODB_DATABASE" "MONGODB_AUTH_DB"
 do
   if [[ -z "${!var}" ]]; then
     echo "${var} is undefined"

deployment/legacy-migration/maintenance-worker/binaries/import_legacy_uploads

@@ -9,5 +9,4 @@ do
   fi
 done
 
-[ -z "$SSH_PRIVATE_KEY" ] || create_private_ssh_key_from_env
 rsync --archive --update --verbose ${SSH_USERNAME}@${SSH_HOST}:${UPLOADS_DIRECTORY}/* /uploads/

deployment/legacy-migration/maintenance-worker/migration/mongo/import.sh

@@ -9,16 +9,17 @@ echo "MONGODB_DATABASE         ${MONGODB_DATABASE}"
 echo "MONGODB_AUTH_DB          ${MONGODB_AUTH_DB}"
 echo "-------------------------------------------------"
 
-[ -z "$SSH_PRIVATE_KEY" ] || create_private_ssh_key_from_env
 
 rm -rf /tmp/mongo-export/*
-mkdir -p /tmp/mongo-export
+mkdir -p /tmp/mongo-export/
 
 ssh -4 -M -S my-ctrl-socket -fnNT -L 27018:localhost:27017 -l ${SSH_USERNAME} ${SSH_HOST}
 
 for collection in "categories" "badges" "users" "contributions" "comments" "follows" "shouts"
 do
-  mongoexport --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --db ${MONGODB_DATABASE} --collection $collection --out "/tmp/mongo-export/$collection.json"
+  mongoexport --db ${MONGODB_DATABASE} --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --collection $collection  --collection $collection --out "/tmp/mongo-export/$collection.json"
+  mkdir -p /tmp/mongo-export/splits/$collection/
+  split -l 1000 -a 3 /tmp/mongo-export/$collection.json /tmp/mongo-export/splits/$collection/
 done
 
 ssh -S my-ctrl-socket -O check -l ${SSH_USERNAME} ${SSH_HOST}

deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql

@@ -1,4 +1,4 @@
-CALL apoc.load.json('file:/tmp/mongo-export/badges.json') YIELD value as badge
+CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as badge
 MERGE(b:Badge {id: badge._id["$oid"]})
 ON CREATE SET
 b.key       = badge.key,

deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql

@@ -1,4 +1,4 @@
-CALL apoc.load.json('file:/tmp/mongo-export/categories.json') YIELD value as category
+CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as category
 MERGE(c:Category {id: category._id["$oid"]})
 ON CREATE SET
 c.name      = category.title,

deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql

@@ -1,4 +1,5 @@
-CALL apoc.load.json('file:/tmp/mongo-export/comments.json') YIELD value as json
+CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as json
+
 MERGE (comment:Comment {id: json._id["$oid"]})
 ON CREATE SET
 comment.content        = json.content,

deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql

@@ -1,4 +1,4 @@
-CALL apoc.load.json('file:/tmp/mongo-export/contributions.json') YIELD value as post
+CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as post
 MERGE (p:Post {id: post._id["$oid"]})
 ON CREATE SET
 p.title          = post.title,

deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql

@@ -1,4 +1,4 @@
-CALL apoc.load.json('file:/tmp/mongo-export/follows.json') YIELD value as follow
+CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as follow
 MATCH (u1:User {id: follow.userId}), (u2:User {id: follow.foreignId})
 MERGE (u1)-[:FOLLOWS]->(u2)
 ;

deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh

@@ -1,9 +1,17 @@
 #!/usr/bin/env bash
 set -e
 
+SECONDS=0
 SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-echo "MATCH (n) OPTIONAL MATCH (n)-[r]-() DELETE n,r;" | cypher-shell -a $NEO4J_URI
+
+echo "MATCH (n) DETACH DELETE n;" | cypher-shell
+
 for collection in "badges" "categories" "users" "follows" "contributions" "shouts" "comments"
 do
-  echo "Import ${collection}..." && cypher-shell -a $NEO4J_URI < $SCRIPT_DIRECTORY/$collection.cql
+  for chunk in /tmp/mongo-export/splits/$collection/*
+  do
+    mv $chunk /tmp/mongo-export/splits/current-chunk.json
+    echo "Import ${chunk}" && cypher-shell < $SCRIPT_DIRECTORY/$collection.cql
+  done
 done
+echo "Time elapsed: $SECONDS seconds"

deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql

@@ -1,4 +1,4 @@
-CALL apoc.load.json('file:/tmp/mongo-export/shouts.json') YIELD value as shout
+CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as shout
 MATCH (u:User {id: shout.userId}), (p:Post {id: shout.foreignId})
 MERGE (u)-[:SHOUTED]->(p)
 ;

deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql

@@ -1,4 +1,4 @@
-CALL apoc.load.json('file:/tmp/mongo-export/users.json') YIELD value as user
+CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as user
 MERGE(u:User {id: user._id["$oid"]})
 ON CREATE SET
 u.name       = user.name,

docker-compose.maintenance.yml

@@ -4,14 +4,17 @@ services:
   maintenance:
     image: humanconnection/maintenance-worker:latest
     build:
-      context: .
+      context: deployment/legacy-migration/maintenance-worker
     volumes:
       - uploads:/uploads
       - neo4j-data:/data
-      - ./migration/:/migration
+      - ./deployment/legacy-migration/maintenance-worker/migration/:/migration
+      - ./deployment/legacy-migration/maintenance-worker/ssh/:/root/.ssh
     networks:
       - hc-network
     environment:
+      - NEO4J_dbms_security_auth__enabled=false
+      - NEO4J_dbms_memory_heap_max__size=2G
       - GRAPHQL_PORT=4000
       - GRAPHQL_URI=http://localhost:4000
       - CLIENT_URI=http://localhost:3000
@@ -19,12 +22,9 @@ services:
       - MOCK=false
       - MAPBOX_TOKEN=pk.eyJ1IjoiaHVtYW4tY29ubmVjdGlvbiIsImEiOiJjajl0cnBubGoweTVlM3VwZ2lzNTNud3ZtIn0.KZ8KK9l70omjXbEkkbHGsQ
       - PRIVATE_KEY_PASSPHRASE=a7dsf78sadg87ad87sfagsadg78
-      - NEO4J_URI=bolt://localhost:7687
       - NEO4J_apoc_import_file_enabled=true
-      - NEO4J_AUTH=none
       - "SSH_USERNAME=${SSH_USERNAME}"
       - "SSH_HOST=${SSH_HOST}"
-      - "SSH_PRIVATE_KEY=${SSH_PRIVATE_KEY}"
       - "MONGODB_USERNAME=${MONGODB_USERNAME}"
       - "MONGODB_PASSWORD=${MONGODB_PASSWORD}"
       - "MONGODB_AUTH_DB=${MONGODB_AUTH_DB}"
@@ -34,9 +34,11 @@ services:
       - 7687:7687
       - 7474:7474
 
-volumes:
-  uploads:
-  neo4j-data:
-
 networks:
   hc-network:
+
+volumes:
+  webapp_node_modules:
+  backend_node_modules:
+  neo4j-data:
+  uploads:

docker-compose.override.yml

@@ -18,6 +18,7 @@ services:
     volumes:
       - ./backend:/nitro-backend
       - backend_node_modules:/nitro-backend/node_modules
+      - uploads:/nitro-backend/public/uploads
     command: yarn run dev
   neo4j:
     environment:
@@ -32,3 +33,4 @@ volumes:
   webapp_node_modules:
   backend_node_modules:
   neo4j-data:
+  uploads:

neo4j/migrate.sh

@@ -4,15 +4,30 @@
 # the initial default user. Before we can create constraints, we have to change
 # the default password. This is a security feature of neo4j.
 if echo ":exit" | cypher-shell  --password neo4j 2> /dev/null ; then
+  if [[ -z "${NEO4J_PASSWORD}" ]]; then
+    echo "NEO4J_PASSWORD environment variable is undefined. I cannot set the initial password."
+  else
     echo "CALL dbms.security.changePassword('${NEO4J_PASSWORD}');" | cypher-shell --password neo4j
+  fi
 fi
 
 set -e
 
 echo '
 CALL db.index.fulltext.createNodeIndex("full_text_search",["Post"],["title", "content"]);
+CREATE CONSTRAINT ON (p:Post)          ASSERT p.id IS UNIQUE;
+CREATE CONSTRAINT ON (c:Comment)       ASSERT c.id IS UNIQUE;
+CREATE CONSTRAINT ON (c:Category)      ASSERT c.id IS UNIQUE;
+CREATE CONSTRAINT ON (u:User)          ASSERT u.id IS UNIQUE;
+CREATE CONSTRAINT ON (o:Organization)  ASSERT o.id IS UNIQUE;
+CREATE CONSTRAINT ON (t:Tag)           ASSERT t.id IS UNIQUE;
+
+
 CREATE CONSTRAINT ON (p:Post)          ASSERT p.slug IS UNIQUE;
 CREATE CONSTRAINT ON (c:Category)      ASSERT c.slug IS UNIQUE;
 CREATE CONSTRAINT ON (u:User)          ASSERT u.slug IS UNIQUE;
 CREATE CONSTRAINT ON (o:Organization)  ASSERT o.slug IS UNIQUE;
 '  | cypher-shell
+
+echo "Successfully created all indices and unique constraints:"
+echo 'CALL db.indexes();' | cypher-shell

webapp/docker-compose.override.yml

@@ -1,16 +0,0 @@
-version: '3.7'
-
-services:
-  webapp:
-    build:
-      context: .
-      target: build-and-test
-    volumes:
-      - .:/nitro-web
-      - node_modules:/nitro-web/node_modules
-      - nuxt:/nitro-web/.nuxt
-    command: yarn run dev
-
-volumes:
-  node_modules:
-  nuxt:

webapp/docker-compose.travis.yml

@@ -1,10 +0,0 @@
-version: "3.7"
-
-services:
-  webapp:
-    build:
-      context: .
-      target: build-and-test
-    environment:
-      - GRAPHQL_URI=http://backend:4123
-      - NODE_ENV=test

webapp/docker-compose.yml

@@ -1,23 +0,0 @@
-version: '3.7'
-
-services:
-  webapp:
-    image: humanconnection/nitro-web:latest
-    build:
-      context: .
-      target: production
-    ports:
-      - 3000:3000
-    networks:
-      - hc-network
-    environment:
-      - HOST=0.0.0.0
-      - GRAPHQL_URI=http://backend:4000
-      - MAPBOX_TOKEN="pk.eyJ1IjoiaHVtYW4tY29ubmVjdGlvbiIsImEiOiJjajl0cnBubGoweTVlM3VwZ2lzNTNud3ZtIn0.bZ8KK9l70omjXbEkkbHGsQ"
-
-networks:
-  hc-network:
-    name: hc-network
-
-volumes:
-  node_modules: