From 497f77ae10c1ac502cf2715b180ea4944b294746 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Wed, 1 May 2019 12:25:28 +0200 Subject: [PATCH 1/7] Breakthrough! Use split+indices for performance @appinteractive thanks for pointing out `split`. You just saved me some days of work to refactor the import statements to use CSV instead of JSON files. @Tirokk when I enter `:schema` in Neo4J web UI, I see the following: ``` :schema Indexes ON :Badge(id) ONLINE ON :Category(id) ONLINE ON :Comment(id) ONLINE ON :Post(id) ONLINE ON :Tag(id) ONLINE ON :User(id) ONLINE No constraints ``` So I temporarily removed the unique constraints on `slug` and added plain indices on `id` for all relevant node types. We cannot omit the `:Label` unfortunately, neo4j does not allow this. So I had to add all indices for all known node labels instead. With indices the import finishes in: ``` Time elapsed: 351 seconds ``` :tada: @appinteractive when I keep the unique indices on slug, I get an error during import that a node with label `:User` and slug `tobias` already exists. Ie. we have unqiue constraint violations in our production data. @mattwr18 @ulfgebhardt @ogerly I started the application on my machine on the production data and it turns out that the index page http://localhost:3000/ takes way to long. Visiting my profile page at http://localhost:3000/profile/5b1693daf850c11207fa6109/robert-schafer is fine, though. Even pagination works. When I visit a post page with not too many comments, the application is fast enough, too: http://localhost:3000/post/5bbf49ebc428ea001c7ca89c/neues-video-format-human-connection-tech-news --- .../maintenance-worker/migration/mongo/import.sh | 7 ++++--- .../maintenance-worker/migration/neo4j/badges.cql | 2 +- .../maintenance-worker/migration/neo4j/categories.cql | 2 +- .../maintenance-worker/migration/neo4j/comments.cql | 3 ++- .../migration/neo4j/contributions.cql | 2 +- .../maintenance-worker/migration/neo4j/follows.cql | 2 +- .../maintenance-worker/migration/neo4j/import.sh | 11 +++++++++-- .../maintenance-worker/migration/neo4j/shouts.cql | 2 +- .../maintenance-worker/migration/neo4j/users.cql | 2 +- 9 files changed, 21 insertions(+), 12 deletions(-) diff --git a/deployment/legacy-migration/maintenance-worker/migration/mongo/import.sh b/deployment/legacy-migration/maintenance-worker/migration/mongo/import.sh index 328560bfc..8958dd2a8 100755 --- a/deployment/legacy-migration/maintenance-worker/migration/mongo/import.sh +++ b/deployment/legacy-migration/maintenance-worker/migration/mongo/import.sh @@ -9,16 +9,17 @@ echo "MONGODB_DATABASE ${MONGODB_DATABASE}" echo "MONGODB_AUTH_DB ${MONGODB_AUTH_DB}" echo "-------------------------------------------------" -[ -z "$SSH_PRIVATE_KEY" ] || create_private_ssh_key_from_env rm -rf /tmp/mongo-export/* -mkdir -p /tmp/mongo-export +mkdir -p /tmp/mongo-export/ ssh -4 -M -S my-ctrl-socket -fnNT -L 27018:localhost:27017 -l ${SSH_USERNAME} ${SSH_HOST} for collection in "categories" "badges" "users" "contributions" "comments" "follows" "shouts" do - mongoexport --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --db ${MONGODB_DATABASE} --collection $collection --out "/tmp/mongo-export/$collection.json" + mongoexport --db ${MONGODB_DATABASE} --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --collection $collection --collection $collection --out "/tmp/mongo-export/$collection.json" + mkdir -p /tmp/mongo-export/splits/$collection/ + split -l 500 /tmp/mongo-export/$collection.json /tmp/mongo-export/splits/$collection/ done ssh -S my-ctrl-socket -O check -l ${SSH_USERNAME} ${SSH_HOST} diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql index f4bf67dda..6b6a09592 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/tmp/mongo-export/badges.json') YIELD value as badge +CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as badge MERGE(b:Badge {id: badge._id["$oid"]}) ON CREATE SET b.key = badge.key, diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql index c22354cbe..776811bec 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/tmp/mongo-export/categories.json') YIELD value as category +CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as category MERGE(c:Category {id: category._id["$oid"]}) ON CREATE SET c.name = category.title, diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql index eb645108a..234d29d26 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql @@ -1,4 +1,5 @@ -CALL apoc.load.json('file:/tmp/mongo-export/comments.json') YIELD value as json +CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as json + MERGE (comment:Comment {id: json._id["$oid"]}) ON CREATE SET comment.content = json.content, diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql index 134c276cf..01647f7fb 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/tmp/mongo-export/contributions.json') YIELD value as post +CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as post MERGE (p:Post {id: post._id["$oid"]}) ON CREATE SET p.title = post.title, diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql index 6f5416723..0cd6d9cfc 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/tmp/mongo-export/follows.json') YIELD value as follow +CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as follow MATCH (u1:User {id: follow.userId}), (u2:User {id: follow.foreignId}) MERGE (u1)-[:FOLLOWS]->(u2) ; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh index 6f539c501..ed033c1b9 100755 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh @@ -2,8 +2,15 @@ set -e SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -echo "MATCH (n) OPTIONAL MATCH (n)-[r]-() DELETE n,r;" | cypher-shell -a $NEO4J_URI +echo "MATCH (n) DETACH DELETE n;" | cypher-shell + +SECONDS=0 for collection in "badges" "categories" "users" "follows" "contributions" "shouts" "comments" do - echo "Import ${collection}..." && cypher-shell -a $NEO4J_URI < $SCRIPT_DIRECTORY/$collection.cql + for chunk in /tmp/mongo-export/splits/$collection/* + do + mv $chunk /tmp/mongo-export/splits/current-chunk.json + echo "Import ${chunk}" && cypher-shell < $SCRIPT_DIRECTORY/$collection.cql + done done +echo "Time elapsed: $SECONDS seconds" diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql index cd72ab66b..5019cdc32 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/tmp/mongo-export/shouts.json') YIELD value as shout +CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as shout MATCH (u:User {id: shout.userId}), (p:Post {id: shout.foreignId}) MERGE (u)-[:SHOUTED]->(p) ; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql index 22eb46882..c877f8377 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/tmp/mongo-export/users.json') YIELD value as user +CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as user MERGE(u:User {id: user._id["$oid"]}) ON CREATE SET u.name = user.name, From b5d91cffef244956951882ef42770238d7c09260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Tue, 7 May 2019 19:14:57 +0200 Subject: [PATCH 2/7] Implement @appinteractive's suggestions This: https://github.com/Human-Connection/Human-Connection/pull/529#discussion_r280065855 --- .../maintenance-worker/migration/mongo/import.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/legacy-migration/maintenance-worker/migration/mongo/import.sh b/deployment/legacy-migration/maintenance-worker/migration/mongo/import.sh index 8958dd2a8..d68a8c2a8 100755 --- a/deployment/legacy-migration/maintenance-worker/migration/mongo/import.sh +++ b/deployment/legacy-migration/maintenance-worker/migration/mongo/import.sh @@ -19,7 +19,7 @@ for collection in "categories" "badges" "users" "contributions" "comments" "foll do mongoexport --db ${MONGODB_DATABASE} --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --collection $collection --collection $collection --out "/tmp/mongo-export/$collection.json" mkdir -p /tmp/mongo-export/splits/$collection/ - split -l 500 /tmp/mongo-export/$collection.json /tmp/mongo-export/splits/$collection/ + split -l 1000 -a 3 /tmp/mongo-export/$collection.json /tmp/mongo-export/splits/$collection/ done ssh -S my-ctrl-socket -O check -l ${SSH_USERNAME} ${SSH_HOST} From 7c139bed1a297c8b587d77be2c5ba26ffba4b055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Fri, 3 May 2019 17:05:37 +0200 Subject: [PATCH 3/7] Remove obsolete binary to create RSA keys We can supply files on kubernetes through secrets --- .../binaries/create_private_ssh_key_from_env | 6 ------ .../maintenance-worker/binaries/import_legacy_uploads | 1 - .../legacy-migration/maintenance-worker/docker-compose.yml | 3 +-- 3 files changed, 1 insertion(+), 9 deletions(-) delete mode 100755 deployment/legacy-migration/maintenance-worker/binaries/create_private_ssh_key_from_env diff --git a/deployment/legacy-migration/maintenance-worker/binaries/create_private_ssh_key_from_env b/deployment/legacy-migration/maintenance-worker/binaries/create_private_ssh_key_from_env deleted file mode 100755 index f44671978..000000000 --- a/deployment/legacy-migration/maintenance-worker/binaries/create_private_ssh_key_from_env +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -e - -mkdir -p ~/.ssh -echo $SSH_PRIVATE_KEY | base64 -d > ~/.ssh/id_rsa -chmod 600 ~/.ssh/id_rsa diff --git a/deployment/legacy-migration/maintenance-worker/binaries/import_legacy_uploads b/deployment/legacy-migration/maintenance-worker/binaries/import_legacy_uploads index 24ae0fca5..d24936e3b 100755 --- a/deployment/legacy-migration/maintenance-worker/binaries/import_legacy_uploads +++ b/deployment/legacy-migration/maintenance-worker/binaries/import_legacy_uploads @@ -9,5 +9,4 @@ do fi done -[ -z "$SSH_PRIVATE_KEY" ] || create_private_ssh_key_from_env rsync --archive --update --verbose ${SSH_USERNAME}@${SSH_HOST}:${UPLOADS_DIRECTORY}/* /uploads/ diff --git a/deployment/legacy-migration/maintenance-worker/docker-compose.yml b/deployment/legacy-migration/maintenance-worker/docker-compose.yml index a45a5163a..03b545982 100644 --- a/deployment/legacy-migration/maintenance-worker/docker-compose.yml +++ b/deployment/legacy-migration/maintenance-worker/docker-compose.yml @@ -9,6 +9,7 @@ services: - uploads:/uploads - neo4j-data:/data - ./migration/:/migration + - ./ssh/:/root/.ssh networks: - hc-network environment: @@ -19,12 +20,10 @@ services: - MOCK=false - MAPBOX_TOKEN=pk.eyJ1IjoiaHVtYW4tY29ubmVjdGlvbiIsImEiOiJjajl0cnBubGoweTVlM3VwZ2lzNTNud3ZtIn0.KZ8KK9l70omjXbEkkbHGsQ - PRIVATE_KEY_PASSPHRASE=a7dsf78sadg87ad87sfagsadg78 - - NEO4J_URI=bolt://localhost:7687 - NEO4J_apoc_import_file_enabled=true - NEO4J_AUTH=none - "SSH_USERNAME=${SSH_USERNAME}" - "SSH_HOST=${SSH_HOST}" - - "SSH_PRIVATE_KEY=${SSH_PRIVATE_KEY}" - "MONGODB_USERNAME=${MONGODB_USERNAME}" - "MONGODB_PASSWORD=${MONGODB_PASSWORD}" - "MONGODB_AUTH_DB=${MONGODB_AUTH_DB}" From 5771efc9205cd638568bc73b35c35e5c552efb5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Fri, 3 May 2019 17:09:45 +0200 Subject: [PATCH 4/7] Add binary `idle` to keep container spinning --- deployment/legacy-migration/maintenance-worker/binaries/idle | 2 ++ .../legacy-migration/maintenance-worker/docker-compose.yml | 1 + 2 files changed, 3 insertions(+) create mode 100755 deployment/legacy-migration/maintenance-worker/binaries/idle diff --git a/deployment/legacy-migration/maintenance-worker/binaries/idle b/deployment/legacy-migration/maintenance-worker/binaries/idle new file mode 100755 index 000000000..f5b1b2454 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/binaries/idle @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +tail -f /dev/null diff --git a/deployment/legacy-migration/maintenance-worker/docker-compose.yml b/deployment/legacy-migration/maintenance-worker/docker-compose.yml index 03b545982..a2e4ebc9e 100644 --- a/deployment/legacy-migration/maintenance-worker/docker-compose.yml +++ b/deployment/legacy-migration/maintenance-worker/docker-compose.yml @@ -2,6 +2,7 @@ version: "3.4" services: maintenance: + command: "idle" # remove this line to start neo4j automatically image: humanconnection/maintenance-worker:latest build: context: . From 13906c29aa738206a8b0836007fa178902ec2a94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Tue, 7 May 2019 19:20:58 +0200 Subject: [PATCH 5/7] Setup unique constraints on ids This will * speed up data import from production * make sure we never have the same ids on certain labels --- neo4j/migrate.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/neo4j/migrate.sh b/neo4j/migrate.sh index 1ec5212ad..f52539555 100755 --- a/neo4j/migrate.sh +++ b/neo4j/migrate.sh @@ -4,13 +4,22 @@ # the initial default user. Before we can create constraints, we have to change # the default password. This is a security feature of neo4j. if echo ":exit" | cypher-shell --password neo4j 2> /dev/null ; then - echo "CALL dbms.security.changePassword('${NEO4J_PASSWORD}');" | cypher-shell --password neo4j + neo4j-admin set-initial-password $NEO4J_PASSWORD fi set -e echo ' CALL db.index.fulltext.createNodeIndex("full_text_search",["Post"],["title", "content"]); +CREATE CONSTRAINT ON (p:Post) ASSERT p.id IS UNIQUE; +CREATE CONSTRAINT ON (c:Comment) ASSERT c.id IS UNIQUE; +CREATE CONSTRAINT ON (c:Category) ASSERT c.id IS UNIQUE; +CREATE CONSTRAINT ON (u:User) ASSERT u.id IS UNIQUE; +CREATE CONSTRAINT ON (o:Organization) ASSERT o.id IS UNIQUE; +CREATE CONSTRAINT ON (t:Tag) ASSERT t.id IS UNIQUE; +CREATE CONSTRAINT ON (c:Category) ASSERT c.id IS UNIQUE; + + CREATE CONSTRAINT ON (p:Post) ASSERT p.slug IS UNIQUE; CREATE CONSTRAINT ON (c:Category) ASSERT c.slug IS UNIQUE; CREATE CONSTRAINT ON (u:User) ASSERT u.slug IS UNIQUE; From 099a5c9cf48d139407253abc2082bdef79b3b05b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Tue, 7 May 2019 19:24:59 +0200 Subject: [PATCH 6/7] Remove obsolete docker-compose files Since we moved to a monorepo, we have it in the top level folder. --- backend/docker-compose.prod.yml | 9 --------- webapp/docker-compose.override.yml | 16 ---------------- webapp/docker-compose.travis.yml | 10 ---------- webapp/docker-compose.yml | 23 ----------------------- 4 files changed, 58 deletions(-) delete mode 100644 backend/docker-compose.prod.yml delete mode 100644 webapp/docker-compose.override.yml delete mode 100644 webapp/docker-compose.travis.yml delete mode 100644 webapp/docker-compose.yml diff --git a/backend/docker-compose.prod.yml b/backend/docker-compose.prod.yml deleted file mode 100644 index c4f5dc4f5..000000000 --- a/backend/docker-compose.prod.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: "3.7" - -services: - neo4j: - environment: - - NEO4J_PASSWORD=letmein - backend: - environment: - - NEO4J_PASSWORD=letmein diff --git a/webapp/docker-compose.override.yml b/webapp/docker-compose.override.yml deleted file mode 100644 index 6edc22f25..000000000 --- a/webapp/docker-compose.override.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: '3.7' - -services: - webapp: - build: - context: . - target: build-and-test - volumes: - - .:/nitro-web - - node_modules:/nitro-web/node_modules - - nuxt:/nitro-web/.nuxt - command: yarn run dev - -volumes: - node_modules: - nuxt: diff --git a/webapp/docker-compose.travis.yml b/webapp/docker-compose.travis.yml deleted file mode 100644 index 0f02ea27d..000000000 --- a/webapp/docker-compose.travis.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: "3.7" - -services: - webapp: - build: - context: . - target: build-and-test - environment: - - GRAPHQL_URI=http://backend:4123 - - NODE_ENV=test diff --git a/webapp/docker-compose.yml b/webapp/docker-compose.yml deleted file mode 100644 index f20fdf9f9..000000000 --- a/webapp/docker-compose.yml +++ /dev/null @@ -1,23 +0,0 @@ -version: '3.7' - -services: - webapp: - image: humanconnection/nitro-web:latest - build: - context: . - target: production - ports: - - 3000:3000 - networks: - - hc-network - environment: - - HOST=0.0.0.0 - - GRAPHQL_URI=http://backend:4000 - - MAPBOX_TOKEN="pk.eyJ1IjoiaHVtYW4tY29ubmVjdGlvbiIsImEiOiJjajl0cnBubGoweTVlM3VwZ2lzNTNud3ZtIn0.bZ8KK9l70omjXbEkkbHGsQ" - -networks: - hc-network: - name: hc-network - -volumes: - node_modules: From bcc2c4dbbb260a81ae2a7958d9ce9d9f38bff9c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Tue, 7 May 2019 19:32:35 +0200 Subject: [PATCH 7/7] Configure scripts and docker-compose.yml After endless try/error I found the way to share volumes between multiple docker-compose.ymls: You have to place those files in the same folder. Also the import scripts must be adapted. --- .../binaries/import_legacy_db | 2 +- .../migration/neo4j/import.sh | 3 ++- ...pose.yml => docker-compose.maintenance.yml | 20 ++++++++++--------- docker-compose.override.yml | 2 ++ neo4j/migrate.sh | 10 ++++++++-- 5 files changed, 24 insertions(+), 13 deletions(-) rename deployment/legacy-migration/maintenance-worker/docker-compose.yml => docker-compose.maintenance.yml (74%) diff --git a/deployment/legacy-migration/maintenance-worker/binaries/import_legacy_db b/deployment/legacy-migration/maintenance-worker/binaries/import_legacy_db index 214da53d8..233798527 100755 --- a/deployment/legacy-migration/maintenance-worker/binaries/import_legacy_db +++ b/deployment/legacy-migration/maintenance-worker/binaries/import_legacy_db @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -e -for var in "SSH_USERNAME" "SSH_HOST" "MONGODB_USERNAME" "MONGODB_PASSWORD" "MONGODB_DATABASE" "MONGODB_AUTH_DB" "NEO4J_URI" +for var in "SSH_USERNAME" "SSH_HOST" "MONGODB_USERNAME" "MONGODB_PASSWORD" "MONGODB_DATABASE" "MONGODB_AUTH_DB" do if [[ -z "${!var}" ]]; then echo "${var} is undefined" diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh index ed033c1b9..b7de74782 100755 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash set -e +SECONDS=0 SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + echo "MATCH (n) DETACH DELETE n;" | cypher-shell -SECONDS=0 for collection in "badges" "categories" "users" "follows" "contributions" "shouts" "comments" do for chunk in /tmp/mongo-export/splits/$collection/* diff --git a/deployment/legacy-migration/maintenance-worker/docker-compose.yml b/docker-compose.maintenance.yml similarity index 74% rename from deployment/legacy-migration/maintenance-worker/docker-compose.yml rename to docker-compose.maintenance.yml index a2e4ebc9e..113b4492c 100644 --- a/deployment/legacy-migration/maintenance-worker/docker-compose.yml +++ b/docker-compose.maintenance.yml @@ -2,18 +2,19 @@ version: "3.4" services: maintenance: - command: "idle" # remove this line to start neo4j automatically image: humanconnection/maintenance-worker:latest build: - context: . + context: deployment/legacy-migration/maintenance-worker volumes: - uploads:/uploads - neo4j-data:/data - - ./migration/:/migration - - ./ssh/:/root/.ssh + - ./deployment/legacy-migration/maintenance-worker/migration/:/migration + - ./deployment/legacy-migration/maintenance-worker/ssh/:/root/.ssh networks: - hc-network environment: + - NEO4J_dbms_security_auth__enabled=false + - NEO4J_dbms_memory_heap_max__size=2G - GRAPHQL_PORT=4000 - GRAPHQL_URI=http://localhost:4000 - CLIENT_URI=http://localhost:3000 @@ -22,7 +23,6 @@ services: - MAPBOX_TOKEN=pk.eyJ1IjoiaHVtYW4tY29ubmVjdGlvbiIsImEiOiJjajl0cnBubGoweTVlM3VwZ2lzNTNud3ZtIn0.KZ8KK9l70omjXbEkkbHGsQ - PRIVATE_KEY_PASSPHRASE=a7dsf78sadg87ad87sfagsadg78 - NEO4J_apoc_import_file_enabled=true - - NEO4J_AUTH=none - "SSH_USERNAME=${SSH_USERNAME}" - "SSH_HOST=${SSH_HOST}" - "MONGODB_USERNAME=${MONGODB_USERNAME}" @@ -34,9 +34,11 @@ services: - 7687:7687 - 7474:7474 -volumes: - uploads: - neo4j-data: - networks: hc-network: + +volumes: + webapp_node_modules: + backend_node_modules: + neo4j-data: + uploads: diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 26dc68619..a71418229 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -18,6 +18,7 @@ services: volumes: - ./backend:/nitro-backend - backend_node_modules:/nitro-backend/node_modules + - uploads:/nitro-backend/public/uploads command: yarn run dev neo4j: environment: @@ -32,3 +33,4 @@ volumes: webapp_node_modules: backend_node_modules: neo4j-data: + uploads: diff --git a/neo4j/migrate.sh b/neo4j/migrate.sh index f52539555..6f3361b8a 100755 --- a/neo4j/migrate.sh +++ b/neo4j/migrate.sh @@ -4,7 +4,11 @@ # the initial default user. Before we can create constraints, we have to change # the default password. This is a security feature of neo4j. if echo ":exit" | cypher-shell --password neo4j 2> /dev/null ; then - neo4j-admin set-initial-password $NEO4J_PASSWORD + if [[ -z "${NEO4J_PASSWORD}" ]]; then + echo "NEO4J_PASSWORD environment variable is undefined. I cannot set the initial password." + else + echo "CALL dbms.security.changePassword('${NEO4J_PASSWORD}');" | cypher-shell --password neo4j + fi fi set -e @@ -17,7 +21,6 @@ CREATE CONSTRAINT ON (c:Category) ASSERT c.id IS UNIQUE; CREATE CONSTRAINT ON (u:User) ASSERT u.id IS UNIQUE; CREATE CONSTRAINT ON (o:Organization) ASSERT o.id IS UNIQUE; CREATE CONSTRAINT ON (t:Tag) ASSERT t.id IS UNIQUE; -CREATE CONSTRAINT ON (c:Category) ASSERT c.id IS UNIQUE; CREATE CONSTRAINT ON (p:Post) ASSERT p.slug IS UNIQUE; @@ -25,3 +28,6 @@ CREATE CONSTRAINT ON (c:Category) ASSERT c.slug IS UNIQUE; CREATE CONSTRAINT ON (u:User) ASSERT u.slug IS UNIQUE; CREATE CONSTRAINT ON (o:Organization) ASSERT o.slug IS UNIQUE; ' | cypher-shell + +echo "Successfully created all indices and unique constraints:" +echo 'CALL db.indexes();' | cypher-shell