From 5492b282bc2e8ff598b9375b003c8473037af6b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Mon, 4 Feb 2019 14:31:40 +0100 Subject: [PATCH] Refactoring: No uploads/ vol,access neo4j directly Digital Ocean does not support shared directories. So we have to uploads the images in `/uploads` via `kubectl cp` or something similar. Likewise, it is not possible to share the exported mongodb .json files with neo4j container. Therefore let's install `cypher-shell`, included in `neo4j` package, to directly open a neo4j connection and bulk import the data. --- db-migration-worker/Dockerfile | 13 +++++++++---- db-migration-worker/migrate.sh | 2 ++ db-migration-worker/{ => mongo}/import.sh | 3 +-- .../import => db-migration-worker/neo4j}/badges.cql | 0 .../neo4j}/categories.cql | 0 .../neo4j}/comments.cql | 0 .../neo4j}/contributions.cql | 0 .../neo4j}/follows.cql | 0 .../import => db-migration-worker/neo4j}/import.sh | 2 ++ .../import => db-migration-worker/neo4j}/shouts.cql | 0 .../import => db-migration-worker/neo4j}/users.cql | 0 docker-compose.override.yml | 4 +--- neo4j/Dockerfile | 1 - neo4j/import/todo | 2 -- 14 files changed, 15 insertions(+), 12 deletions(-) create mode 100644 db-migration-worker/migrate.sh rename db-migration-worker/{ => mongo}/import.sh (93%) rename {neo4j/import => db-migration-worker/neo4j}/badges.cql (100%) rename {neo4j/import => db-migration-worker/neo4j}/categories.cql (100%) rename {neo4j/import => db-migration-worker/neo4j}/comments.cql (100%) rename {neo4j/import => db-migration-worker/neo4j}/contributions.cql (100%) rename {neo4j/import => db-migration-worker/neo4j}/follows.cql (100%) rename {neo4j/import => db-migration-worker/neo4j}/import.sh (97%) rename {neo4j/import => db-migration-worker/neo4j}/shouts.cql (100%) rename {neo4j/import => db-migration-worker/neo4j}/users.cql (100%) delete mode 100644 neo4j/import/todo diff --git a/db-migration-worker/Dockerfile b/db-migration-worker/Dockerfile index d7265ac02..7d6203057 100644 --- a/db-migration-worker/Dockerfile +++ b/db-migration-worker/Dockerfile @@ -1,8 +1,13 @@ +FROM neo4j:3.5.0 as neo4j FROM mongo:4 -RUN apt-get update \ - && apt-get -y install --no-install-recommends openssh-client rsync \ +RUN apt-get update +RUN apt-get -y install --no-install-recommends wget apt-transport-https +RUN wget -O - https://debian.neo4j.org/neotechnology.gpg.key | apt-key add - +RUN echo 'deb https://debian.neo4j.org/repo stable/' | tee /etc/apt/sources.list.d/neo4j.list +RUN apt-get update +RUN apt-get -y install --no-install-recommends openjdk-8-jre openssh-client neo4j rsync \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -COPY import.sh . - +COPY neo4j/ mongo/ ./ +COPY migrate.sh /usr/local/bin/migrate diff --git a/db-migration-worker/migrate.sh b/db-migration-worker/migrate.sh new file mode 100644 index 000000000..0aea5bc89 --- /dev/null +++ b/db-migration-worker/migrate.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +/mongo/import.sh && /neo4j/import.sh diff --git a/db-migration-worker/import.sh b/db-migration-worker/mongo/import.sh similarity index 93% rename from db-migration-worker/import.sh rename to db-migration-worker/mongo/import.sh index 8e949a123..d93900339 100755 --- a/db-migration-worker/import.sh +++ b/db-migration-worker/mongo/import.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +set -e for var in "SSH_USERNAME" "SSH_HOST" "MONGODB_USERNAME" "MONGODB_PASSWORD" "MONGODB_DATABASE" "MONGODB_AUTH_DB" "UPLOADS_DIRECTORY" do @@ -25,8 +26,6 @@ mongodump --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONG ssh -S my-ctrl-socket -O check -l ${SSH_USERNAME} ${SSH_HOST} ssh -S my-ctrl-socket -O exit -l ${SSH_USERNAME} ${SSH_HOST} -rsync --archive --update --verbose ${SSH_USERNAME}@${SSH_HOST}:${UPLOADS_DIRECTORY}/* /uploads/ - for collection in "categories" "badges" "users" "contributions" "comments" "follows" "shouts" do mongoexport --db ${MONGODB_DATABASE} --collection $collection --out "/mongo-export/$collection.json" diff --git a/neo4j/import/badges.cql b/db-migration-worker/neo4j/badges.cql similarity index 100% rename from neo4j/import/badges.cql rename to db-migration-worker/neo4j/badges.cql diff --git a/neo4j/import/categories.cql b/db-migration-worker/neo4j/categories.cql similarity index 100% rename from neo4j/import/categories.cql rename to db-migration-worker/neo4j/categories.cql diff --git a/neo4j/import/comments.cql b/db-migration-worker/neo4j/comments.cql similarity index 100% rename from neo4j/import/comments.cql rename to db-migration-worker/neo4j/comments.cql diff --git a/neo4j/import/contributions.cql b/db-migration-worker/neo4j/contributions.cql similarity index 100% rename from neo4j/import/contributions.cql rename to db-migration-worker/neo4j/contributions.cql diff --git a/neo4j/import/follows.cql b/db-migration-worker/neo4j/follows.cql similarity index 100% rename from neo4j/import/follows.cql rename to db-migration-worker/neo4j/follows.cql diff --git a/neo4j/import/import.sh b/db-migration-worker/neo4j/import.sh similarity index 97% rename from neo4j/import/import.sh rename to db-migration-worker/neo4j/import.sh index 80b6595fc..d5a846384 100755 --- a/neo4j/import/import.sh +++ b/db-migration-worker/neo4j/import.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +set -e + SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" echo "MATCH (n) OPTIONAL MATCH (n)-[r]-() DELETE n,r;" | cypher-shell for collection in "badges" "categories" "users" "follows" "contributions" "shouts" "comments" diff --git a/neo4j/import/shouts.cql b/db-migration-worker/neo4j/shouts.cql similarity index 100% rename from neo4j/import/shouts.cql rename to db-migration-worker/neo4j/shouts.cql diff --git a/neo4j/import/users.cql b/db-migration-worker/neo4j/users.cql similarity index 100% rename from neo4j/import/users.cql rename to db-migration-worker/neo4j/users.cql diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 0592b801b..205d964e0 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -9,7 +9,6 @@ services: volumes: - .:/nitro-backend - /nitro-backend/node_modules - - uploads:/nitro-backend/public/uploads command: yarn run dev neo4j: volumes: @@ -25,11 +24,11 @@ services: context: db-migration-worker volumes: - mongo-export:/mongo-export - - uploads:/uploads - ./db-migration-worker/.ssh/:/root/.ssh/ networks: - hc-network environment: + - NEO4J_URI=bolt://neo4j:7687 - "SSH_USERNAME=${SSH_USERNAME}" - "SSH_HOST=${SSH_HOST}" - "MONGODB_USERNAME=${MONGODB_USERNAME}" @@ -41,4 +40,3 @@ services: volumes: mongo-export: - uploads: diff --git a/neo4j/Dockerfile b/neo4j/Dockerfile index 07344b47b..cb7fd228f 100644 --- a/neo4j/Dockerfile +++ b/neo4j/Dockerfile @@ -1,3 +1,2 @@ FROM neo4j:3.5.0 RUN wget https://github.com/neo4j-contrib/neo4j-apoc-procedures/releases/download/3.5.0.1/apoc-3.5.0.1-all.jar -P plugins/ -COPY import ./import diff --git a/neo4j/import/todo b/neo4j/import/todo deleted file mode 100644 index 139597f9c..000000000 --- a/neo4j/import/todo +++ /dev/null @@ -1,2 +0,0 @@ - -