From 064e98943dcfc60e5738a21a19c0a3dac32d16cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Mon, 4 Feb 2019 12:37:03 +0100 Subject: [PATCH 01/13] Remove sub-directories in /mongo-export On Digital Ocean a persistent volume has a folder `lost+found` by default. This script stops throws error messages because it can't remove that folder if we don't add `-r` flag. --- db-migration-worker/import.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-migration-worker/import.sh b/db-migration-worker/import.sh index 0251a3582..8e949a123 100755 --- a/db-migration-worker/import.sh +++ b/db-migration-worker/import.sh @@ -18,7 +18,7 @@ echo "UPLOADS_DIRECTORY ${UPLOADS_DIRECTORY}" echo "-------------------------------------------------" mongo ${MONGODB_DATABASE} --eval "db.dropDatabase();" -rm -f /mongo-export/* +rm -rf /mongo-export/* ssh -4 -M -S my-ctrl-socket -fnNT -L 27018:localhost:27017 -l ${SSH_USERNAME} ${SSH_HOST} mongodump --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --gzip --archive | mongorestore --gzip --archive From 20908cd0628ee452086a52deff7dcfd2a584beaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Mon, 4 Feb 2019 12:41:03 +0100 Subject: [PATCH 02/13] Docker and .gitignore entire .ssh folder On kubernetes, the ssh keys are added as a volume from a secret --- db-migration-worker/.dockerignore | 1 + db-migration-worker/.gitignore | 2 +- db-migration-worker/.ssh/.dockerignore | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 db-migration-worker/.dockerignore delete mode 100644 db-migration-worker/.ssh/.dockerignore diff --git a/db-migration-worker/.dockerignore b/db-migration-worker/.dockerignore new file mode 100644 index 000000000..59ba63a8b --- /dev/null +++ b/db-migration-worker/.dockerignore @@ -0,0 +1 @@ +.ssh/ diff --git a/db-migration-worker/.gitignore b/db-migration-worker/.gitignore index 87cb01310..59ba63a8b 100644 --- a/db-migration-worker/.gitignore +++ b/db-migration-worker/.gitignore @@ -1 +1 @@ -.ssh/id_rsa +.ssh/ diff --git a/db-migration-worker/.ssh/.dockerignore b/db-migration-worker/.ssh/.dockerignore deleted file mode 100644 index 87cb01310..000000000 --- a/db-migration-worker/.ssh/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -.ssh/id_rsa From 5492b282bc2e8ff598b9375b003c8473037af6b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Mon, 4 Feb 2019 14:31:40 +0100 Subject: [PATCH 03/13] Refactoring: No uploads/ vol,access neo4j directly Digital Ocean does not support shared directories. So we have to uploads the images in `/uploads` via `kubectl cp` or something similar. Likewise, it is not possible to share the exported mongodb .json files with neo4j container. Therefore let's install `cypher-shell`, included in `neo4j` package, to directly open a neo4j connection and bulk import the data. --- db-migration-worker/Dockerfile | 13 +++++++++---- db-migration-worker/migrate.sh | 2 ++ db-migration-worker/{ => mongo}/import.sh | 3 +-- .../import => db-migration-worker/neo4j}/badges.cql | 0 .../neo4j}/categories.cql | 0 .../neo4j}/comments.cql | 0 .../neo4j}/contributions.cql | 0 .../neo4j}/follows.cql | 0 .../import => db-migration-worker/neo4j}/import.sh | 2 ++ .../import => db-migration-worker/neo4j}/shouts.cql | 0 .../import => db-migration-worker/neo4j}/users.cql | 0 docker-compose.override.yml | 4 +--- neo4j/Dockerfile | 1 - neo4j/import/todo | 2 -- 14 files changed, 15 insertions(+), 12 deletions(-) create mode 100644 db-migration-worker/migrate.sh rename db-migration-worker/{ => mongo}/import.sh (93%) rename {neo4j/import => db-migration-worker/neo4j}/badges.cql (100%) rename {neo4j/import => db-migration-worker/neo4j}/categories.cql (100%) rename {neo4j/import => db-migration-worker/neo4j}/comments.cql (100%) rename {neo4j/import => db-migration-worker/neo4j}/contributions.cql (100%) rename {neo4j/import => db-migration-worker/neo4j}/follows.cql (100%) rename {neo4j/import => db-migration-worker/neo4j}/import.sh (97%) rename {neo4j/import => db-migration-worker/neo4j}/shouts.cql (100%) rename {neo4j/import => db-migration-worker/neo4j}/users.cql (100%) delete mode 100644 neo4j/import/todo diff --git a/db-migration-worker/Dockerfile b/db-migration-worker/Dockerfile index d7265ac02..7d6203057 100644 --- a/db-migration-worker/Dockerfile +++ b/db-migration-worker/Dockerfile @@ -1,8 +1,13 @@ +FROM neo4j:3.5.0 as neo4j FROM mongo:4 -RUN apt-get update \ - && apt-get -y install --no-install-recommends openssh-client rsync \ +RUN apt-get update +RUN apt-get -y install --no-install-recommends wget apt-transport-https +RUN wget -O - https://debian.neo4j.org/neotechnology.gpg.key | apt-key add - +RUN echo 'deb https://debian.neo4j.org/repo stable/' | tee /etc/apt/sources.list.d/neo4j.list +RUN apt-get update +RUN apt-get -y install --no-install-recommends openjdk-8-jre openssh-client neo4j rsync \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -COPY import.sh . - +COPY neo4j/ mongo/ ./ +COPY migrate.sh /usr/local/bin/migrate diff --git a/db-migration-worker/migrate.sh b/db-migration-worker/migrate.sh new file mode 100644 index 000000000..0aea5bc89 --- /dev/null +++ b/db-migration-worker/migrate.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +/mongo/import.sh && /neo4j/import.sh diff --git a/db-migration-worker/import.sh b/db-migration-worker/mongo/import.sh similarity index 93% rename from db-migration-worker/import.sh rename to db-migration-worker/mongo/import.sh index 8e949a123..d93900339 100755 --- a/db-migration-worker/import.sh +++ b/db-migration-worker/mongo/import.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +set -e for var in "SSH_USERNAME" "SSH_HOST" "MONGODB_USERNAME" "MONGODB_PASSWORD" "MONGODB_DATABASE" "MONGODB_AUTH_DB" "UPLOADS_DIRECTORY" do @@ -25,8 +26,6 @@ mongodump --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONG ssh -S my-ctrl-socket -O check -l ${SSH_USERNAME} ${SSH_HOST} ssh -S my-ctrl-socket -O exit -l ${SSH_USERNAME} ${SSH_HOST} -rsync --archive --update --verbose ${SSH_USERNAME}@${SSH_HOST}:${UPLOADS_DIRECTORY}/* /uploads/ - for collection in "categories" "badges" "users" "contributions" "comments" "follows" "shouts" do mongoexport --db ${MONGODB_DATABASE} --collection $collection --out "/mongo-export/$collection.json" diff --git a/neo4j/import/badges.cql b/db-migration-worker/neo4j/badges.cql similarity index 100% rename from neo4j/import/badges.cql rename to db-migration-worker/neo4j/badges.cql diff --git a/neo4j/import/categories.cql b/db-migration-worker/neo4j/categories.cql similarity index 100% rename from neo4j/import/categories.cql rename to db-migration-worker/neo4j/categories.cql diff --git a/neo4j/import/comments.cql b/db-migration-worker/neo4j/comments.cql similarity index 100% rename from neo4j/import/comments.cql rename to db-migration-worker/neo4j/comments.cql diff --git a/neo4j/import/contributions.cql b/db-migration-worker/neo4j/contributions.cql similarity index 100% rename from neo4j/import/contributions.cql rename to db-migration-worker/neo4j/contributions.cql diff --git a/neo4j/import/follows.cql b/db-migration-worker/neo4j/follows.cql similarity index 100% rename from neo4j/import/follows.cql rename to db-migration-worker/neo4j/follows.cql diff --git a/neo4j/import/import.sh b/db-migration-worker/neo4j/import.sh similarity index 97% rename from neo4j/import/import.sh rename to db-migration-worker/neo4j/import.sh index 80b6595fc..d5a846384 100755 --- a/neo4j/import/import.sh +++ b/db-migration-worker/neo4j/import.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +set -e + SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" echo "MATCH (n) OPTIONAL MATCH (n)-[r]-() DELETE n,r;" | cypher-shell for collection in "badges" "categories" "users" "follows" "contributions" "shouts" "comments" diff --git a/neo4j/import/shouts.cql b/db-migration-worker/neo4j/shouts.cql similarity index 100% rename from neo4j/import/shouts.cql rename to db-migration-worker/neo4j/shouts.cql diff --git a/neo4j/import/users.cql b/db-migration-worker/neo4j/users.cql similarity index 100% rename from neo4j/import/users.cql rename to db-migration-worker/neo4j/users.cql diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 0592b801b..205d964e0 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -9,7 +9,6 @@ services: volumes: - .:/nitro-backend - /nitro-backend/node_modules - - uploads:/nitro-backend/public/uploads command: yarn run dev neo4j: volumes: @@ -25,11 +24,11 @@ services: context: db-migration-worker volumes: - mongo-export:/mongo-export - - uploads:/uploads - ./db-migration-worker/.ssh/:/root/.ssh/ networks: - hc-network environment: + - NEO4J_URI=bolt://neo4j:7687 - "SSH_USERNAME=${SSH_USERNAME}" - "SSH_HOST=${SSH_HOST}" - "MONGODB_USERNAME=${MONGODB_USERNAME}" @@ -41,4 +40,3 @@ services: volumes: mongo-export: - uploads: diff --git a/neo4j/Dockerfile b/neo4j/Dockerfile index 07344b47b..cb7fd228f 100644 --- a/neo4j/Dockerfile +++ b/neo4j/Dockerfile @@ -1,3 +1,2 @@ FROM neo4j:3.5.0 RUN wget https://github.com/neo4j-contrib/neo4j-apoc-procedures/releases/download/3.5.0.1/apoc-3.5.0.1-all.jar -P plugins/ -COPY import ./import diff --git a/neo4j/import/todo b/neo4j/import/todo deleted file mode 100644 index 139597f9c..000000000 --- a/neo4j/import/todo +++ /dev/null @@ -1,2 +0,0 @@ - - From cdf92cc6eaa8061703abf321d6bd5ce12d6b2460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Mon, 4 Feb 2019 15:36:39 +0100 Subject: [PATCH 04/13] Remove obsolete kubernetes folder --- kubernetes/README.md | 51 ------------------- kubernetes/backend-deployment.yaml | 41 --------------- kubernetes/backend-service.json | 35 ------------- kubernetes/dashboard-admin.yml | 14 ----- .../neo4j-data-persistentvolumeclaim.yaml | 14 ----- kubernetes/neo4j-deployment.yaml | 40 --------------- kubernetes/neo4j-service.json | 43 ---------------- 7 files changed, 238 deletions(-) delete mode 100644 kubernetes/README.md delete mode 100644 kubernetes/backend-deployment.yaml delete mode 100644 kubernetes/backend-service.json delete mode 100644 kubernetes/dashboard-admin.yml delete mode 100644 kubernetes/neo4j-data-persistentvolumeclaim.yaml delete mode 100644 kubernetes/neo4j-deployment.yaml delete mode 100644 kubernetes/neo4j-service.json diff --git a/kubernetes/README.md b/kubernetes/README.md deleted file mode 100644 index ffbf225b2..000000000 --- a/kubernetes/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Usage with minikube -Assuming you installed the packages git, docker ([all-distributions](https://docs.docker.com/install/)), minikube ([ubuntu]([minikube](https://computingforgeeks.com/how-to-install-minikube-on-ubuntu-18-04/)), [all-distributions](https://kubernetes.io/docs/tasks/tools/install-minikube/)) and virtualbox here... - -First of all start minikube on your machine: -```sh -minikube start -``` - -You can always get an overview and see what's going on with your minikube: -```sh -minikube dashboard -``` - -From now on stay in your favorite work directory. First let's clone the necessary sources: -```sh -git clone https://github.com/Human-Connection/Nitro-Backend.git -git clone https://github.com/Human-Connection/Nitro-Web.git -``` - -Build Docker images, using the Minikube Docker daemon: -```sh -eval $(minikube docker-env) -docker build -t humanconnection/nitro-backend:latest Nitro-Backend/ -docker build -t humanconnection/neo4j:latest -f Nitro-Backend/Dockerfile.neo4j Nitro-Backend/ -``` - -Check that the image is in Minikube’s Docker registry: -```sh -minikube ssh docker images -``` - -Now change into directory Nitro-Backend/kubernetes and create services and deployments: -```sh -cd Nitro-Backend/kubernetes -kubectl create -f neo4j-deployment.yaml,neo4j-data-persistentvolumeclaim.yaml,backend-deployment.yaml,neo4j-service.json,backend-service.json -``` - -You can see the backend in action with: -```sh -minikube service backend -``` - -### Troubleshoot - -1. If you get an error message along th lines of 'The vboxdrv kernel module is not loaded.' - then you have the same issue i had. to solve this you need to install the propper linux kernel host modules package. Here an example for Manjaro: -https://forum.manjaro.org/t/installing-virtualbox-kernel-modules/6999 -2. When you can not start minikube, try also to remove the cluster with `minikube delete` and start again with `minikube start`. Sometimes this fix startup problems of the cluster. -3. Now again you might run into trouble with an error like 'kubectl could not be found on your path.' In this case run the following command: - ```sh - curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.10.0/bin/linux/amd64/kubectl && chmod +x kubectl && sudo cp kubectl /usr/local/bin/ && rm kubectl - ``` diff --git a/kubernetes/backend-deployment.yaml b/kubernetes/backend-deployment.yaml deleted file mode 100644 index f75920f0d..000000000 --- a/kubernetes/backend-deployment.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - annotations: - kompose.cmd: kompose convert -f ../docker-compose.yml - kompose.version: 1.16.0 (0c01309) - creationTimestamp: null - labels: - io.kompose.service: backend - name: backend -spec: - replicas: 1 - strategy: {} - template: - metadata: - creationTimestamp: null - labels: - io.kompose.service: backend - spec: - containers: - - env: - - name: CLIENT_URI - value: http://localhost:3000 - - name: GRAPHQL_PORT - value: "4000" - - name: GRAPHQL_URI - value: http://localhost:4000 - - name: JWT_SECRET - value: b/&&7b78BF&fv/Vd - - name: MOCK - value: "false" - - name: NEO4J_URI - value: bolt://neo4j:7687 - image: humanconnection/nitro-backend:latest - name: backend - ports: - - containerPort: 4000 - resources: {} - imagePullPolicy: IfNotPresent - restartPolicy: Always -status: {} diff --git a/kubernetes/backend-service.json b/kubernetes/backend-service.json deleted file mode 100644 index 28581be45..000000000 --- a/kubernetes/backend-service.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "kind": "Service", - "apiVersion": "v1", - "metadata": { - "name": "backend", - "namespace": "default", - "selfLink": "/api/v1/namespaces/default/services/backend", - "uid": "57749bc1-dea3-11e8-ace2-080027b5d96d", - "resourceVersion": "8321", - "creationTimestamp": "2018-11-02T13:29:36Z", - "labels": { - "io.kompose.service": "backend" - } - }, - "spec": { - "ports": [ - { - "protocol": "TCP", - "port": 4000, - "targetPort": 4000, - "nodePort": 32633 - } - ], - "selector": { - "io.kompose.service": "backend" - }, - "clusterIP": "10.110.129.92", - "type": "LoadBalancer", - "sessionAffinity": "None", - "externalTrafficPolicy": "Cluster" - }, - "status": { - "loadBalancer": {} - } -} diff --git a/kubernetes/dashboard-admin.yml b/kubernetes/dashboard-admin.yml deleted file mode 100644 index 5d76d3902..000000000 --- a/kubernetes/dashboard-admin.yml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: kubernetes-dashboard - labels: - k8s-app: kubernetes-dashboard -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: -- kind: ServiceAccount - name: kubernetes-dashboard - namespace: kube-system diff --git a/kubernetes/neo4j-data-persistentvolumeclaim.yaml b/kubernetes/neo4j-data-persistentvolumeclaim.yaml deleted file mode 100644 index de9c89e2f..000000000 --- a/kubernetes/neo4j-data-persistentvolumeclaim.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - creationTimestamp: null - labels: - io.kompose.service: neo4j-data - name: neo4j-data -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 100Mi -status: {} diff --git a/kubernetes/neo4j-deployment.yaml b/kubernetes/neo4j-deployment.yaml deleted file mode 100644 index 13f4d52e3..000000000 --- a/kubernetes/neo4j-deployment.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - annotations: - kompose.cmd: kompose convert -f ../docker-compose.yml - kompose.version: 1.16.0 (0c01309) - creationTimestamp: null - labels: - io.kompose.service: neo4j - name: neo4j -spec: - replicas: 1 - strategy: - type: Recreate - template: - metadata: - creationTimestamp: null - labels: - io.kompose.service: neo4j - spec: - containers: - - env: - - name: NEO4J_AUTH - value: none - image: humanconnection/neo4j:latest - name: neo4j - ports: - - containerPort: 7687 - - containerPort: 7474 - resources: {} - imagePullPolicy: IfNotPresent - volumeMounts: - - mountPath: /data - name: neo4j-data - restartPolicy: Always - volumes: - - name: neo4j-data - persistentVolumeClaim: - claimName: neo4j-data -status: {} diff --git a/kubernetes/neo4j-service.json b/kubernetes/neo4j-service.json deleted file mode 100644 index 89ac10367..000000000 --- a/kubernetes/neo4j-service.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "kind": "Service", - "apiVersion": "v1", - "metadata": { - "name": "neo4j", - "namespace": "default", - "selfLink": "/api/v1/namespaces/default/services/neo4j", - "uid": "57de6181-dea3-11e8-ace2-080027b5d96d", - "resourceVersion": "8326", - "creationTimestamp": "2018-11-02T13:29:37Z", - "labels": { - "io.kompose.service": "neo4j" - } - }, - "spec": { - "ports": [ - { - "name": "port-1", - "protocol": "TCP", - "port": 7687, - "targetPort": 7687, - "nodePort": 30116 - }, - { - "name": "port-2", - "protocol": "TCP", - "port": 7474, - "targetPort": 7474, - "nodePort": 32658 - } - ], - "selector": { - "io.kompose.service": "neo4j" - }, - "clusterIP": "10.108.175.122", - "type": "LoadBalancer", - "sessionAffinity": "None", - "externalTrafficPolicy": "Cluster" - }, - "status": { - "loadBalancer": {} - } -} From 01a145be29a272ae1cdd8ed05f4a634131cfd79e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Mon, 4 Feb 2019 15:43:21 +0100 Subject: [PATCH 05/13] Container db-migration-worker need no volumes You can simply run `docker-compose exec db-migration-worker migrate` and that's it. Of course you have to provide all the environment variables. --- .dockerignore | 1 - db-migration-worker/Dockerfile | 3 +-- db-migration-worker/migrate.sh | 4 +++- db-migration-worker/{ => migration}/mongo/import.sh | 3 +-- db-migration-worker/{ => migration}/neo4j/badges.cql | 0 .../{ => migration}/neo4j/categories.cql | 0 .../{ => migration}/neo4j/comments.cql | 0 .../{ => migration}/neo4j/contributions.cql | 0 .../{ => migration}/neo4j/follows.cql | 0 db-migration-worker/{ => migration}/neo4j/import.sh | 12 ++++++++++-- db-migration-worker/{ => migration}/neo4j/shouts.cql | 0 db-migration-worker/{ => migration}/neo4j/users.cql | 0 docker-compose.override.yml | 3 ++- 13 files changed, 17 insertions(+), 9 deletions(-) mode change 100644 => 100755 db-migration-worker/migrate.sh rename db-migration-worker/{ => migration}/mongo/import.sh (91%) rename db-migration-worker/{ => migration}/neo4j/badges.cql (100%) rename db-migration-worker/{ => migration}/neo4j/categories.cql (100%) rename db-migration-worker/{ => migration}/neo4j/comments.cql (100%) rename db-migration-worker/{ => migration}/neo4j/contributions.cql (100%) rename db-migration-worker/{ => migration}/neo4j/follows.cql (100%) rename db-migration-worker/{ => migration}/neo4j/import.sh (55%) rename db-migration-worker/{ => migration}/neo4j/shouts.cql (100%) rename db-migration-worker/{ => migration}/neo4j/users.cql (100%) diff --git a/.dockerignore b/.dockerignore index dba77200f..31f5b28f3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,7 +11,6 @@ docker-compose*.yml ./*.png ./*.log -kubernetes/ node_modules/ scripts/ dist/ diff --git a/db-migration-worker/Dockerfile b/db-migration-worker/Dockerfile index 7d6203057..cd6e57258 100644 --- a/db-migration-worker/Dockerfile +++ b/db-migration-worker/Dockerfile @@ -1,4 +1,3 @@ -FROM neo4j:3.5.0 as neo4j FROM mongo:4 RUN apt-get update @@ -9,5 +8,5 @@ RUN apt-get update RUN apt-get -y install --no-install-recommends openjdk-8-jre openssh-client neo4j rsync \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -COPY neo4j/ mongo/ ./ +COPY migration ./migration COPY migrate.sh /usr/local/bin/migrate diff --git a/db-migration-worker/migrate.sh b/db-migration-worker/migrate.sh old mode 100644 new mode 100755 index 0aea5bc89..108453bba --- a/db-migration-worker/migrate.sh +++ b/db-migration-worker/migrate.sh @@ -1,2 +1,4 @@ #!/usr/bin/env bash -/mongo/import.sh && /neo4j/import.sh +set -e +/migration/mongo/import.sh +/migration/neo4j/import.sh diff --git a/db-migration-worker/mongo/import.sh b/db-migration-worker/migration/mongo/import.sh similarity index 91% rename from db-migration-worker/mongo/import.sh rename to db-migration-worker/migration/mongo/import.sh index d93900339..4f59551aa 100755 --- a/db-migration-worker/mongo/import.sh +++ b/db-migration-worker/migration/mongo/import.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -for var in "SSH_USERNAME" "SSH_HOST" "MONGODB_USERNAME" "MONGODB_PASSWORD" "MONGODB_DATABASE" "MONGODB_AUTH_DB" "UPLOADS_DIRECTORY" +for var in "SSH_USERNAME" "SSH_HOST" "MONGODB_USERNAME" "MONGODB_PASSWORD" "MONGODB_DATABASE" "MONGODB_AUTH_DB" do if [[ -z "${!var}" ]]; then echo "${var} is undefined" @@ -15,7 +15,6 @@ echo "MONGODB_USERNAME ${MONGODB_USERNAME}" echo "MONGODB_PASSWORD ${MONGODB_PASSWORD}" echo "MONGODB_DATABASE ${MONGODB_DATABASE}" echo "MONGODB_AUTH_DB ${MONGODB_AUTH_DB}" -echo "UPLOADS_DIRECTORY ${UPLOADS_DIRECTORY}" echo "-------------------------------------------------" mongo ${MONGODB_DATABASE} --eval "db.dropDatabase();" diff --git a/db-migration-worker/neo4j/badges.cql b/db-migration-worker/migration/neo4j/badges.cql similarity index 100% rename from db-migration-worker/neo4j/badges.cql rename to db-migration-worker/migration/neo4j/badges.cql diff --git a/db-migration-worker/neo4j/categories.cql b/db-migration-worker/migration/neo4j/categories.cql similarity index 100% rename from db-migration-worker/neo4j/categories.cql rename to db-migration-worker/migration/neo4j/categories.cql diff --git a/db-migration-worker/neo4j/comments.cql b/db-migration-worker/migration/neo4j/comments.cql similarity index 100% rename from db-migration-worker/neo4j/comments.cql rename to db-migration-worker/migration/neo4j/comments.cql diff --git a/db-migration-worker/neo4j/contributions.cql b/db-migration-worker/migration/neo4j/contributions.cql similarity index 100% rename from db-migration-worker/neo4j/contributions.cql rename to db-migration-worker/migration/neo4j/contributions.cql diff --git a/db-migration-worker/neo4j/follows.cql b/db-migration-worker/migration/neo4j/follows.cql similarity index 100% rename from db-migration-worker/neo4j/follows.cql rename to db-migration-worker/migration/neo4j/follows.cql diff --git a/db-migration-worker/neo4j/import.sh b/db-migration-worker/migration/neo4j/import.sh similarity index 55% rename from db-migration-worker/neo4j/import.sh rename to db-migration-worker/migration/neo4j/import.sh index d5a846384..9a5045d62 100755 --- a/db-migration-worker/neo4j/import.sh +++ b/db-migration-worker/migration/neo4j/import.sh @@ -1,9 +1,17 @@ #!/usr/bin/env bash set -e +for var in "NEO4J_URI" +do + if [[ -z "${!var}" ]]; then + echo "${var} is undefined" + exit 1 + fi +done + SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -echo "MATCH (n) OPTIONAL MATCH (n)-[r]-() DELETE n,r;" | cypher-shell +echo "MATCH (n) OPTIONAL MATCH (n)-[r]-() DELETE n,r;" | cypher-shell -a $NEO4J_URI for collection in "badges" "categories" "users" "follows" "contributions" "shouts" "comments" do - echo "Import ${collection}..." && cypher-shell < $SCRIPT_DIRECTORY/$collection.cql + echo "Import ${collection}..." && cypher-shell -a $NEO4J_URI < $SCRIPT_DIRECTORY/$collection.cql done diff --git a/db-migration-worker/neo4j/shouts.cql b/db-migration-worker/migration/neo4j/shouts.cql similarity index 100% rename from db-migration-worker/neo4j/shouts.cql rename to db-migration-worker/migration/neo4j/shouts.cql diff --git a/db-migration-worker/neo4j/users.cql b/db-migration-worker/migration/neo4j/users.cql similarity index 100% rename from db-migration-worker/neo4j/users.cql rename to db-migration-worker/migration/neo4j/users.cql diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 205d964e0..6bc594099 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -27,6 +27,8 @@ services: - ./db-migration-worker/.ssh/:/root/.ssh/ networks: - hc-network + depends_on: + - backend environment: - NEO4J_URI=bolt://neo4j:7687 - "SSH_USERNAME=${SSH_USERNAME}" @@ -35,7 +37,6 @@ services: - "MONGODB_PASSWORD=${MONGODB_PASSWORD}" - "MONGODB_AUTH_DB=${MONGODB_AUTH_DB}" - "MONGODB_DATABASE=${MONGODB_DATABASE}" - - "UPLOADS_DIRECTORY=${UPLOADS_DIRECTORY}" command: "--smallfiles --logpath=/dev/null" volumes: From 955383f169ee5d6aed2f1ca00ce8dddeedb75436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Mon, 4 Feb 2019 19:19:31 +0100 Subject: [PATCH 06/13] Separate `docker-compose.yml` to test db migration I think it's not that relevant to our new contributors to install the db-migration-worker. --- docker-compose.db-migration.yml | 29 +++++++++++++++++++++++++++++ docker-compose.override.yml | 27 --------------------------- 2 files changed, 29 insertions(+), 27 deletions(-) create mode 100644 docker-compose.db-migration.yml diff --git a/docker-compose.db-migration.yml b/docker-compose.db-migration.yml new file mode 100644 index 000000000..360fbef82 --- /dev/null +++ b/docker-compose.db-migration.yml @@ -0,0 +1,29 @@ +version: "3.7" + +services: + neo4j: + volumes: + - mongo-export:/mongo-export + environment: + - NEO4J_apoc_import_file_enabled=true + db-migration-worker: + build: + context: db-migration-worker + volumes: + - mongo-export:/mongo-export + - ./db-migration-worker/.ssh/:/root/.ssh/ + networks: + - hc-network + depends_on: + - backend + environment: + - NEO4J_URI=bolt://neo4j:7687 + - "SSH_USERNAME=${SSH_USERNAME}" + - "SSH_HOST=${SSH_HOST}" + - "MONGODB_USERNAME=${MONGODB_USERNAME}" + - "MONGODB_PASSWORD=${MONGODB_PASSWORD}" + - "MONGODB_AUTH_DB=${MONGODB_AUTH_DB}" + - "MONGODB_DATABASE=${MONGODB_DATABASE}" + +volumes: + mongo-export: diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 6bc594099..ef7d52c7e 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -11,33 +11,6 @@ services: - /nitro-backend/node_modules command: yarn run dev neo4j: - volumes: - - mongo-export:/mongo-export - - ./neo4j/import:/var/lib/neo4j/import ports: - 7687:7687 - 7474:7474 - environment: - - NEO4J_apoc_import_file_enabled=true - db-migration-worker: - build: - context: db-migration-worker - volumes: - - mongo-export:/mongo-export - - ./db-migration-worker/.ssh/:/root/.ssh/ - networks: - - hc-network - depends_on: - - backend - environment: - - NEO4J_URI=bolt://neo4j:7687 - - "SSH_USERNAME=${SSH_USERNAME}" - - "SSH_HOST=${SSH_HOST}" - - "MONGODB_USERNAME=${MONGODB_USERNAME}" - - "MONGODB_PASSWORD=${MONGODB_PASSWORD}" - - "MONGODB_AUTH_DB=${MONGODB_AUTH_DB}" - - "MONGODB_DATABASE=${MONGODB_DATABASE}" - command: "--smallfiles --logpath=/dev/null" - -volumes: - mongo-export: From 084f610ec8c9a6e06c16fc4ce8f3a252341f9ffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Mon, 4 Feb 2019 19:44:39 +0100 Subject: [PATCH 07/13] Turnaround: *use* volumes for sharing data ... but use it in the same pod. It seems to be possible to have shared volumes in the same pod: https://kubernetes.io/docs/tasks/access-application-cluster/communicate-containers-same-pod-shared-volume/ --- db-migration-worker/Dockerfile | 1 + db-migration-worker/sync_uploads.sh | 12 ++++++++++++ docker-compose.db-migration.yml | 6 ++++++ 3 files changed, 19 insertions(+) create mode 100755 db-migration-worker/sync_uploads.sh diff --git a/db-migration-worker/Dockerfile b/db-migration-worker/Dockerfile index cd6e57258..8a7f8d758 100644 --- a/db-migration-worker/Dockerfile +++ b/db-migration-worker/Dockerfile @@ -10,3 +10,4 @@ RUN apt-get -y install --no-install-recommends openjdk-8-jre openssh-client neo4 && rm -rf /var/lib/apt/lists/* COPY migration ./migration COPY migrate.sh /usr/local/bin/migrate +COPY sync_uploads.sh /usr/local/bin/sync_uploads diff --git a/db-migration-worker/sync_uploads.sh b/db-migration-worker/sync_uploads.sh new file mode 100755 index 000000000..d24936e3b --- /dev/null +++ b/db-migration-worker/sync_uploads.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -e + +for var in "SSH_USERNAME" "SSH_HOST" "UPLOADS_DIRECTORY" +do + if [[ -z "${!var}" ]]; then + echo "${var} is undefined" + exit 1 + fi +done + +rsync --archive --update --verbose ${SSH_USERNAME}@${SSH_HOST}:${UPLOADS_DIRECTORY}/* /uploads/ diff --git a/docker-compose.db-migration.yml b/docker-compose.db-migration.yml index 360fbef82..7d421e968 100644 --- a/docker-compose.db-migration.yml +++ b/docker-compose.db-migration.yml @@ -1,6 +1,9 @@ version: "3.7" services: + backend: + volumes: + - uploads:/nitro-backend/public/uploads neo4j: volumes: - mongo-export:/mongo-export @@ -11,6 +14,7 @@ services: context: db-migration-worker volumes: - mongo-export:/mongo-export + - uploads:/uploads - ./db-migration-worker/.ssh/:/root/.ssh/ networks: - hc-network @@ -24,6 +28,8 @@ services: - "MONGODB_PASSWORD=${MONGODB_PASSWORD}" - "MONGODB_AUTH_DB=${MONGODB_AUTH_DB}" - "MONGODB_DATABASE=${MONGODB_DATABASE}" + - "UPLOADS_DIRECTORY=${UPLOADS_DIRECTORY}" volumes: mongo-export: + uploads: From 2bc1e88920d6de1954d37a5bd2093f9650a92c6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Mon, 4 Feb 2019 19:54:43 +0100 Subject: [PATCH 08/13] Make CodeFactor happy --- db-migration-worker/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db-migration-worker/Dockerfile b/db-migration-worker/Dockerfile index 8a7f8d758..865a4c330 100644 --- a/db-migration-worker/Dockerfile +++ b/db-migration-worker/Dockerfile @@ -1,11 +1,11 @@ FROM mongo:4 -RUN apt-get update -RUN apt-get -y install --no-install-recommends wget apt-transport-https +RUN apt-get update && apt-get -y install --no-install-recommends wget apt-transport-https \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* RUN wget -O - https://debian.neo4j.org/neotechnology.gpg.key | apt-key add - RUN echo 'deb https://debian.neo4j.org/repo stable/' | tee /etc/apt/sources.list.d/neo4j.list -RUN apt-get update -RUN apt-get -y install --no-install-recommends openjdk-8-jre openssh-client neo4j rsync \ +RUN apt-get update && apt-get -y install --no-install-recommends openjdk-8-jre openssh-client neo4j rsync \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* COPY migration ./migration From 5b674cd95fe8cb299b010b8783243186abc8e8dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Mon, 4 Feb 2019 21:33:48 +0100 Subject: [PATCH 09/13] Hopefully fix `migrate` on Digital Ocean It kept crashing, maybe because we're piping too much stuff to STDOUT. Let's see. --- db-migration-worker/migration/mongo/import.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db-migration-worker/migration/mongo/import.sh b/db-migration-worker/migration/mongo/import.sh index 4f59551aa..fb13a151f 100755 --- a/db-migration-worker/migration/mongo/import.sh +++ b/db-migration-worker/migration/mongo/import.sh @@ -21,7 +21,8 @@ mongo ${MONGODB_DATABASE} --eval "db.dropDatabase();" rm -rf /mongo-export/* ssh -4 -M -S my-ctrl-socket -fnNT -L 27018:localhost:27017 -l ${SSH_USERNAME} ${SSH_HOST} -mongodump --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --gzip --archive | mongorestore --gzip --archive +mongodump --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --gzip --archive=/tmp/mongodump.archive +mongorestore --gzip --archive=/tmp/mongodump.archive ssh -S my-ctrl-socket -O check -l ${SSH_USERNAME} ${SSH_HOST} ssh -S my-ctrl-socket -O exit -l ${SSH_USERNAME} ${SSH_HOST} From 82c679aaf143b0a77cb57b0174049ceeea75c409 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Mon, 4 Feb 2019 21:42:44 +0100 Subject: [PATCH 10/13] Check the environment variables early on I was missing NEO4J_URI and was told very late - let's check env vars at the very beginning. --- db-migration-worker/migrate.sh | 8 ++++++++ db-migration-worker/migration/mongo/import.sh | 8 -------- db-migration-worker/migration/neo4j/import.sh | 8 -------- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/db-migration-worker/migrate.sh b/db-migration-worker/migrate.sh index 108453bba..214da53d8 100755 --- a/db-migration-worker/migrate.sh +++ b/db-migration-worker/migrate.sh @@ -1,4 +1,12 @@ #!/usr/bin/env bash set -e +for var in "SSH_USERNAME" "SSH_HOST" "MONGODB_USERNAME" "MONGODB_PASSWORD" "MONGODB_DATABASE" "MONGODB_AUTH_DB" "NEO4J_URI" +do + if [[ -z "${!var}" ]]; then + echo "${var} is undefined" + exit 1 + fi +done + /migration/mongo/import.sh /migration/neo4j/import.sh diff --git a/db-migration-worker/migration/mongo/import.sh b/db-migration-worker/migration/mongo/import.sh index fb13a151f..7cf3e91e4 100755 --- a/db-migration-worker/migration/mongo/import.sh +++ b/db-migration-worker/migration/mongo/import.sh @@ -1,14 +1,6 @@ #!/usr/bin/env bash set -e -for var in "SSH_USERNAME" "SSH_HOST" "MONGODB_USERNAME" "MONGODB_PASSWORD" "MONGODB_DATABASE" "MONGODB_AUTH_DB" -do - if [[ -z "${!var}" ]]; then - echo "${var} is undefined" - exit 1 - fi -done - echo "SSH_USERNAME ${SSH_USERNAME}" echo "SSH_HOST ${SSH_HOST}" echo "MONGODB_USERNAME ${MONGODB_USERNAME}" diff --git a/db-migration-worker/migration/neo4j/import.sh b/db-migration-worker/migration/neo4j/import.sh index 9a5045d62..6f539c501 100755 --- a/db-migration-worker/migration/neo4j/import.sh +++ b/db-migration-worker/migration/neo4j/import.sh @@ -1,14 +1,6 @@ #!/usr/bin/env bash set -e -for var in "NEO4J_URI" -do - if [[ -z "${!var}" ]]; then - echo "${var} is undefined" - exit 1 - fi -done - SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" echo "MATCH (n) OPTIONAL MATCH (n)-[r]-() DELETE n,r;" | cypher-shell -a $NEO4J_URI for collection in "badges" "categories" "users" "follows" "contributions" "shouts" "comments" From 6610266a5bc2b47edc8bcd62bd5928081b339d60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Tue, 5 Feb 2019 14:32:20 +0100 Subject: [PATCH 11/13] Mount /migration for quick development --- docker-compose.db-migration.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.db-migration.yml b/docker-compose.db-migration.yml index 7d421e968..02f054d1b 100644 --- a/docker-compose.db-migration.yml +++ b/docker-compose.db-migration.yml @@ -15,6 +15,7 @@ services: volumes: - mongo-export:/mongo-export - uploads:/uploads + - ./db-migration-worker/migration/:/migration - ./db-migration-worker/.ssh/:/root/.ssh/ networks: - hc-network From 0705ab1bf5585d7afa3b39a567f2bbfe9d403f10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Tue, 5 Feb 2019 14:48:13 +0100 Subject: [PATCH 12/13] Fix missing contribution author Bug: Only if a tag and a category was present, the author was assigned. --- .../migration/neo4j/contributions.cql | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/db-migration-worker/migration/neo4j/contributions.cql b/db-migration-worker/migration/neo4j/contributions.cql index 86226b98f..0c7b18959 100644 --- a/db-migration-worker/migration/neo4j/contributions.cql +++ b/db-migration-worker/migration/neo4j/contributions.cql @@ -11,13 +11,15 @@ p.createdAt = post.createdAt.`$date`, p.updatedAt = post.updatedAt.`$date`, p.deleted = post.deleted, p.disabled = NOT post.isEnabled -WITH p, post, post.tags AS tags, post.categoryIds as categoryIds -UNWIND tags AS tag +WITH p, post +MATCH (u:User {id: post.userId}) +MERGE (u)-[:WROTE]->(p) +WITH p, post, post.categoryIds as categoryIds UNWIND categoryIds AS categoryId -MATCH (c:Category {id: categoryId}), - (u:User {id: post.userId}) +MATCH (c:Category {id: categoryId}) +MERGE (p)-[:CATEGORIZED]->(c) +WITH p, post.tags AS tags +UNWIND tags AS tag MERGE (t:Tag {id: apoc.create.uuid(), name: tag}) MERGE (p)-[:TAGGED]->(t) -MERGE (u)-[:WROTE]->(p) -MERGE (p)-[:CATEGORIZED]->(c) ; From 9656bd4eb7758ec4f09624c9736471347a240889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Tue, 5 Feb 2019 20:02:59 +0100 Subject: [PATCH 13/13] Fix comments import @appinteractive --- .../migration/neo4j/comments.cql | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/db-migration-worker/migration/neo4j/comments.cql b/db-migration-worker/migration/neo4j/comments.cql index 16537b730..6709acbc8 100644 --- a/db-migration-worker/migration/neo4j/comments.cql +++ b/db-migration-worker/migration/neo4j/comments.cql @@ -1,12 +1,14 @@ -CALL apoc.load.json('file:/mongo-export/comments.json') YIELD value as comment -MERGE (c:Comment {id: comment._id["$oid"]}) +CALL apoc.load.json('file:/mongo-export/comments.json') YIELD value as json +MERGE (comment:Comment {id: json._id["$oid"]}) ON CREATE SET -c.content = comment.content, -c.contentExcerpt = comment.contentExcerpt, -c.deleted = comment.deleted, -c.disabled = false -WITH comment -MATCH (p:Post {id: comment.contributionId}), (u:User {id: comment.userId}) -MERGE (c)-[:COMMENTS]->(p) -MERGE (u)-[:WROTE]->(c) +comment.content = json.content, +comment.contentExcerpt = json.contentExcerpt, +comment.deleted = json.deleted, +comment.disabled = false +WITH comment, json, json.contributionId as postId +MATCH (post:Post {id: postId}) +WITH comment, post, json.userId as userId +MATCH (author:User {id: userId}) +MERGE (comment)-[:COMMENTS]->(post) +MERGE (author)-[:WROTE]->(comment) ;