Merge pull request #145 from Human-Connection/refactoring_db_migration_worker

Refactoring db migration worker

.dockerignore

@@ -11,7 +11,6 @@ docker-compose*.yml
 ./*.png
 ./*.log
 
-kubernetes/
 node_modules/
 scripts/
 dist/

db-migration-worker/.dockerignore

@@ -0,0 +1 @@
+.ssh/

db-migration-worker/.gitignore

@@ -1 +1 @@
-.ssh/id_rsa
+.ssh/

db-migration-worker/.ssh/.dockerignore

@@ -1 +0,0 @@
-.ssh/id_rsa

db-migration-worker/Dockerfile

@@ -1,8 +1,13 @@
 FROM mongo:4
 
-RUN apt-get update \
+RUN apt-get update &&  apt-get -y install --no-install-recommends wget apt-transport-https \
- && apt-get -y install --no-install-recommends openssh-client rsync \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/*
-COPY import.sh .
+RUN wget -O - https://debian.neo4j.org/neotechnology.gpg.key | apt-key add -
-
+RUN echo 'deb https://debian.neo4j.org/repo stable/' | tee /etc/apt/sources.list.d/neo4j.list
+RUN apt-get update && apt-get -y install --no-install-recommends openjdk-8-jre openssh-client neo4j rsync \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+COPY migration ./migration
+COPY migrate.sh /usr/local/bin/migrate
+COPY sync_uploads.sh /usr/local/bin/sync_uploads

db-migration-worker/migrate.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -e
+for var in "SSH_USERNAME" "SSH_HOST" "MONGODB_USERNAME" "MONGODB_PASSWORD" "MONGODB_DATABASE" "MONGODB_AUTH_DB" "NEO4J_URI"
+do
+  if [[ -z "${!var}" ]]; then
+    echo "${var} is undefined"
+    exit 1
+  fi
+done
+
+/migration/mongo/import.sh
+/migration/neo4j/import.sh

db-migration-worker/migration/mongo/import.sh

@@ -1,12 +1,5 @@
 #!/usr/bin/env bash
-
+set -e
-for var in "SSH_USERNAME" "SSH_HOST" "MONGODB_USERNAME" "MONGODB_PASSWORD" "MONGODB_DATABASE" "MONGODB_AUTH_DB" "UPLOADS_DIRECTORY"
-do
-  if [[ -z "${!var}" ]]; then
-    echo "${var} is undefined"
-    exit 1
-  fi
-done
 
 echo "SSH_USERNAME             ${SSH_USERNAME}"
 echo "SSH_HOST                 ${SSH_HOST}"
@@ -14,19 +7,17 @@ echo "MONGODB_USERNAME         ${MONGODB_USERNAME}"
 echo "MONGODB_PASSWORD         ${MONGODB_PASSWORD}"
 echo "MONGODB_DATABASE         ${MONGODB_DATABASE}"
 echo "MONGODB_AUTH_DB          ${MONGODB_AUTH_DB}"
-echo "UPLOADS_DIRECTORY        ${UPLOADS_DIRECTORY}"
 echo "-------------------------------------------------"
 
 mongo ${MONGODB_DATABASE} --eval "db.dropDatabase();"
-rm -f /mongo-export/*
+rm -rf /mongo-export/*
 
 ssh -4 -M -S my-ctrl-socket -fnNT -L 27018:localhost:27017 -l ${SSH_USERNAME} ${SSH_HOST}
-mongodump --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --gzip --archive | mongorestore --gzip --archive
+mongodump --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --gzip --archive=/tmp/mongodump.archive
+mongorestore --gzip  --archive=/tmp/mongodump.archive
 ssh -S my-ctrl-socket -O check -l ${SSH_USERNAME} ${SSH_HOST}
 ssh -S my-ctrl-socket -O exit  -l ${SSH_USERNAME} ${SSH_HOST}
 
-rsync --archive --update --verbose ${SSH_USERNAME}@${SSH_HOST}:${UPLOADS_DIRECTORY}/* /uploads/
-
 for collection in "categories" "badges" "users" "contributions" "comments" "follows" "shouts"
 do
   mongoexport --db ${MONGODB_DATABASE} --collection $collection --out "/mongo-export/$collection.json"

db-migration-worker/migration/neo4j/comments.cql

@@ -0,0 +1,14 @@
+CALL apoc.load.json('file:/mongo-export/comments.json') YIELD value as json
+MERGE (comment:Comment {id: json._id["$oid"]})
+ON CREATE SET
+comment.content        = json.content,
+comment.contentExcerpt = json.contentExcerpt,
+comment.deleted        = json.deleted,
+comment.disabled       = false
+WITH comment, json, json.contributionId as postId
+MATCH (post:Post {id: postId})
+WITH comment, post, json.userId as userId
+MATCH (author:User {id: userId})
+MERGE (comment)-[:COMMENTS]->(post)
+MERGE (author)-[:WROTE]->(comment)
+;

db-migration-worker/migration/neo4j/contributions.cql

@@ -11,13 +11,15 @@ p.createdAt      = post.createdAt.`$date`,
 p.updatedAt      = post.updatedAt.`$date`,
 p.deleted        = post.deleted,
 p.disabled       = NOT post.isEnabled
-WITH p, post, post.tags AS tags, post.categoryIds as categoryIds
+WITH p, post
-UNWIND tags AS tag
+MATCH (u:User {id: post.userId})
+MERGE (u)-[:WROTE]->(p)
+WITH p, post, post.categoryIds as categoryIds
 UNWIND categoryIds AS categoryId
-MATCH (c:Category {id: categoryId}),
+MATCH (c:Category {id: categoryId})
-   (u:User {id: post.userId})
+MERGE (p)-[:CATEGORIZED]->(c)
+WITH p, post.tags AS tags
+UNWIND tags AS tag
 MERGE (t:Tag {id: apoc.create.uuid(), name: tag})
 MERGE (p)-[:TAGGED]->(t)
-MERGE (u)-[:WROTE]->(p)
-MERGE (p)-[:CATEGORIZED]->(c)
 ;

db-migration-worker/migration/neo4j/import.sh

@@ -1,7 +1,9 @@
 #!/usr/bin/env bash
+set -e
+
 SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-echo "MATCH (n) OPTIONAL MATCH (n)-[r]-() DELETE n,r;" | cypher-shell
+echo "MATCH (n) OPTIONAL MATCH (n)-[r]-() DELETE n,r;" | cypher-shell -a $NEO4J_URI
 for collection in "badges" "categories" "users" "follows" "contributions" "shouts" "comments"
 do
-  echo "Import ${collection}..." && cypher-shell < $SCRIPT_DIRECTORY/$collection.cql
+  echo "Import ${collection}..." && cypher-shell -a $NEO4J_URI < $SCRIPT_DIRECTORY/$collection.cql
 done

db-migration-worker/sync_uploads.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -e
+
+for var in "SSH_USERNAME" "SSH_HOST" "UPLOADS_DIRECTORY"
+do
+  if [[ -z "${!var}" ]]; then
+    echo "${var} is undefined"
+    exit 1
+  fi
+done
+
+rsync --archive --update --verbose ${SSH_USERNAME}@${SSH_HOST}:${UPLOADS_DIRECTORY}/* /uploads/

docker-compose.db-migration.yml

@@ -0,0 +1,36 @@
+version: "3.7"
+
+services:
+  backend:
+    volumes:
+      - uploads:/nitro-backend/public/uploads
+  neo4j:
+    volumes:
+      - mongo-export:/mongo-export
+    environment:
+      - NEO4J_apoc_import_file_enabled=true
+  db-migration-worker:
+    build:
+      context: db-migration-worker
+    volumes:
+      - mongo-export:/mongo-export
+      - uploads:/uploads
+      - ./db-migration-worker/migration/:/migration
+      - ./db-migration-worker/.ssh/:/root/.ssh/
+    networks:
+      - hc-network
+    depends_on:
+      - backend
+    environment:
+      - NEO4J_URI=bolt://neo4j:7687
+      - "SSH_USERNAME=${SSH_USERNAME}"
+      - "SSH_HOST=${SSH_HOST}"
+      - "MONGODB_USERNAME=${MONGODB_USERNAME}"
+      - "MONGODB_PASSWORD=${MONGODB_PASSWORD}"
+      - "MONGODB_AUTH_DB=${MONGODB_AUTH_DB}"
+      - "MONGODB_DATABASE=${MONGODB_DATABASE}"
+      - "UPLOADS_DIRECTORY=${UPLOADS_DIRECTORY}"
+
+volumes:
+  mongo-export:
+  uploads:

docker-compose.override.yml

@@ -9,36 +9,8 @@ services:
     volumes:
       - .:/nitro-backend
       - /nitro-backend/node_modules
-      - uploads:/nitro-backend/public/uploads
     command: yarn run dev
   neo4j:
-    volumes:
-      - mongo-export:/mongo-export
-      - ./neo4j/import:/var/lib/neo4j/import
     ports:
       - 7687:7687
       - 7474:7474
-    environment:
-      - NEO4J_apoc_import_file_enabled=true
-  db-migration-worker:
-    build:
-      context: db-migration-worker
-    volumes:
-      - mongo-export:/mongo-export
-      - uploads:/uploads
-      - ./db-migration-worker/.ssh/:/root/.ssh/
-    networks:
-      - hc-network
-    environment:
-      - "SSH_USERNAME=${SSH_USERNAME}"
-      - "SSH_HOST=${SSH_HOST}"
-      - "MONGODB_USERNAME=${MONGODB_USERNAME}"
-      - "MONGODB_PASSWORD=${MONGODB_PASSWORD}"
-      - "MONGODB_AUTH_DB=${MONGODB_AUTH_DB}"
-      - "MONGODB_DATABASE=${MONGODB_DATABASE}"
-      - "UPLOADS_DIRECTORY=${UPLOADS_DIRECTORY}"
-    command: "--smallfiles --logpath=/dev/null"
-
-volumes:
-  mongo-export:
-  uploads:

kubernetes/README.md

@@ -1,51 +0,0 @@
-# Usage with minikube
-Assuming you installed the packages git, docker ([all-distributions](https://docs.docker.com/install/)), minikube ([ubuntu]([minikube](https://computingforgeeks.com/how-to-install-minikube-on-ubuntu-18-04/)), [all-distributions](https://kubernetes.io/docs/tasks/tools/install-minikube/)) and virtualbox here...
-
-First of all start minikube on your machine:
-```sh
-minikube start
-```
-
-You can always get an overview and see what's going on with your minikube:
-```sh
-minikube dashboard
-```
-
-From now on stay in your favorite work directory. First let's clone the necessary sources:
-```sh
-git clone https://github.com/Human-Connection/Nitro-Backend.git
-git clone https://github.com/Human-Connection/Nitro-Web.git
-```
-
-Build Docker images, using the Minikube Docker daemon:
-```sh
-eval $(minikube docker-env)
-docker build -t humanconnection/nitro-backend:latest Nitro-Backend/
-docker build -t humanconnection/neo4j:latest -f Nitro-Backend/Dockerfile.neo4j Nitro-Backend/
-```
-
-Check that the image is in Minikube’s Docker registry:
-```sh
-minikube ssh docker images 
-```
-
-Now change into directory Nitro-Backend/kubernetes and create services and deployments:
-```sh
-cd Nitro-Backend/kubernetes
-kubectl create -f neo4j-deployment.yaml,neo4j-data-persistentvolumeclaim.yaml,backend-deployment.yaml,neo4j-service.json,backend-service.json
-```
-
-You can see the backend in action with:
-```sh
-minikube service backend
-```
-
-### Troubleshoot
-
-1. If you get an error message along th lines of 'The vboxdrv kernel module is not loaded.' - then you have the same issue i had. to solve this you need to install the propper linux kernel host modules package. Here an example for Manjaro:  
-https://forum.manjaro.org/t/installing-virtualbox-kernel-modules/6999  
-2. When you can not start minikube, try also to remove the cluster with `minikube delete` and start again with `minikube start`. Sometimes this fix startup problems of the cluster.   
-3. Now again you might run into trouble with an error like 'kubectl could not be found on your path.' In this case run the following command:
-    ```sh
-    curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.10.0/bin/linux/amd64/kubectl && chmod +x kubectl && sudo cp kubectl /usr/local/bin/ && rm kubectl
-    ```

kubernetes/backend-deployment.yaml

@@ -1,41 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  annotations:
-    kompose.cmd: kompose convert -f ../docker-compose.yml
-    kompose.version: 1.16.0 (0c01309)
-  creationTimestamp: null
-  labels:
-    io.kompose.service: backend
-  name: backend
-spec:
-  replicas: 1
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        io.kompose.service: backend
-    spec:
-      containers:
-      - env:
-        - name: CLIENT_URI
-          value: http://localhost:3000
-        - name: GRAPHQL_PORT
-          value: "4000"
-        - name: GRAPHQL_URI
-          value: http://localhost:4000
-        - name: JWT_SECRET
-          value: b/&&7b78BF&fv/Vd
-        - name: MOCK
-          value: "false"
-        - name: NEO4J_URI
-          value: bolt://neo4j:7687
-        image: humanconnection/nitro-backend:latest
-        name: backend
-        ports:
-        - containerPort: 4000
-        resources: {}
-        imagePullPolicy: IfNotPresent
-      restartPolicy: Always
-status: {}

kubernetes/backend-service.json

@@ -1,35 +0,0 @@
-{
-  "kind": "Service",
-  "apiVersion": "v1",
-  "metadata": {
-    "name": "backend",
-    "namespace": "default",
-    "selfLink": "/api/v1/namespaces/default/services/backend",
-    "uid": "57749bc1-dea3-11e8-ace2-080027b5d96d",
-    "resourceVersion": "8321",
-    "creationTimestamp": "2018-11-02T13:29:36Z",
-    "labels": {
-      "io.kompose.service": "backend"
-    }
-  },
-  "spec": {
-    "ports": [
-      {
-        "protocol": "TCP",
-        "port": 4000,
-        "targetPort": 4000,
-        "nodePort": 32633
-      }
-    ],
-    "selector": {
-      "io.kompose.service": "backend"
-    },
-    "clusterIP": "10.110.129.92",
-    "type": "LoadBalancer",
-    "sessionAffinity": "None",
-    "externalTrafficPolicy": "Cluster"
-  },
-  "status": {
-    "loadBalancer": {}
-  }
-}

kubernetes/dashboard-admin.yml

@@ -1,14 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRoleBinding
-metadata:
-  name: kubernetes-dashboard
-  labels:
-    k8s-app: kubernetes-dashboard
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: cluster-admin
-subjects:
-- kind: ServiceAccount
-  name: kubernetes-dashboard
-  namespace: kube-system

kubernetes/neo4j-data-persistentvolumeclaim.yaml

@@ -1,14 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  creationTimestamp: null
-  labels:
-    io.kompose.service: neo4j-data
-  name: neo4j-data
-spec:
-  accessModes:
-  - ReadWriteOnce
-  resources:
-    requests:
-      storage: 100Mi
-status: {}

kubernetes/neo4j-deployment.yaml

@@ -1,40 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  annotations:
-    kompose.cmd: kompose convert -f ../docker-compose.yml
-    kompose.version: 1.16.0 (0c01309)
-  creationTimestamp: null
-  labels:
-    io.kompose.service: neo4j
-  name: neo4j
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        io.kompose.service: neo4j
-    spec:
-      containers:
-      - env:
-        - name: NEO4J_AUTH
-          value: none
-        image: humanconnection/neo4j:latest
-        name: neo4j
-        ports:
-        - containerPort: 7687
-        - containerPort: 7474
-        resources: {}
-        imagePullPolicy: IfNotPresent
-        volumeMounts:
-        - mountPath: /data
-          name: neo4j-data
-      restartPolicy: Always
-      volumes:
-      - name: neo4j-data
-        persistentVolumeClaim:
-          claimName: neo4j-data
-status: {}

kubernetes/neo4j-service.json

@@ -1,43 +0,0 @@
-{
-  "kind": "Service",
-  "apiVersion": "v1",
-  "metadata": {
-    "name": "neo4j",
-    "namespace": "default",
-    "selfLink": "/api/v1/namespaces/default/services/neo4j",
-    "uid": "57de6181-dea3-11e8-ace2-080027b5d96d",
-    "resourceVersion": "8326",
-    "creationTimestamp": "2018-11-02T13:29:37Z",
-    "labels": {
-      "io.kompose.service": "neo4j"
-    }
-  },
-  "spec": {
-    "ports": [
-      {
-        "name": "port-1",
-        "protocol": "TCP",
-        "port": 7687,
-        "targetPort": 7687,
-        "nodePort": 30116
-      },
-      {
-        "name": "port-2",
-        "protocol": "TCP",
-        "port": 7474,
-        "targetPort": 7474,
-        "nodePort": 32658
-      }
-    ],
-    "selector": {
-      "io.kompose.service": "neo4j"
-    },
-    "clusterIP": "10.108.175.122",
-    "type": "LoadBalancer",
-    "sessionAffinity": "None",
-    "externalTrafficPolicy": "Cluster"
-  },
-  "status": {
-    "loadBalancer": {}
-  }
-}

neo4j/Dockerfile

@@ -1,3 +1,2 @@
 FROM neo4j:3.5.0
 RUN wget https://github.com/neo4j-contrib/neo4j-apoc-procedures/releases/download/3.5.0.1/apoc-3.5.0.1-all.jar -P plugins/
-COPY import ./import

neo4j/import/comments.cql

@@ -1,12 +0,0 @@
-CALL apoc.load.json('file:/mongo-export/comments.json') YIELD value as comment
-MERGE (c:Comment {id: comment._id["$oid"]})
-ON CREATE SET
-c.content        = comment.content,
-c.contentExcerpt = comment.contentExcerpt,
-c.deleted        = comment.deleted,
-c.disabled       = false
-WITH comment
-MATCH (p:Post {id: comment.contributionId}), (u:User {id: comment.userId})
-MERGE (c)-[:COMMENTS]->(p)
-MERGE (u)-[:WROTE]->(c)
-;

neo4j/import/todo

@@ -1,2 +0,0 @@
-
-