From fb929da2cd1dc7d5b3436ff3e19283faaa69c3a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Mon, 4 Feb 2019 01:34:17 +0100 Subject: [PATCH] Refactor db-migration-worker deployment Instead of creating a deployment with a replica set, we just create the pod once. Also the pod should have everything in the future to run the database migration. Ie. have `cypher-shell` to write directly to the database in the current network. All required configuration is passed manually to the `db-migration-worker`-pod directly. SSH-keys are copied through a secrets file. This altogether made many configuration files obsolete. --- .gitignore | 2 + README.md | 40 ++++++--- configmap-db-migration-worker.template.yaml | 12 --- db-migration-worker.yaml | 39 +++++++++ staging/.gitignore | 2 - staging/deployment-db-migration-worker.yaml | 92 --------------------- staging/volume-claim-mongo-exports.yaml | 12 --- staging/volume-claim-uploads.yaml | 12 --- 8 files changed, 70 insertions(+), 141 deletions(-) create mode 100644 .gitignore delete mode 100644 configmap-db-migration-worker.template.yaml create mode 100644 db-migration-worker.yaml delete mode 100644 staging/.gitignore delete mode 100644 staging/deployment-db-migration-worker.yaml delete mode 100644 staging/volume-claim-mongo-exports.yaml delete mode 100644 staging/volume-claim-uploads.yaml diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..18b453e6b --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +secrets.yaml +*/secrets.yaml diff --git a/README.md b/README.md index 7f998fcf2..6c8227f59 100644 --- a/README.md +++ b/README.md @@ -109,20 +109,38 @@ Wait until all pods turn green and they don't show a warning `Waiting: ContainerCreating` anymore. -### Provision db-migration-worker -Copy your private ssh key and the `.known-hosts` file of your remote legacy -server. -```shell - -# check the corresponding db-migration-worker pod -$ kubectl --namespace=staging get pods -# change below -$ kubectl cp path/to/your/ssh/keys/.ssh staging/nitro-db-migration-worker-:/root/ +### Migrate database of Human Connection legacy server +Create a configmap with the specific connection data of your legacy server: +```sh +$ kubectl create configmap db-migration-worker \ + --namespace=staging \ + --from-literal=SSH_USERNAME=someuser \ + --from-literal=SSH_HOST=yourhost \ + --from-literal=MONGODB_USERNAME=hc-api \ + --from-literal=MONGODB_PASSWORD=secretpassword \ + --from-literal=MONGODB_AUTH_DB=hc_api \ + --from-literal=MONGODB_DATABASE=hc_api \ + --from-literal=UPLOADS_DIRECTORY=/var/www/api/uploads ``` +Create a secret with your public and private ssh keys: +```sh +$ kubectl create secret generic ssh-keys \ + --namespace=staging \ + --from-file=id_rsa=/path/to/.ssh/id_rsa \ + --from-file=id_rsa.pub=/path/to/.ssh/id_rsa.pub \ + --from-file=known_hosts=/path/to/.ssh/known_hosts +``` +As the [kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/secret/#use-case-pod-with-ssh-keys) +points out, you should be careful with your ssh keys. Anyone with access to your +cluster will have access to your ssh keys. Better create a new pair with +`ssh-keygen` and copy the public key to your legacy server with `ssh-copy-id`. +Create the pod and the required volume: +```sh +$ kubectl apply -f db-migration-worker.yaml +``` Run the migration: ```shell # change below -$ kubectl --namespace=staging exec -it nitro-db-migration-worker- ./import.sh -$ kubectl --namespace=staging exec -it nitro-neo4j- ./import/import.sh +$ kubectl --namespace=staging exec -it nitro-db-migration-worker ./import.sh ``` diff --git a/configmap-db-migration-worker.template.yaml b/configmap-db-migration-worker.template.yaml deleted file mode 100644 index e00077577..000000000 --- a/configmap-db-migration-worker.template.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -data: - SSH_USERNAME: "" - SSH_HOST: "" - MONGODB_USERNAME: "hc-api" - MONGODB_AUTH_DB: "hc_api" - MONGODB_DATABASE: "hc_api" - UPLOADS_DIRECTORY: "/var/www/api/uploads" -metadata: - name: staging-db-migration-worker - namespace: staging diff --git a/db-migration-worker.yaml b/db-migration-worker.yaml new file mode 100644 index 000000000..e0b520e58 --- /dev/null +++ b/db-migration-worker.yaml @@ -0,0 +1,39 @@ +--- + kind: Pod + apiVersion: v1 + metadata: + name: nitro-db-migration-worker + namespace: staging + spec: + volumes: + - name: secret-volume + secret: + secretName: ssh-keys + defaultMode: 0400 + - name: mongo-export + persistentVolumeClaim: + claimName: mongo-export-claim + containers: + - name: nitro-db-migration-worker + image: humanconnection/db-migration-worker:latest + envFrom: + - configMapRef: + name: db-migration-worker + volumeMounts: + - name: secret-volume + readOnly: false + mountPath: /root/.ssh + - name: mongo-export + mountPath: /mongo-export/ +--- + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: mongo-export-claim + namespace: staging + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/staging/.gitignore b/staging/.gitignore deleted file mode 100644 index 599426dbb..000000000 --- a/staging/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -configmap-db-migration-worker.yaml -secrets.yaml diff --git a/staging/deployment-db-migration-worker.yaml b/staging/deployment-db-migration-worker.yaml deleted file mode 100644 index f4d427096..000000000 --- a/staging/deployment-db-migration-worker.yaml +++ /dev/null @@ -1,92 +0,0 @@ ---- - apiVersion: extensions/v1beta1 - kind: Deployment - metadata: - name: nitro-db-migration-worker - namespace: staging - spec: - replicas: 1 - minReadySeconds: 15 - progressDeadlineSeconds: 60 - selector: - matchLabels: - workload.user.cattle.io/workloadselector: deployment-staging-db-migration-worker - template: - metadata: - labels: - workload.user.cattle.io/workloadselector: deployment-staging-db-migration-worker - name: nitro-db-migration-worker - spec: - containers: - - env: - - name: COMMIT - value: - - name: SSH_USERNAME - valueFrom: - configMapKeyRef: - name: staging-db-migration-worker - key: SSH_USERNAME - - name: SSH_HOST - valueFrom: - configMapKeyRef: - name: staging-db-migration-worker - key: SSH_HOST - - name: MONGODB_USERNAME - valueFrom: - configMapKeyRef: - name: staging-db-migration-worker - key: MONGODB_USERNAME - - name: MONGODB_AUTH_DB - valueFrom: - configMapKeyRef: - name: staging-db-migration-worker - key: MONGODB_AUTH_DB - - name: MONGODB_DATABASE - valueFrom: - configMapKeyRef: - name: staging-db-migration-worker - key: MONGODB_DATABASE - - name: UPLOADS_DIRECTORY - valueFrom: - configMapKeyRef: - name: staging-db-migration-worker - key: UPLOADS_DIRECTORY - - name: MONGODB_PASSWORD - valueFrom: - secretKeyRef: - name: staging - key: MONGODB_PASSWORD - optional: false - image: humanconnection/db-migration-worker:latest - name: nitro-db-migration-worker - resources: {} - imagePullPolicy: Always - volumeMounts: - - mountPath: /root/ - name: ssh-keys-directory - - mountPath: /mongo-export/ - name: mongo-export - volumes: - - name: ssh-keys-directory - persistentVolumeClaim: - claimName: ssh-keys-claim - - name: mongo-export - persistentVolumeClaim: - claimName: mongo-export-claim - restartPolicy: Always - terminationGracePeriodSeconds: 30 - status: {} ---- - kind: PersistentVolumeClaim - apiVersion: v1 - metadata: - name: ssh-keys-claim - namespace: staging - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - # waaay too much - # unfortunately Digital Oceans volumes start at 1Gi - storage: 1Gi diff --git a/staging/volume-claim-mongo-exports.yaml b/staging/volume-claim-mongo-exports.yaml deleted file mode 100644 index 563a9cfe6..000000000 --- a/staging/volume-claim-mongo-exports.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- - kind: PersistentVolumeClaim - apiVersion: v1 - metadata: - name: mongo-export-claim - namespace: staging - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi diff --git a/staging/volume-claim-uploads.yaml b/staging/volume-claim-uploads.yaml deleted file mode 100644 index a48d28ddc..000000000 --- a/staging/volume-claim-uploads.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- - kind: PersistentVolumeClaim - apiVersion: v1 - metadata: - name: uploads-claim - namespace: staging - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 8Gi