From 6ed5ad58d5cd216847d32ca7e8e73fd645ce72c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Wed, 24 Apr 2019 00:26:37 +0200 Subject: [PATCH] Merge maintenance with deployment --- SUMMARY.md | 2 +- deployment/README.md | 54 ------------------ deployment/legacy-migration/README.md | 55 +++++++++++++++++++ .../maintenance-worker}/.dockerignore | 0 .../maintenance-worker}/.gitignore | 0 .../maintenance-worker}/Dockerfile | 0 .../maintenance-worker}/README.md | 0 .../binaries/create_private_ssh_key_from_env | 0 .../binaries/import_legacy_db | 0 .../binaries/import_legacy_uploads | 0 .../maintenance-worker}/known_hosts | 0 .../migration/mongo/import.sh | 0 .../migration/neo4j/badges.cql | 0 .../migration/neo4j/categories.cql | 0 .../migration/neo4j/comments.cql | 0 .../migration/neo4j/contributions.cql | 0 .../migration/neo4j/follows.cql | 0 .../migration/neo4j/import.sh | 0 .../migration/neo4j/shouts.cql | 0 .../migration/neo4j/users.cql | 0 docker-compose.maintenance.yml | 4 +- 21 files changed, 58 insertions(+), 57 deletions(-) create mode 100644 deployment/legacy-migration/README.md rename {maintenance => deployment/legacy-migration/maintenance-worker}/.dockerignore (100%) rename {maintenance => deployment/legacy-migration/maintenance-worker}/.gitignore (100%) rename {maintenance => deployment/legacy-migration/maintenance-worker}/Dockerfile (100%) rename {maintenance => deployment/legacy-migration/maintenance-worker}/README.md (100%) rename {maintenance => deployment/legacy-migration/maintenance-worker}/binaries/create_private_ssh_key_from_env (100%) rename {maintenance => deployment/legacy-migration/maintenance-worker}/binaries/import_legacy_db (100%) rename {maintenance => deployment/legacy-migration/maintenance-worker}/binaries/import_legacy_uploads (100%) rename {maintenance => deployment/legacy-migration/maintenance-worker}/known_hosts (100%) rename {maintenance => deployment/legacy-migration/maintenance-worker}/migration/mongo/import.sh (100%) rename {maintenance => deployment/legacy-migration/maintenance-worker}/migration/neo4j/badges.cql (100%) rename {maintenance => deployment/legacy-migration/maintenance-worker}/migration/neo4j/categories.cql (100%) rename {maintenance => deployment/legacy-migration/maintenance-worker}/migration/neo4j/comments.cql (100%) rename {maintenance => deployment/legacy-migration/maintenance-worker}/migration/neo4j/contributions.cql (100%) rename {maintenance => deployment/legacy-migration/maintenance-worker}/migration/neo4j/follows.cql (100%) rename {maintenance => deployment/legacy-migration/maintenance-worker}/migration/neo4j/import.sh (100%) rename {maintenance => deployment/legacy-migration/maintenance-worker}/migration/neo4j/shouts.cql (100%) rename {maintenance => deployment/legacy-migration/maintenance-worker}/migration/neo4j/users.cql (100%) diff --git a/SUMMARY.md b/SUMMARY.md index 937b2b06b..54a751238 100644 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -26,7 +26,7 @@ * [Kubernetes Dashboard](deployment/digital-ocean/dashboard/README.md) * [HTTPS](deployment/digital-ocean/https/README.md) * [Neo4J DB Backup](deployment/backup.md) -* [Maintenance](maintenance/README.md) + * [Legacy Migration](deployment/legacy-migration/README.md) * [Feature Specification](cypress/features.md) * [Code of conduct](CODE_OF_CONDUCT.md) * [License](LICENSE.md) diff --git a/deployment/README.md b/deployment/README.md index 77e03e722..e191bddc0 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -45,57 +45,3 @@ $ kubectl apply -f human-connection/ ``` This can take a while because kubernetes will download the docker images. Sit back and relax and have a look into your kubernetes dashboard. Wait until all pods turn green and they don't show a warning `Waiting: ContainerCreating` anymore. - - -#### Legacy data migration - -This setup is completely optional and only required if you have data on a server which is running our legacy code and you want to import that data. It will import the uploads folder and migrate a dump of mongodb into neo4j. - -**Prepare migration of Human Connection legacy server** - -Create a configmap with the specific connection data of your legacy server: - -```bash -$ kubectl create configmap maintenance-worker \ - --namespace=human-connection \ - --from-literal=SSH_USERNAME=someuser \ - --from-literal=SSH_HOST=yourhost \ - --from-literal=MONGODB_USERNAME=hc-api \ - --from-literal=MONGODB_PASSWORD=secretpassword \ - --from-literal=MONGODB_AUTH_DB=hc_api \ - --from-literal=MONGODB_DATABASE=hc_api \ - --from-literal=UPLOADS_DIRECTORY=/var/www/api/uploads \ - --from-literal=NEO4J_URI=bolt://localhost:7687 -``` - -Create a secret with your public and private ssh keys. As the [kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/secret/#use-case-pod-with-ssh-keys) points out, you should be careful with your ssh keys. Anyone with access to your cluster will have access to your ssh keys. Better create a new pair with `ssh-keygen` and copy the public key to your legacy server with `ssh-copy-id`: - -```bash -$ kubectl create secret generic ssh-keys \ - --namespace=human-connection \ - --from-file=id_rsa=/path/to/.ssh/id_rsa \ - --from-file=id_rsa.pub=/path/to/.ssh/id_rsa.pub \ - --from-file=known_hosts=/path/to/.ssh/known_hosts -``` - -**Migrate legacy database** - -Patch the existing deployments to use a multi-container setup: - -```bash -cd legacy-migration -kubectl apply -f volume-claim-mongo-export.yaml -kubectl patch --namespace=human-connection deployment nitro-backend --patch "$(cat deployment-backend.yaml)" -kubectl patch --namespace=human-connection deployment nitro-neo4j --patch "$(cat deployment-neo4j.yaml)" -cd .. -``` - -Run the migration: - -```text -$ kubectl --namespace=human-connection get pods -# change below -$ kubectl --namespace=human-connection exec -it nitro-neo4j-65bbdb597c-nc2lv migrate -$ kubectl --namespace=human-connection exec -it nitro-backend-c6cc5ff69-8h96z sync_uploads -``` - diff --git a/deployment/legacy-migration/README.md b/deployment/legacy-migration/README.md new file mode 100644 index 000000000..419a9a1dc --- /dev/null +++ b/deployment/legacy-migration/README.md @@ -0,0 +1,55 @@ +# Legacy data migration + +This setup is **completely optional** and only required if you have data on a +server which is running our legacy code and you want to import that data. It +will import the uploads folder and migrate a dump of the legacy Mongo database +into our new Neo4J graph database. + +**Prepare migration of Human Connection legacy server** + +Create a configmap with the specific connection data of your legacy server: + +```bash +$ kubectl create configmap maintenance-worker \ + --namespace=human-connection \ + --from-literal=SSH_USERNAME=someuser \ + --from-literal=SSH_HOST=yourhost \ + --from-literal=MONGODB_USERNAME=hc-api \ + --from-literal=MONGODB_PASSWORD=secretpassword \ + --from-literal=MONGODB_AUTH_DB=hc_api \ + --from-literal=MONGODB_DATABASE=hc_api \ + --from-literal=UPLOADS_DIRECTORY=/var/www/api/uploads \ + --from-literal=NEO4J_URI=bolt://localhost:7687 +``` + +Create a secret with your public and private ssh keys. As the [kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/secret/#use-case-pod-with-ssh-keys) points out, you should be careful with your ssh keys. Anyone with access to your cluster will have access to your ssh keys. Better create a new pair with `ssh-keygen` and copy the public key to your legacy server with `ssh-copy-id`: + +```bash +$ kubectl create secret generic ssh-keys \ + --namespace=human-connection \ + --from-file=id_rsa=/path/to/.ssh/id_rsa \ + --from-file=id_rsa.pub=/path/to/.ssh/id_rsa.pub \ + --from-file=known_hosts=/path/to/.ssh/known_hosts +``` + +**Migrate legacy database** + +Patch the existing deployments to use a multi-container setup: + +```bash +cd legacy-migration +kubectl apply -f volume-claim-mongo-export.yaml +kubectl patch --namespace=human-connection deployment nitro-backend --patch "$(cat deployment-backend.yaml)" +kubectl patch --namespace=human-connection deployment nitro-neo4j --patch "$(cat deployment-neo4j.yaml)" +cd .. +``` + +Run the migration: + +```text +$ kubectl --namespace=human-connection get pods +# change below +$ kubectl --namespace=human-connection exec -it nitro-neo4j-65bbdb597c-nc2lv migrate +$ kubectl --namespace=human-connection exec -it nitro-backend-c6cc5ff69-8h96z sync_uploads +``` + diff --git a/maintenance/.dockerignore b/deployment/legacy-migration/maintenance-worker/.dockerignore similarity index 100% rename from maintenance/.dockerignore rename to deployment/legacy-migration/maintenance-worker/.dockerignore diff --git a/maintenance/.gitignore b/deployment/legacy-migration/maintenance-worker/.gitignore similarity index 100% rename from maintenance/.gitignore rename to deployment/legacy-migration/maintenance-worker/.gitignore diff --git a/maintenance/Dockerfile b/deployment/legacy-migration/maintenance-worker/Dockerfile similarity index 100% rename from maintenance/Dockerfile rename to deployment/legacy-migration/maintenance-worker/Dockerfile diff --git a/maintenance/README.md b/deployment/legacy-migration/maintenance-worker/README.md similarity index 100% rename from maintenance/README.md rename to deployment/legacy-migration/maintenance-worker/README.md diff --git a/maintenance/binaries/create_private_ssh_key_from_env b/deployment/legacy-migration/maintenance-worker/binaries/create_private_ssh_key_from_env similarity index 100% rename from maintenance/binaries/create_private_ssh_key_from_env rename to deployment/legacy-migration/maintenance-worker/binaries/create_private_ssh_key_from_env diff --git a/maintenance/binaries/import_legacy_db b/deployment/legacy-migration/maintenance-worker/binaries/import_legacy_db similarity index 100% rename from maintenance/binaries/import_legacy_db rename to deployment/legacy-migration/maintenance-worker/binaries/import_legacy_db diff --git a/maintenance/binaries/import_legacy_uploads b/deployment/legacy-migration/maintenance-worker/binaries/import_legacy_uploads similarity index 100% rename from maintenance/binaries/import_legacy_uploads rename to deployment/legacy-migration/maintenance-worker/binaries/import_legacy_uploads diff --git a/maintenance/known_hosts b/deployment/legacy-migration/maintenance-worker/known_hosts similarity index 100% rename from maintenance/known_hosts rename to deployment/legacy-migration/maintenance-worker/known_hosts diff --git a/maintenance/migration/mongo/import.sh b/deployment/legacy-migration/maintenance-worker/migration/mongo/import.sh similarity index 100% rename from maintenance/migration/mongo/import.sh rename to deployment/legacy-migration/maintenance-worker/migration/mongo/import.sh diff --git a/maintenance/migration/neo4j/badges.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql similarity index 100% rename from maintenance/migration/neo4j/badges.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql diff --git a/maintenance/migration/neo4j/categories.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql similarity index 100% rename from maintenance/migration/neo4j/categories.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql diff --git a/maintenance/migration/neo4j/comments.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql similarity index 100% rename from maintenance/migration/neo4j/comments.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql diff --git a/maintenance/migration/neo4j/contributions.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql similarity index 100% rename from maintenance/migration/neo4j/contributions.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql diff --git a/maintenance/migration/neo4j/follows.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql similarity index 100% rename from maintenance/migration/neo4j/follows.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql diff --git a/maintenance/migration/neo4j/import.sh b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh similarity index 100% rename from maintenance/migration/neo4j/import.sh rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh diff --git a/maintenance/migration/neo4j/shouts.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql similarity index 100% rename from maintenance/migration/neo4j/shouts.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql diff --git a/maintenance/migration/neo4j/users.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql similarity index 100% rename from maintenance/migration/neo4j/users.cql rename to deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql diff --git a/docker-compose.maintenance.yml b/docker-compose.maintenance.yml index 33a9d28f6..551a7e244 100644 --- a/docker-compose.maintenance.yml +++ b/docker-compose.maintenance.yml @@ -4,11 +4,11 @@ services: maintenance: image: humanconnection/maintenance-worker:latest build: - context: maintenance + context: deployment/legacy-migration/maintenance-worker volumes: - uploads:/uploads - neo4j-data:/data - - ./maintenance/migration/:/migration + - ./deployment/legacy-migration/maintenance-worker/migration/:/migration networks: - hc-network environment: