diff --git a/deployment/Minikube.md b/deployment/Minikube.md new file mode 100644 index 0000000..8e6240a --- /dev/null +++ b/deployment/Minikube.md @@ -0,0 +1,25 @@ +# Minikube + +There are many Kubernetes providers, but if you're just getting started, Minikube is a tool that you can use to get your feet wet. + +After you [installed Minikube](https://kubernetes.io/docs/tasks/tools/install-minikube/) +open your minikube dashboard: + +```text +$ minikube dashboard +``` + +This will give you an overview. Some of the steps below need some timing to make resources available to other dependent deployments. Keeping an eye on the dashboard is a great way to check that. + +Follow the installation instruction for [Kubernetes with Helm](./kubernetes/README.md). + +If all the pods and services have settled and everything looks green in your +minikube dashboard, expose the services you want on your host system. + +For example: + +```text +$ minikube service webapp --namespace=ocelotsocialnetwork +# optionally +$ minikube service backend --namespace=ocelotsocialnetwork +``` \ No newline at end of file diff --git a/deployment/README.md b/deployment/README.md new file mode 100644 index 0000000..f0d573a --- /dev/null +++ b/deployment/README.md @@ -0,0 +1,14 @@ +# Deployment + +You have the following options for a deployment: + +- [Kubernetes with Helm](./kubernetes/README.md) + +## After Deployment + +After the first deployment of the new network on your server, the database is initialized with the default administrator: + +- E-mail: admin@example.org +- Password: 1234 + +TODO: When you are logged in for the first time, please change your (the admin's) e-mail to an existing one and change your password to a secure one !!! \ No newline at end of file diff --git a/deployment/kubernetes/.gitignore b/deployment/kubernetes/.gitignore new file mode 100644 index 0000000..e0473b0 --- /dev/null +++ b/deployment/kubernetes/.gitignore @@ -0,0 +1,3 @@ +/dns.values.yaml +/nginx.values.yaml +/values.yaml diff --git a/deployment/kubernetes/Chart.yaml b/deployment/kubernetes/Chart.yaml new file mode 100644 index 0000000..e7f36a7 --- /dev/null +++ b/deployment/kubernetes/Chart.yaml @@ -0,0 +1,39 @@ +type: application +apiVersion: v2 +name: ocelot-social +version: "1.0.0" +# The appVersion defines which docker image is pulled. +# Having it set to latest will pull the latest build on dockerhub. +# You are free to define a specific version here tho. +# e.g. appVersion: "0.6.10" +# Be aware that this requires all dockers to have the same version available. +appVersion: "latest" +description: The Helm chart for ocelot.social +home: https://ocelot.social +sources: + - https://github.com/Ocelot-Social-Community/ + - https://github.com/Ocelot-Social-Community/Ocelot-Social + - https://github.com/Ocelot-Social-Community/Ocelot-Social-Deploy-Rebranding +maintainers: + - name: Ulf Gebhardt + email: ulf.gebhardt@webcraft-media.de + url: https://www.webcraft-media.de/#!ulf_gebhardt +icon: https://github.com/Ocelot-Social-Community/Ocelot-Social/raw/master/webapp/static/img/custom/welcome.svg +deprecated: false + +# Unused Fields +#dependencies: # A list of the chart requirements (optional) +# - name: ingress-nginx +# version: v1.10.0 +# repository: https://kubernetes.github.io/ingress-nginx +# condition: (optional) A yaml path that resolves to a boolean, used for enabling/disabling charts (e.g. subchart1.enabled ) +# tags: # (optional) +# - Tags can be used to group charts for enabling/disabling together +# import-values: # (optional) +# - ImportValues holds the mapping of source values to parent key to be imported. Each item can be a string or pair of child/parent sublist items. +# alias: (optional) Alias to be used for the chart. Useful when you have to add the same chart multiple times +#kubeVersion: A SemVer range of compatible Kubernetes versions (optional) +#keywords: +# - A list of keywords about this project (optional) +#annotations: +# example: A list of annotations keyed by name (optional). \ No newline at end of file diff --git a/deployment/kubernetes/LICENSE b/deployment/kubernetes/LICENSE new file mode 100644 index 0000000..a027039 --- /dev/null +++ b/deployment/kubernetes/LICENSE @@ -0,0 +1,12 @@ +# LICENSE + +MIT License + +Copyright \(c\) 2021 Ulf Gebhardt + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files \(the "Software"\), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/deployment/kubernetes/README.md b/deployment/kubernetes/README.md new file mode 100644 index 0000000..e71eced --- /dev/null +++ b/deployment/kubernetes/README.md @@ -0,0 +1,128 @@ +# Helm installation of Ocelot.social + +Deploying Ocelot.social with Helm is very straight forward. All you have to do is to change certain parameters, like domain names and API keys, then you just install our provided Helm chart to your cluster. + +## Configuration + +You can customize the network with your configuration by changing the `values.yaml`, all variables will be available as environment variables in your deployed kubernetes pods. For more details refer to the `values.yaml.dist` file. + +Besides the `values.yaml.dist` file we provide a `nginx.values.yaml.dist` and `dns.values.yaml.dist`. The `nginx.values.yaml` is the configuration for the ingress-nginx helm chart, while the `dns.values.yaml` file is for automatically updating the dns values on digital ocean and therefore optional. + +As hinted above you should copy the given files and rename them accordingly. Then go ahead and modify the values in the newly created files accordingly. + +## Installation + +Due to the many limitations of Helm you still have to do several manual steps. Those occur before you run the actual ocelot helm chart. Obviously it is expected of you to have `helm` and `kubectl` installed. For Digital Ocean you might require `doctl` aswell. + +### Cert Manager (https) + +Please refer to [cert-manager.io docs](https://cert-manager.io/docs/installation/kubernetes/) for more details. + +1. Create Namespace + +```bash +kubectl --kubeconfig=/../kubeconfig.yaml create namespace cert-manager +``` + +2. Add Helm Repo & update + +```bash +helm repo add jetstack https://charts.jetstack.io +helm repo update +``` + +3. Install Cert-Manager Helm chart +```bash +# this can not be included sine the CRDs cant be installed properly via helm... +helm --kubeconfig=/../kubeconfig.yaml \ + install cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --version v1.1.0 \ + --set installCRDs=true +``` + +### Ingress-Nginx + +1. Add Helm Repo & update +```bash +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +``` + +2. Install ingress-nginx +```bash +helm --kubeconfig=/../kubeconfig.yaml install ingress-nginx ingress-nginx/ingress-nginx -f nginx.values.yaml +``` + +### Digital Ocean Firewall + +This is only necessary if you run Digital Ocean without load balancer ([see here for more info](https://stackoverflow.com/questions/54119399/expose-port-80-on-digital-oceans-managed-kubernetes-without-a-load-balancer/55968709)) . + +1. Authenticate towards DO with your local `doctl` + +You will need a DO token for that. +```bash +doctl auth init +``` + +2. Generate DO firewall +```bash +doctl compute firewall create \ +--inbound-rules="protocol:tcp,ports:80,address:0.0.0.0/0,address:::/0 protocol:tcp,ports:443,address:0.0.0.0/0,address:::/0" \ +--tag-names=k8s:1ebf0cdc-86c9-4384-936b-40010b71d049 \ +--name=my-domain-http-https +``` + +### DNS + +This chart is only necessary (recommended is more precise) if you run Digital Ocean without load balancer. +You need to generate a token for the `dns.values.yaml`. + +1. Add Helm Repo & update +```bash +helm repo add bitnami https://charts.bitnami.com/bitnami +helm repo update +``` + +2. Install DNS +```bash +helm --kubeconfig=/../kubeconfig.yaml install dns bitnami/external-dns -f dns.values.yaml +``` + +### Ocelot.social + +All commands for ocelot need to be executed in the kubernetes folder. Therefore `cd deployment/kubernetes/` is expected to be run before every command. Furthermore the given commands will install ocelot into the default namespace. This can be modified to by attaching `--namespace not.default`. + +#### Install +```bash +helm --kubeconfig=/../kubeconfig.yaml install ocelot ./ +``` + +#### Upgrade +```bash +helm --kubeconfig=/../kubeconfig.yaml upgrade ocelot ./ +``` + +#### Uninstall +Be aware that if you uninstall ocelot the formerly bound volumes become unbound. Those volumes contain all data from uploads and database. You have to manually free their reference in order to bind them again when reinstalling. Once unbound from their former container references they should automatically be rebound (considering the sizes did not change) + +```bash +helm --kubeconfig=/../kubeconfig.yaml uninstall ocelot +``` + +## Error reporting + +We use [Sentry](https://github.com/getsentry/sentry) for error reporting in both +our backend and web frontend. You can either use a hosted or a self-hosted +instance. Just set the two `DSN` in your +[configmap](../templates/configmap.template.yaml) and update the `COMMIT` +during a deployment with your commit or the version of your release. + +### Self-hosted Sentry + +For data privacy it is recommended to set up your own instance of sentry. +If you are lucky enough to have a kubernetes cluster with the required hardware +support, try this [helm chart](https://github.com/helm/charts/tree/master/stable/sentry). + +On our kubernetes cluster we get "mult-attach" errors for persistent volumes. +Apparently Digital Ocean's kubernetes clusters do not fulfill the requirements. \ No newline at end of file diff --git a/deployment/kubernetes/charts/.gitkeep b/deployment/kubernetes/charts/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/deployment/kubernetes/crds/.gitkeep b/deployment/kubernetes/crds/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/deployment/kubernetes/dns.values.yaml.dist b/deployment/kubernetes/dns.values.yaml.dist new file mode 100644 index 0000000..16d2975 --- /dev/null +++ b/deployment/kubernetes/dns.values.yaml.dist @@ -0,0 +1,10 @@ +provider: digitalocean +digitalocean: + # create the API token at https://cloud.digitalocean.com/account/api/tokens + # needs read + write + apiToken: "TODO" +domainFilters: + # domains you want external-dns to be able to edit + - TODO.TODO +rbac: + create: true \ No newline at end of file diff --git a/deployment/kubernetes/nginx.values.yaml.dist b/deployment/kubernetes/nginx.values.yaml.dist new file mode 100644 index 0000000..06dad81 --- /dev/null +++ b/deployment/kubernetes/nginx.values.yaml.dist @@ -0,0 +1,11 @@ +controller: + kind: DaemonSet + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + ingressClass: nginx + daemonset: + useHostPort: true + service: + type: ClusterIP +rbac: + create: true \ No newline at end of file diff --git a/deployment/kubernetes/templates/NOTES.txt b/deployment/kubernetes/templates/NOTES.txt new file mode 100644 index 0000000..3db4648 --- /dev/null +++ b/deployment/kubernetes/templates/NOTES.txt @@ -0,0 +1 @@ +You installed ocelot-social! Congrats <3 \ No newline at end of file diff --git a/deployment/kubernetes/templates/backend/ConfigMap.yml b/deployment/kubernetes/templates/backend/ConfigMap.yml new file mode 100644 index 0000000..ec7a566 --- /dev/null +++ b/deployment/kubernetes/templates/backend/ConfigMap.yml @@ -0,0 +1,26 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: configmap-{{ .Release.Name }}-backend + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "configmap-backend" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +data: + PUBLIC_REGISTRATION: "{{ .Values.PUBLIC_REGISTRATION }}" + CLIENT_URI: "{{ .Values.BACKEND.CLIENT_URI }}" + EMAIL_DEFAULT_SENDER: "{{ .Values.BACKEND.EMAIL_DEFAULT_SENDER }}" + SMTP_HOST: "{{ .Values.BACKEND.SMTP_HOST }}" + SMTP_PORT: "{{ .Values.BACKEND.SMTP_PORT }}" + SMTP_IGNORE_TLS: "{{ .Values.BACKEND.SMTP_IGNORE_TLS }}" + SMTP_SECURE: "{{ .Values.BACKEND.SMTP_SECURE }}" + GRAPHQL_URI: "http://{{ .Release.Name }}-backend:4000" + NEO4J_URI: "bolt://{{ .Release.Name }}-neo4j:7687" + #REDIS_DOMAIN: ---toBeSet(IP)--- + #REDIS_PORT: "6379" + #SENTRY_DSN_WEBAPP: "---toBeSet---" + #SENTRY_DSN_BACKEND: "---toBeSet---" \ No newline at end of file diff --git a/deployment/kubernetes/templates/backend/Deployment.yaml b/deployment/kubernetes/templates/backend/Deployment.yaml new file mode 100644 index 0000000..1d7bcfa --- /dev/null +++ b/deployment/kubernetes/templates/backend/Deployment.yaml @@ -0,0 +1,56 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Release.Name }}-backend + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "deployment-backend" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +spec: + replicas: 1 + minReadySeconds: {{ .Values.BACKEND.MIN_READY_SECONDS }} + progressDeadlineSeconds: {{ .Values.BACKEND.PROGRESS_DEADLINE_SECONDS }} + strategy: + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-backend + template: + metadata: + annotations: + backup.velero.io/backup-volumes: uploads + # make sure the pod is redeployed + rollme: {{ randAlphaNum 5 | quote }} + labels: + app: {{ .Release.Name }}-backend + spec: + containers: + - name: container-{{ .Release.Name }}-backend + image: "{{ .Values.BACKEND.DOCKER_IMAGE_REPO }}:{{ .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.BACKEND.DOCKER_IMAGE_PULL_POLICY }} + envFrom: + - configMapRef: + name: configmap-{{ .Release.Name }}-backend + - secretRef: + name: secret-{{ .Release.Name }}-backend + ports: + - containerPort: 4000 + protocol: TCP + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /app/public/uploads + name: uploads + dnsPolicy: ClusterFirst + schedulerName: default-scheduler + restartPolicy: {{ .Values.BACKEND.CONTAINER_RESTART_POLICY }} + terminationGracePeriodSeconds: {{ .Values.BACKEND.CONTAINER_TERMINATION_GRACE_PERIOD_SECONDS }} + volumes: + - name: uploads + persistentVolumeClaim: + claimName: volume-claim-{{ .Release.Name }}-uploads \ No newline at end of file diff --git a/deployment/kubernetes/templates/backend/PersistentVolumeClaim.yaml b/deployment/kubernetes/templates/backend/PersistentVolumeClaim.yaml new file mode 100644 index 0000000..758e9e1 --- /dev/null +++ b/deployment/kubernetes/templates/backend/PersistentVolumeClaim.yaml @@ -0,0 +1,24 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: volume-claim-{{ .Release.Name }}-uploads + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "volume-claim-backend" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +spec: + #dataSource: + # name: uploads-snapshot + # kind: VolumeSnapshot + # apiGroup: snapshot.storage.k8s.io + storageClassName: storage-{{ .Release.Name }}-persistent + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.BACKEND.STORAGE_UPLOADS }} + diff --git a/deployment/kubernetes/templates/backend/Secret.yaml b/deployment/kubernetes/templates/backend/Secret.yaml new file mode 100644 index 0000000..51e5516 --- /dev/null +++ b/deployment/kubernetes/templates/backend/Secret.yaml @@ -0,0 +1,21 @@ +kind: Secret +apiVersion: v1 +metadata: + name: secret-{{ .Release.Name }}-backend + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "secret-backend" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +stringData: + JWT_SECRET: "{{ .Values.BACKEND.JWT_SECRET }}" + MAPBOX_TOKEN: "{{ .Values.BACKEND.MAPBOX_TOKEN }}" + PRIVATE_KEY_PASSPHRASE: "{{ .Values.BACKEND.PRIVATE_KEY_PASSPHRASE }}" + SMTP_USERNAME: "{{ .Values.BACKEND.SMTP_USERNAME }}" + SMTP_PASSWORD: "{{ .Values.BACKEND.SMTP_PASSWORD }}" + #NEO4J_USERNAME: "" + #NEO4J_PASSWORD: "" + #REDIS_PASSWORD: ---toBeSet--- diff --git a/deployment/kubernetes/templates/backend/Service.yaml b/deployment/kubernetes/templates/backend/Service.yaml new file mode 100644 index 0000000..9029be5 --- /dev/null +++ b/deployment/kubernetes/templates/backend/Service.yaml @@ -0,0 +1,20 @@ +kind: Service +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-backend + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "service-backend" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +spec: + ports: + - name: {{ .Release.Name }}-graphql + port: 4000 + targetPort: 4000 + protocol: TCP + selector: + app: {{ .Release.Name }}-backend diff --git a/deployment/kubernetes/templates/issuer/letsencrypt-production.yaml b/deployment/kubernetes/templates/issuer/letsencrypt-production.yaml new file mode 100644 index 0000000..2836cce --- /dev/null +++ b/deployment/kubernetes/templates/issuer/letsencrypt-production.yaml @@ -0,0 +1,22 @@ +apiVersion: cert-manager.io/v1alpha2 +kind: ClusterIssuer +metadata: + name: letsencrypt-production + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "letsencrypt-production" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: {{ .Values.LETSENCRYPT.EMAIL }} + privateKeySecretRef: + name: letsencrypt-production + solvers: + - http01: + ingress: + class: nginx diff --git a/deployment/kubernetes/templates/issuer/letsencrypt-staging.yaml b/deployment/kubernetes/templates/issuer/letsencrypt-staging.yaml new file mode 100644 index 0000000..7190e65 --- /dev/null +++ b/deployment/kubernetes/templates/issuer/letsencrypt-staging.yaml @@ -0,0 +1,22 @@ +apiVersion: cert-manager.io/v1alpha2 +kind: ClusterIssuer +metadata: + name: letsencrypt-staging + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "letsencrypt-staging" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +spec: + acme: + server: https://acme-staging-v02.api.letsencrypt.org/directory + email: {{ .Values.LETSENCRYPT.EMAIL }} + privateKeySecretRef: + name: letsencrypt-staging + solvers: + - http01: + ingress: + class: nginx diff --git a/deployment/kubernetes/templates/jobs/job-db-init.yaml b/deployment/kubernetes/templates/jobs/job-db-init.yaml new file mode 100644 index 0000000..f207bd8 --- /dev/null +++ b/deployment/kubernetes/templates/jobs/job-db-init.yaml @@ -0,0 +1,29 @@ +kind: Job +apiVersion: batch/v1 +metadata: + name: job-{{ .Release.Name }}-db-init + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "job-db-init" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + annotations: + "helm.sh/hook": post-install + "helm.sh/hook-delete-policy": hook-succeeded, hook-failed + "helm.sh/hook-weight": "0" +spec: + template: + spec: + restartPolicy: Never + containers: + - name: job-{{ .Release.Name }}-db-init + image: "{{ .Values.BACKEND.DOCKER_IMAGE_REPO }}:{{ .Chart.AppVersion }}" + command: ["/bin/sh", "-c", "yarn prod:migrate init"] + envFrom: + - configMapRef: + name: configmap-{{ .Release.Name }}-backend + - secretRef: + name: secret-{{ .Release.Name }}-backend \ No newline at end of file diff --git a/deployment/kubernetes/templates/jobs/job-db-migrate.yaml b/deployment/kubernetes/templates/jobs/job-db-migrate.yaml new file mode 100644 index 0000000..950793d --- /dev/null +++ b/deployment/kubernetes/templates/jobs/job-db-migrate.yaml @@ -0,0 +1,29 @@ +kind: Job +apiVersion: batch/v1 +metadata: + name: job-{{ .Release.Name }}-db-migrate + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "job-db-migrate" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, hook-failed + "helm.sh/hook-weight": "5" +spec: + template: + spec: + restartPolicy: Never + containers: + - name: job-{{ .Release.Name }}-db-migrations + image: "{{ .Values.BACKEND.DOCKER_IMAGE_REPO }}:{{ .Chart.AppVersion }}" + command: ["/bin/sh", "-c", "yarn prod:migrate up"] + envFrom: + - configMapRef: + name: configmap-{{ .Release.Name }}-backend + - secretRef: + name: secret-{{ .Release.Name }}-backend \ No newline at end of file diff --git a/deployment/kubernetes/templates/maintenance/ConfigMap.yml b/deployment/kubernetes/templates/maintenance/ConfigMap.yml new file mode 100644 index 0000000..fe29afb --- /dev/null +++ b/deployment/kubernetes/templates/maintenance/ConfigMap.yml @@ -0,0 +1,14 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: configmap-{{ .Release.Name }}-maintenance + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "configmap-maintenance" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +data: + HOST: "0.0.0.0" \ No newline at end of file diff --git a/deployment/kubernetes/templates/maintenance/Deployment.yaml b/deployment/kubernetes/templates/maintenance/Deployment.yaml new file mode 100644 index 0000000..6c4e7f8 --- /dev/null +++ b/deployment/kubernetes/templates/maintenance/Deployment.yaml @@ -0,0 +1,39 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Release.Name }}-maintenance + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "deployment-maintenance" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +spec: + strategy: + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-maintenance + template: + metadata: + labels: + app: {{ .Release.Name }}-maintenance + # make sure the pod is redeployed + rollme: {{ randAlphaNum 5 | quote }} + spec: + containers: + - name: maintenance + image: "{{ .Values.MAINTENANCE.DOCKER_IMAGE_REPO }}:{{ .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.MAINTENANCE.DOCKER_IMAGE_PULL_POLICY }} + envFrom: + - configMapRef: + name: configmap-{{ .Release.Name }}-webapp + - secretRef: + name: secret-{{ .Release.Name }}-webapp + ports: + - containerPort: 80 + restartPolicy: {{ .Values.MAINTENANCE.CONTAINER_RESTART_POLICY }} + terminationGracePeriodSeconds: {{ .Values.MAINTENANCE.CONTAINER_TERMINATION_GRACE_PERIOD_SECONDS }} diff --git a/deployment/kubernetes/templates/maintenance/Secret.yaml b/deployment/kubernetes/templates/maintenance/Secret.yaml new file mode 100644 index 0000000..b4752e5 --- /dev/null +++ b/deployment/kubernetes/templates/maintenance/Secret.yaml @@ -0,0 +1,13 @@ +kind: Secret +apiVersion: v1 +metadata: + name: secret-{{ .Release.Name }}-maintenance + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "secret-maintenance" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +stringData: diff --git a/deployment/kubernetes/templates/maintenance/Service.yaml b/deployment/kubernetes/templates/maintenance/Service.yaml new file mode 100644 index 0000000..95f042d --- /dev/null +++ b/deployment/kubernetes/templates/maintenance/Service.yaml @@ -0,0 +1,20 @@ +kind: Service +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-maintenance + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "service-maintenance" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +spec: + ports: + - name: {{ .Release.Name }}-http + port: 80 + targetPort: 80 + protocol: TCP + selector: + app: {{ .Release.Name }}-maintenance diff --git a/deployment/kubernetes/templates/neo4j/ConfigMap.yml b/deployment/kubernetes/templates/neo4j/ConfigMap.yml new file mode 100644 index 0000000..9f0aa4b --- /dev/null +++ b/deployment/kubernetes/templates/neo4j/ConfigMap.yml @@ -0,0 +1,21 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: configmap-{{ .Release.Name }}-neo4j + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "configmap-neo4j" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +data: + NEO4J_ACCEPT_LICENSE_AGREEMENT: "{{ .Values.NEO4J.ACCEPT_LICENSE_AGREEMENT }}" + NEO4J_AUTH: "{{ .Values.NEO4J.AUTH }}" + NEO4J_dbms_connector_bolt_thread__pool__max__size: "{{ .Values.NEO4J.DBMS_CONNECTOR_BOLT_THREAD_POOL_MAX_SIZE }}" + NEO4J_dbms_memory_heap_initial__size: "{{ .Values.NEO4J.DBMS_MEMORY_HEAP_INITIAL_SIZE }}" + NEO4J_dbms_memory_heap_max__size: "{{ .Values.NEO4J.DBMS_MEMORY_HEAP_MAX_SIZE }}" + NEO4J_dbms_memory_pagecache_size: "{{ .Values.NEO4J.DBMS_MEMORY_PAGECACHE_SIZE }}" + NEO4J_dbms_security_procedures_unrestricted: "{{ .Values.NEO4J.DBMS_SECURITY_PROCEDURES_UNRESTRICTED }}" + NEO4J_apoc_import_file_enabled: "{{ .Values.NEO4J.APOC_IMPORT_FILE_ENABLED }}" \ No newline at end of file diff --git a/deployment/kubernetes/templates/neo4j/Deployment.yaml b/deployment/kubernetes/templates/neo4j/Deployment.yaml new file mode 100644 index 0000000..2176fb2 --- /dev/null +++ b/deployment/kubernetes/templates/neo4j/Deployment.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-neo4j + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "deployment-neo4j" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +spec: + replicas: 1 + strategy: + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-neo4j + template: + metadata: + name: neo4j + annotations: + backup.velero.io/backup-volumes: neo4j-data + # make sure the pod is redeployed + rollme: {{ randAlphaNum 5 | quote }} + labels: + app: {{ .Release.Name }}-neo4j + spec: + containers: + - name: container-{{ .Release.Name }}-neo4j + image: "{{ .Values.NEO4J.DOCKER_IMAGE_REPO }}:{{ .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.NEO4J.DOCKER_IMAGE_PULL_POLICY }} + ports: + - containerPort: 7687 + - containerPort: 7474 + resources: + requests: + memory: {{ .Values.NEO4J.RESOURCE_REQUESTS_MEMORY | default "1G" | quote }} + limits: + memory: {{ .Values.NEO4J.RESOURCE_LIMITS_MEMORY | default "1G" | quote }} + envFrom: + - configMapRef: + name: configmap-{{ .Release.Name }}-neo4j + - secretRef: + name: secret-{{ .Release.Name }}-neo4j + volumeMounts: + - mountPath: /data/ + name: neo4j-data + volumes: + - name: neo4j-data + persistentVolumeClaim: + claimName: volume-claim-{{ .Release.Name }}-neo4j + restartPolicy: {{ .Values.NEO4J.CONTAINER_RESTART_POLICY }} + terminationGracePeriodSeconds: {{ .Values.NEO4J.CONTAINER_TERMINATION_GRACE_PERIOD_SECONDS }} diff --git a/deployment/kubernetes/templates/neo4j/PersistentVolumeClaim.yaml b/deployment/kubernetes/templates/neo4j/PersistentVolumeClaim.yaml new file mode 100644 index 0000000..3aab02d --- /dev/null +++ b/deployment/kubernetes/templates/neo4j/PersistentVolumeClaim.yaml @@ -0,0 +1,19 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: volume-claim-{{ .Release.Name }}-neo4j + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "volume-claim-neo4j" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +spec: + storageClassName: storage-{{ .Release.Name }}-persistent + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.NEO4J.STORAGE }} \ No newline at end of file diff --git a/deployment/kubernetes/templates/neo4j/Secret.yaml b/deployment/kubernetes/templates/neo4j/Secret.yaml new file mode 100644 index 0000000..d8b1c17 --- /dev/null +++ b/deployment/kubernetes/templates/neo4j/Secret.yaml @@ -0,0 +1,15 @@ +kind: Secret +apiVersion: v1 +metadata: + name: secret-{{ .Release.Name }}-neo4j + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "secret-neo4j" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +stringData: + NEO4J_USERNAME: "" + NEO4J_PASSWORD: "" \ No newline at end of file diff --git a/deployment/kubernetes/templates/neo4j/Service.yaml b/deployment/kubernetes/templates/neo4j/Service.yaml new file mode 100644 index 0000000..4ed56bd --- /dev/null +++ b/deployment/kubernetes/templates/neo4j/Service.yaml @@ -0,0 +1,23 @@ +kind: Service +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-neo4j + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "service-neo4j" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +spec: + ports: + - name: {{ .Release.Name }}-bolt + port: 7687 + targetPort: 7687 + protocol: TCP + #- name: {{ .Release.Name }}-http + # port: 7474 + # targetPort: 7474 + selector: + app: {{ .Release.Name }}-neo4j diff --git a/deployment/kubernetes/templates/storage/persistent.yml b/deployment/kubernetes/templates/storage/persistent.yml new file mode 100644 index 0000000..2ac07c5 --- /dev/null +++ b/deployment/kubernetes/templates/storage/persistent.yml @@ -0,0 +1,16 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: storage-{{ .Release.Name }}-persistent + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "storage-persistent" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +provisioner: {{ .Values.STORAGE.PROVISIONER }} +reclaimPolicy: {{ .Values.STORAGE.RECLAIM_POLICY }} +volumeBindingMode: {{ .Values.STORAGE.VOLUME_BINDING_MODE }} +allowVolumeExpansion: {{ .Values.STORAGE.ALLOW_VOLUME_EXPANSION }} \ No newline at end of file diff --git a/deployment/kubernetes/templates/webapp/ConfigMap.yml b/deployment/kubernetes/templates/webapp/ConfigMap.yml new file mode 100644 index 0000000..e7093e6 --- /dev/null +++ b/deployment/kubernetes/templates/webapp/ConfigMap.yml @@ -0,0 +1,17 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: configmap-{{ .Release.Name }}-webapp + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "configmap-webapp" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +data: + HOST: "0.0.0.0" + PUBLIC_REGISTRATION: "{{ .Values.PUBLIC_REGISTRATION }}" + WEBSOCKETS_URI: "{{ .Values.WEBAPP.WEBSOCKETS_URI }}" + GRAPHQL_URI: "http://{{ .Release.Name }}-backend:4000" \ No newline at end of file diff --git a/deployment/kubernetes/templates/webapp/Deployment.yaml b/deployment/kubernetes/templates/webapp/Deployment.yaml new file mode 100644 index 0000000..93d1639 --- /dev/null +++ b/deployment/kubernetes/templates/webapp/Deployment.yaml @@ -0,0 +1,43 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Release.Name }}-webapp + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "deployment-webapp" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +spec: + replicas: {{ .Values.WEBAPP.REPLICAS }} + minReadySeconds: {{ .Values.WEBAPP.MIN_READY_SECONDS }} + progressDeadlineSeconds: {{ .Values.WEBAPP.PROGRESS_DEADLINE_SECONDS }} + strategy: + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-webapp + template: + metadata: + annotations: + # make sure the pod is redeployed + rollme: {{ randAlphaNum 5 | quote }} + labels: + app: {{ .Release.Name }}-webapp + spec: + containers: + - name: container-{{ .Release.Name }}-webapp + image: "{{ .Values.WEBAPP.DOCKER_IMAGE_REPO }}:{{ .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.WEBAPP.DOCKER_IMAGE_PULL_POLICY }} + ports: + - containerPort: 3000 + envFrom: + - configMapRef: + name: configmap-{{ .Release.Name }}-webapp + - secretRef: + name: secret-{{ .Release.Name }}-webapp + restartPolicy: {{ .Values.WEBAPP.CONTAINER_RESTART_POLICY }} + terminationGracePeriodSeconds: {{ .Values.WEBAPP.CONTAINER_TERMINATION_GRACE_PERIOD_SECONDS }} \ No newline at end of file diff --git a/deployment/kubernetes/templates/webapp/Ingress.yaml b/deployment/kubernetes/templates/webapp/Ingress.yaml new file mode 100644 index 0000000..d7b12bd --- /dev/null +++ b/deployment/kubernetes/templates/webapp/Ingress.yaml @@ -0,0 +1,36 @@ +kind: Ingress +apiVersion: networking.k8s.io/v1 +metadata: + name: ingress-{{ .Release.Name }}-webapp + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "ingress-webapp" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + annotations: + kubernetes.io/ingress.class: "nginx" + cert-manager.io/cluster-issuer: {{ .Values.LETSENCRYPT.ISSUER }} + nginx.ingress.kubernetes.io/proxy-body-size: {{ .Values.NGINX.PROXY_BODY_SIZE }} +spec: + tls: + - hosts: + {{- range .Values.LETSENCRYPT.DOMAINS }} + - {{ . }} + {{- end }} + secretName: tls + rules: + {{- range .Values.LETSENCRYPT.DOMAINS }} + - host: {{ . }} + http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + service: + name: {{ $.Release.Name }}-webapp + port: + number: 3000 + {{- end }} \ No newline at end of file diff --git a/deployment/kubernetes/templates/webapp/Secret.yaml b/deployment/kubernetes/templates/webapp/Secret.yaml new file mode 100644 index 0000000..15b1db4 --- /dev/null +++ b/deployment/kubernetes/templates/webapp/Secret.yaml @@ -0,0 +1,13 @@ +kind: Secret +apiVersion: v1 +metadata: + name: secret-{{ .Release.Name }}-webapp + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "secret-webapp" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +stringData: diff --git a/deployment/kubernetes/templates/webapp/Service.yaml b/deployment/kubernetes/templates/webapp/Service.yaml new file mode 100644 index 0000000..0c3112e --- /dev/null +++ b/deployment/kubernetes/templates/webapp/Service.yaml @@ -0,0 +1,20 @@ +kind: Service +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-webapp + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "service-webapp" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +spec: + ports: + - name: {{ .Release.Name }}-http + port: 3000 + targetPort: 3000 + protocol: TCP + selector: + app: {{ .Release.Name }}-webapp diff --git a/deployment/kubernetes/values.yaml.dist b/deployment/kubernetes/values.yaml.dist new file mode 100644 index 0000000..e259e91 --- /dev/null +++ b/deployment/kubernetes/values.yaml.dist @@ -0,0 +1,104 @@ +# Change all the below if needed +PUBLIC_REGISTRATION: false + +BACKEND: + # Change all the below if needed + # DOCKER_IMAGE_REPO - change that to your branded docker image + # Label is appended based on .Chart.appVersion + DOCKER_IMAGE_REPO: "ocelotsocialnetwork/backend" + CLIENT_URI: "https://staging.ocelot.social" + JWT_SECRET: "b/&&7b78BF&fv/Vd" + MAPBOX_TOKEN: "pk.eyJ1IjoiYnVzZmFrdG9yIiwiYSI6ImNraDNiM3JxcDBhaWQydG1uczhpZWtpOW4ifQ.7TNRTO-o9aK1Y6MyW_Nd4g" + PRIVATE_KEY_PASSPHRASE: "a7dsf78sadg87ad87sfagsadg78" + EMAIL_DEFAULT_SENDER: "devops@ocelot.social" + SMTP_HOST: "mail.ocelot.social" + SMTP_USERNAME: "devops@ocelot.social" + SMTP_PASSWORD: "devops@ocelot.social" + SMTP_PORT: "465" + SMTP_IGNORE_TLS: 'true' + SMTP_SECURE: 'false' + + # Most likely you don't need to change this + MIN_READY_SECONDS: "15" + PROGRESS_DEADLINE_SECONDS: "60" + CONTAINER_RESTART_POLICY: "Always" + CONTAINER_TERMINATION_GRACE_PERIOD_SECONDS: "30" + DOCKER_IMAGE_PULL_POLICY: "Always" + STORAGE_UPLOADS: "25Gi" + +WEBAPP: + # Change all the below if needed + # DOCKER_IMAGE_REPO - change that to your branded docker image + # Label is appended based on .Chart.appVersion + DOCKER_IMAGE_REPO: "ocelotsocialnetwork/webapp" + WEBSOCKETS_URI: "wss://staging.ocelot.social/api/graphql" + + # Most likely you don't need to change this + REPLICAS: "2" + MIN_READY_SECONDS: "15" + PROGRESS_DEADLINE_SECONDS: "60" + CONTAINER_RESTART_POLICY: "Always" + CONTAINER_TERMINATION_GRACE_PERIOD_SECONDS: "30" + DOCKER_IMAGE_PULL_POLICY: "Always" + +NEO4J: + # Most likely you don't need to change this + DOCKER_IMAGE_REPO: "ocelotsocialnetwork/neo4j" + DOCKER_IMAGE_PULL_POLICY: "Always" + CONTAINER_RESTART_POLICY: "Always" + CONTAINER_TERMINATION_GRACE_PERIOD_SECONDS: "30" + STORAGE: "5Gi" + # RESOURCE_REQUESTS_MEMORY configures the memory available for requests. + RESOURCE_REQUESTS_MEMORY: "2G" + # RESOURCE_LIMITS_MEMORY configures the memory limits available. + RESOURCE_LIMITS_MEMORY: "4G" + # required for Neo4j Enterprice version + #ACCEPT_LICENSE_AGREEMENT: "yes" + ACCEPT_LICENSE_AGREEMENT: "no" + AUTH: "none" + #DBMS_CONNECTOR_BOLT_THREAD_POOL_MAX_SIZE: "10000" # hc value + DBMS_CONNECTOR_BOLT_THREAD_POOL_MAX_SIZE: "400" # default value + #DBMS_MEMORY_HEAP_INITIAL_SIZE: "500MB" # HC value + DBMS_MEMORY_HEAP_INITIAL_SIZE: "" # default + #DBMS_MEMORY_HEAP_MAX_SIZE: "500MB" # HC value + DBMS_MEMORY_HEAP_MAX_SIZE: "" # default + #DBMS_MEMORY_PAGECACHE_SIZE: "490M" # HC value + DBMS_MEMORY_PAGECACHE_SIZE: "" # default + #APOC_IMPORT_FILE_ENABLED: "true" # HC value + APOC_IMPORT_FILE_ENABLED: "false" # default + DBMS_SECURITY_PROCEDURES_UNRESTRICTED: "algo.*,apoc.*" + +MAINTENANCE: + # Change all the below if needed + # DOCKER_IMAGE_REPO - change that to your branded docker image + # Label is appended based on .Chart.appVersion + DOCKER_IMAGE_REPO: "ocelotsocialnetwork/maintenance" + + # Most likely you don't need to change this + CONTAINER_RESTART_POLICY: "Always" + CONTAINER_TERMINATION_GRACE_PERIOD_SECONDS: "30" + DOCKER_IMAGE_PULL_POLICY: "Always" + +LETSENCRYPT: + # Change all the below if needed + # ISSUER is used by cert-manager to set up certificates with the given provider. + # change it to "letsencrypt-production" once you are ready to have valid cetrificates. + # Be aware that the is an issuing limit with letsencrypt, so a dry run with staging might be wise + ISSUER: "letsencrypt-staging" + EMAIL: "devops@ocelot.social" + DOMAINS: + - "staging.ocelot.social" + - "www.staging.ocelot.social" + +NGINX: + # Most likely you don't need to change this + PROXY_BODY_SIZE: "10m" + +STORAGE: + # Change all the below if needed + PROVISIONER: "dobs.csi.digitalocean.com" + + # Most likely you don't need to change this + RECLAIM_POLICY: "Retain" + VOLUME_BINDING_MODE: "Immediate" + ALLOW_VOLUME_EXPANSION: true \ No newline at end of file diff --git a/deployment/old/Maintenance.md b/deployment/old/Maintenance.md new file mode 100644 index 0000000..08a177e --- /dev/null +++ b/deployment/old/Maintenance.md @@ -0,0 +1,45 @@ +# Maintenance mode + +> Despite our best efforts, systems sometimes require downtime for a variety of reasons. + +Quote from [here](https://www.nrmitchi.com/2017/11/easy-maintenance-mode-in-kubernetes/) + +We use our maintenance mode for manual database backup and restore. Also we +bring the database into maintenance mode for manual database migrations. + +## Deploy the service + +We prepared sample configuration, so you can simply run: + +```sh +# in folder deployment/ +$ kubectl apply -f ./ocelot-social/maintenance/ +``` + +This will fire up a maintenance service. + +## Bring application into maintenance mode + +Now if you want to have a controlled downtime and you want to bring your +application into maintenance mode, you can edit your global ingress server. + +E.g. copy file [`deployment/digital-ocean/https/templates/ingress.template.yaml`](../../digital-ocean/https/templates/ingress.template.yaml) to new file `deployment/digital-ocean/https/ingress.yaml` and change the following: + +```yaml +... + + - host: develop-k8s.ocelot.social + http: + paths: + - path: / + backend: + # serviceName: web + serviceName: maintenance + # servicePort: 3000 + servicePort: 80 +``` + +Then run `$ kubectl apply -f deployment/digital-ocean/https/ingress.yaml`. If you +want to deactivate the maintenance server, just undo the edit and apply the +configuration again. + diff --git a/deployment/old/digital-ocean/README.md b/deployment/old/digital-ocean/README.md new file mode 100644 index 0000000..2ded383 --- /dev/null +++ b/deployment/old/digital-ocean/README.md @@ -0,0 +1,39 @@ +# Digital Ocean + +As a start, read the [introduction into Kubernetes](https://www.digitalocean.com/community/tutorials/an-introduction-to-kubernetes) by the folks at Digital Ocean. The following section should enable you to deploy ocelot.social to your Kubernetes cluster. + +## Connect to your local cluster + +1. Create a cluster at [Digital Ocean](https://www.digitalocean.com/). +2. Download the `***-kubeconfig.yaml` from the Web UI. +3. Move the file to the default location where kubectl expects it to be: `mv ***-kubeconfig.yaml ~/.kube/config`. Alternatively you can set the config on every command: `--kubeconfig ***-kubeconfig.yaml` +4. Now check if you can connect to the cluster and if its your newly created one by running: `kubectl get nodes` + +The output should look about like this: + +```sh +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +nifty-driscoll-uu1w Ready 69d v1.13.2 +nifty-driscoll-uuiw Ready 69d v1.13.2 +nifty-driscoll-uusn Ready 69d v1.13.2 +``` + +If you got the steps right above and see your nodes you can continue. + +Digital Ocean Kubernetes clusters don't have a graphical interface, so I suggest +to setup the [Kubernetes dashboard](./dashboard/README.md) as a next step. +Configuring [HTTPS](./https/README.md) is bit tricky and therefore I suggest to +do this as a last step. + +## Spaces + +We are storing our images in the s3-compatible [DigitalOcean Spaces](https://www.digitalocean.com/docs/spaces/). + +We still want to take backups of our images in case something happens to the images in the cloud. See these [instructions](https://www.digitalocean.com/docs/spaces/resources/s3cmd-usage/) about getting set up with `s3cmd` to take a copy of all images in a `Spaces` namespace, i.e. `ocelot-social-uploads`. + +After configuring `s3cmd` with your credentials, etc. you should be able to make a backup with this command. + +```sh +s3cmg get --recursive --skip-existing s3://ocelot-social-uploads +``` diff --git a/deployment/old/digital-ocean/dashboard/README.md b/deployment/old/digital-ocean/dashboard/README.md new file mode 100644 index 0000000..5f66afe --- /dev/null +++ b/deployment/old/digital-ocean/dashboard/README.md @@ -0,0 +1,55 @@ +# Install Kubernetes Dashboard + +The kubernetes dashboard is optional but very helpful for debugging. If you want to install it, you have to do so only **once** per cluster: + +```bash +# in folder deployment/digital-ocean/ +$ kubectl apply -f dashboard/ +$ kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta4/aio/deploy/recommended.yaml +``` + +### Login to your dashboard + +Proxy the remote kubernetes dashboard to localhost: + +```bash +$ kubectl proxy +``` + +Visit: + +[http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/](http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/) + +You should see a login screen. + +To get your token for the dashboard you can run this command: + +```bash +$ kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}') +``` + +It should print something like: + +```text +Name: admin-user-token-6gl6l +Namespace: kube-system +Labels: +Annotations: kubernetes.io/service-account.name=admin-user + kubernetes.io/service-account.uid=b16afba9-dfec-11e7-bbb9-901b0e532516 + +Type: kubernetes.io/service-account-token + +Data +==== +ca.crt: 1025 bytes +namespace: 11 bytes +token: eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi11c2VyLXRva2VuLTZnbDZsIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImFkbWluLXVzZXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiJiMTZhZmJhOS1kZmVjLTExZTctYmJiOS05MDFiMGU1MzI1MTYiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZS1zeXN0ZW06YWRtaW4tdXNlciJ9.M70CU3lbu3PP4OjhFms8PVL5pQKj-jj4RNSLA4YmQfTXpPUuxqXjiTf094_Rzr0fgN_IVX6gC4fiNUL5ynx9KU-lkPfk0HnX8scxfJNzypL039mpGt0bbe1IXKSIRaq_9VW59Xz-yBUhycYcKPO9RM2Qa1Ax29nqNVko4vLn1_1wPqJ6XSq3GYI8anTzV8Fku4jasUwjrws6Cn6_sPEGmL54sq5R4Z5afUtv-mItTmqZZdxnkRqcJLlg2Y8WbCPogErbsaCDJoABQ7ppaqHetwfM_0yMun6ABOQbIwwl8pspJhpplKwyo700OSpvTT9zlBsu-b35lzXGBRHzv5g_RA +``` + +Grab the token from above and paste it into the [login screen](http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/) + +When you are logged in, you should see sth. like: + +![Dashboard](./dashboard-screenshot.png) + +Feel free to save the login token from above in your password manager. Unlike the `kubeconfig` file, this token does not expire. diff --git a/deployment/old/digital-ocean/dashboard/admin-user.yaml b/deployment/old/digital-ocean/dashboard/admin-user.yaml new file mode 100644 index 0000000..27b6bb8 --- /dev/null +++ b/deployment/old/digital-ocean/dashboard/admin-user.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: admin-user + namespace: kube-system diff --git a/deployment/old/digital-ocean/dashboard/dashboard-screenshot.png b/deployment/old/digital-ocean/dashboard/dashboard-screenshot.png new file mode 100644 index 0000000..6aefb54 Binary files /dev/null and b/deployment/old/digital-ocean/dashboard/dashboard-screenshot.png differ diff --git a/deployment/old/digital-ocean/dashboard/role-binding.yaml b/deployment/old/digital-ocean/dashboard/role-binding.yaml new file mode 100644 index 0000000..faa8927 --- /dev/null +++ b/deployment/old/digital-ocean/dashboard/role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: admin-user +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: admin-user + namespace: kube-system diff --git a/deployment/old/digital-ocean/https/README.md b/deployment/old/digital-ocean/https/README.md new file mode 100644 index 0000000..b306a48 --- /dev/null +++ b/deployment/old/digital-ocean/https/README.md @@ -0,0 +1,126 @@ +## Create Letsencrypt Issuers and Ingress Services + +Copy the configuration templates and change the file according to your needs. + +```bash +# in folder deployment/digital-ocean/https/ +cp templates/issuer.template.yaml ./issuer.yaml +cp templates/ingress.template.yaml ./ingress.yaml +``` + +At least, **change email addresses** in `issuer.yaml`. For sure you also want +to _change the domain name_ in `ingress.yaml`. + +Once you are done, apply the configuration: + +```bash +# in folder deployment/digital-ocean/https/ +$ kubectl apply -f . +``` + +{% hint style="info" %} +CAUTION: It seems that the behaviour of Digital Ocean has changed and the load balancer is not created automatically anymore. +And to create a load balancer costs money. Please refine the following documentation if required. +{% endhint %} + +{% tabs %} +{% tab title="Without Load Balancer" %} + +A solution without a load balance you can find [here](../no-loadbalancer/README.md). + +{% endtab %} +{% tab title="With Digital Ocean Load Balancer" %} + +{% hint style="info" %} +CAUTION: It seems that the behaviour of Digital Ocean has changed and the load balancer is not created automatically anymore. +Please refine the following documentation if required. +{% endhint %} + +In earlier days by now, your cluster should have a load balancer assigned with an external IP +address. On Digital Ocean, this is how it should look like: + +![Screenshot of Digital Ocean dashboard showing external ip address](./ip-address.png) + +If the load balancer isn't created automatically you have to create it your self on Digital Ocean under Networks. +In case you don't need a Digital Ocean load balancer (which costs money by the way) have a look in the tab `Without Load Balancer`. + +{% endtab %} +{% endtabs %} + +Check the ingress server is working correctly: + +```bash +$ curl -kivL -H 'Host: ' 'https://' + +``` + +If the response looks good, configure your domain registrar for the new IP address and the domain. + +Now let's get a valid HTTPS certificate. According to the tutorial above, check your tls certificate for staging: + +```bash +$ kubectl -n ocelot-social describe certificate tls +< +... +Spec: + ... + Issuer Ref: + Group: cert-manager.io + Kind: ClusterIssuer + Name: letsencrypt-staging +... +Events: + +> +$ kubectl -n ocelot-social describe secret tls +< +... +Annotations: ... + cert-manager.io/issuer-kind: ClusterIssuer + cert-manager.io/issuer-name: letsencrypt-staging +... +> +``` + +If everything looks good, update the cluster-issuer of your ingress. Change the annotation `cert-manager.io/cluster-issuer` from `letsencrypt-staging` (for testing by getting a dummy certificate – no blocking by letsencrypt, because of to many request cycles) to `letsencrypt-prod` (for production with a real certificate – possible blocking by letsencrypt for several days, because of to many request cycles) in your ingress configuration in `ingress.yaml`. + +```bash +# in folder deployment/digital-ocean/https/ +$ kubectl apply -f ingress.yaml +``` + +Take a minute and have a look if the certificate is now newly generated by `letsencrypt-prod`, the cluster-issuer for production: + +```bash +$ kubectl -n ocelot-social describe certificate tls +< +... +Spec: + ... + Issuer Ref: + Group: cert-manager.io + Kind: ClusterIssuer + Name: letsencrypt-prod +... +Events: + +> +$ kubectl -n ocelot-social describe secret tls +< +... +Annotations: ... + cert-manager.io/issuer-kind: ClusterIssuer + cert-manager.io/issuer-name: letsencrypt-prod +... +> +``` + +In case the certificate is not newly created delete the former secret to force a refresh: + +```bash +$ kubectl -n ocelot-social delete secret tls +``` + +Now, HTTPS should be configured on your domain. Congrats! + +For troubleshooting have a look at the cert-manager's [Troubleshooting](https://cert-manager.io/docs/faq/troubleshooting/) or [Troubleshooting Issuing ACME Certificates](https://cert-manager.io/docs/faq/acme/). diff --git a/deployment/old/digital-ocean/https/ip-address.png b/deployment/old/digital-ocean/https/ip-address.png new file mode 100644 index 0000000..db52315 Binary files /dev/null and b/deployment/old/digital-ocean/https/ip-address.png differ diff --git a/deployment/old/legacy-migration/README.md b/deployment/old/legacy-migration/README.md new file mode 100644 index 0000000..66100a3 --- /dev/null +++ b/deployment/old/legacy-migration/README.md @@ -0,0 +1,85 @@ +# Legacy data migration + +This setup is **completely optional** and only required if you have data on a +server which is running our legacy code and you want to import that data. It +will import the uploads folder and migrate a dump of the legacy Mongo database +into our new Neo4J graph database. + +## Configure Maintenance-Worker Pod + +Create a configmap with the specific connection data of your legacy server: + +```bash +$ kubectl create configmap maintenance-worker \ + -n ocelot-social \ + --from-literal=SSH_USERNAME=someuser \ + --from-literal=SSH_HOST=yourhost \ + --from-literal=MONGODB_USERNAME=hc-api \ + --from-literal=MONGODB_PASSWORD=secretpassword \ + --from-literal=MONGODB_AUTH_DB=hc_api \ + --from-literal=MONGODB_DATABASE=hc_api \ + --from-literal=UPLOADS_DIRECTORY=/var/www/api/uploads +``` + +Create a secret with your public and private ssh keys. As the [kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/secret/#use-case-pod-with-ssh-keys) points out, you should be careful with your ssh keys. Anyone with access to your cluster will have access to your ssh keys. Better create a new pair with `ssh-keygen` and copy the public key to your legacy server with `ssh-copy-id`: + +```bash +$ kubectl create secret generic ssh-keys \ + -n ocelot-social \ + --from-file=id_rsa=/path/to/.ssh/id_rsa \ + --from-file=id_rsa.pub=/path/to/.ssh/id_rsa.pub \ + --from-file=known_hosts=/path/to/.ssh/known_hosts +``` + +## Deploy a Temporary Maintenance-Worker Pod + +Bring the application into maintenance mode. + +{% hint style="info" %} TODO: implement maintenance mode {% endhint %} + + +Then temporarily delete backend and database deployments + +```bash +$ kubectl -n ocelot-social get deployments +NAME READY UP-TO-DATE AVAILABLE AGE +backend 1/1 1 1 3d11h +neo4j 1/1 1 1 3d11h +webapp 2/2 2 2 73d +$ kubectl -n ocelot-social delete deployment neo4j +deployment.extensions "neo4j" deleted +$ kubectl -n ocelot-social delete deployment backend +deployment.extensions "backend" deleted +``` + +Deploy one-time develop-maintenance-worker pod: + +```bash +# in deployment/legacy-migration/ +$ kubectl apply -f maintenance-worker.yaml +pod/develop-maintenance-worker created +``` + +Import legacy database and uploads: + +```bash +$ kubectl -n ocelot-social exec -it develop-maintenance-worker bash +$ import_legacy_db +$ import_legacy_uploads +$ exit +``` + +Delete the pod when you're done: + +```bash +$ kubectl -n ocelot-social delete pod develop-maintenance-worker +``` + +Oh, and of course you have to get those deleted deployments back. One way of +doing it would be: + +```bash +# in folder deployment/ +$ kubectl apply -f human-connection/deployment-backend.yaml -f human-connection/deployment-neo4j.yaml +``` + diff --git a/deployment/old/legacy-migration/maintenance-worker.yaml b/deployment/old/legacy-migration/maintenance-worker.yaml new file mode 100644 index 0000000..d8b118b --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker.yaml @@ -0,0 +1,40 @@ +--- + kind: Pod + apiVersion: v1 + metadata: + name: develop-maintenance-worker + namespace: ocelot-social + spec: + containers: + - name: develop-maintenance-worker + image: ocelotsocialnetwork/develop-maintenance-worker:latest + imagePullPolicy: Always + resources: + requests: + memory: "2G" + limits: + memory: "8G" + envFrom: + - configMapRef: + name: maintenance-worker + - configMapRef: + name: configmap + volumeMounts: + - name: secret-volume + readOnly: false + mountPath: /root/.ssh + - name: uploads + mountPath: /uploads + - name: neo4j-data + mountPath: /data/ + volumes: + - name: secret-volume + secret: + secretName: ssh-keys + defaultMode: 0400 + - name: uploads + persistentVolumeClaim: + claimName: uploads-claim + - name: neo4j-data + persistentVolumeClaim: + claimName: neo4j-data-claim diff --git a/deployment/old/legacy-migration/maintenance-worker/.dockerignore b/deployment/old/legacy-migration/maintenance-worker/.dockerignore new file mode 100644 index 0000000..59ba63a --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/.dockerignore @@ -0,0 +1 @@ +.ssh/ diff --git a/deployment/old/legacy-migration/maintenance-worker/.gitignore b/deployment/old/legacy-migration/maintenance-worker/.gitignore new file mode 100644 index 0000000..485bc00 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/.gitignore @@ -0,0 +1,2 @@ +.ssh/ +ssh/ \ No newline at end of file diff --git a/deployment/old/legacy-migration/maintenance-worker/Dockerfile b/deployment/old/legacy-migration/maintenance-worker/Dockerfile new file mode 100644 index 0000000..760cc06 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/Dockerfile @@ -0,0 +1,21 @@ +FROM ocelotsocialnetwork/develop-neo4j:latest + +ENV NODE_ENV=maintenance +EXPOSE 7687 7474 + +ENV BUILD_DEPS="gettext" \ + RUNTIME_DEPS="libintl" + +RUN set -x && \ + apk add --update $RUNTIME_DEPS && \ + apk add --virtual build_deps $BUILD_DEPS && \ + cp /usr/bin/envsubst /usr/local/bin/envsubst && \ + apk del build_deps + + +RUN apk upgrade --update +RUN apk add --no-cache mongodb-tools openssh nodejs yarn rsync + +COPY known_hosts /root/.ssh/known_hosts +COPY migration /migration +COPY ./binaries/* /usr/local/bin/ diff --git a/deployment/old/legacy-migration/maintenance-worker/binaries/.env b/deployment/old/legacy-migration/maintenance-worker/binaries/.env new file mode 100644 index 0000000..7739180 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/binaries/.env @@ -0,0 +1,6 @@ +# SSH Access +# SSH_USERNAME='username' +# SSH_HOST='example.org' + +# UPLOADS_DIRECTORY=/var/www/api/uploads +OUTPUT_DIRECTORY='/uploads/' \ No newline at end of file diff --git a/deployment/old/legacy-migration/maintenance-worker/binaries/idle b/deployment/old/legacy-migration/maintenance-worker/binaries/idle new file mode 100755 index 0000000..f5b1b24 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/binaries/idle @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +tail -f /dev/null diff --git a/deployment/old/legacy-migration/maintenance-worker/binaries/import_legacy_db b/deployment/old/legacy-migration/maintenance-worker/binaries/import_legacy_db new file mode 100755 index 0000000..6ffdf8e --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/binaries/import_legacy_db @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -e +for var in "SSH_USERNAME" "SSH_HOST" "MONGODB_USERNAME" "MONGODB_PASSWORD" "MONGODB_DATABASE" "MONGODB_AUTH_DB" +do + if [[ -z "${!var}" ]]; then + echo "${var} is undefined" + exit 1 + fi +done + +/migration/mongo/export.sh +/migration/neo4j/import.sh diff --git a/deployment/old/legacy-migration/maintenance-worker/binaries/import_legacy_uploads b/deployment/old/legacy-migration/maintenance-worker/binaries/import_legacy_uploads new file mode 100755 index 0000000..5c0b67d --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/binaries/import_legacy_uploads @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -e + +# import .env config +set -o allexport +source $(dirname "$0")/.env +set +o allexport + +for var in "SSH_USERNAME" "SSH_HOST" "UPLOADS_DIRECTORY" +do + if [[ -z "${!var}" ]]; then + echo "${var} is undefined" + exit 1 + fi +done + +rsync --archive --update --verbose ${SSH_USERNAME}@${SSH_HOST}:${UPLOADS_DIRECTORY}/ ${OUTPUT_DIRECTORY} diff --git a/deployment/old/legacy-migration/maintenance-worker/known_hosts b/deployment/old/legacy-migration/maintenance-worker/known_hosts new file mode 100644 index 0000000..947840c --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/known_hosts @@ -0,0 +1,3 @@ +|1|GuOYlVEhTowidPs18zj9p5F2j3o=|sDHJYLz9Ftv11oXeGEjs7SpVyg0= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM5N29bI5CeKu1/RBPyM2fwyf7fuajOO+tyhKe1+CC2sZ1XNB5Ff6t6MtCLNRv2mUuvzTbW/HkisDiA5tuXUHOk= +|1|2KP9NV+Q5g2MrtjAeFSVcs8YeOI=|nf3h4wWVwC4xbBS1kzgzE2tBldk= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNhRK6BeIEUxXlS0z/pOfkUkSPfn33g4J1U3L+MyUQYHm+7agT08799ANJhnvELKE1tt4Vx80I9UR81oxzZcy3E= +|1|HonYIRNhKyroUHPKU1HSZw0+Qzs=|5T1btfwFBz2vNSldhqAIfTbfIgQ= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNhRK6BeIEUxXlS0z/pOfkUkSPfn33g4J1U3L+MyUQYHm+7agT08799ANJhnvELKE1tt4Vx80I9UR81oxzZcy3E= diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/mongo/.env b/deployment/old/legacy-migration/maintenance-worker/migration/mongo/.env new file mode 100644 index 0000000..376cb56 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/mongo/.env @@ -0,0 +1,17 @@ +# SSH Access +# SSH_USERNAME='username' +# SSH_HOST='example.org' + +# Mongo DB on Remote Maschine +# MONGODB_USERNAME='mongouser' +# MONGODB_PASSWORD='mongopassword' +# MONGODB_DATABASE='mongodatabase' +# MONGODB_AUTH_DB='admin' + +# Export Settings +# On Windows this resolves to C:\Users\dornhoeschen\AppData\Local\Temp\mongo-export (MinGW) +EXPORT_PATH='/tmp/mongo-export/' +EXPORT_MONGOEXPORT_BIN='mongoexport' +MONGO_EXPORT_SPLIT_SIZE=6000 +# On Windows use something like this +# EXPORT_MONGOEXPORT_BIN='C:\Program Files\MongoDB\Server\3.6\bin\mongoexport.exe' diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/mongo/export.sh b/deployment/old/legacy-migration/maintenance-worker/migration/mongo/export.sh new file mode 100755 index 0000000..b56ace8 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/mongo/export.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -e + +# import .env config +set -o allexport +source $(dirname "$0")/.env +set +o allexport + +# Export collection function defintion +function export_collection () { + "${EXPORT_MONGOEXPORT_BIN}" --db ${MONGODB_DATABASE} --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --collection $1 --out "${EXPORT_PATH}$1.json" + mkdir -p ${EXPORT_PATH}splits/$1/ + split -l ${MONGO_EXPORT_SPLIT_SIZE} -a 3 ${EXPORT_PATH}$1.json ${EXPORT_PATH}splits/$1/ +} + +# Export collection with query function defintion +function export_collection_query () { + "${EXPORT_MONGOEXPORT_BIN}" --db ${MONGODB_DATABASE} --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --collection $1 --out "${EXPORT_PATH}$1_$3.json" --query "$2" + mkdir -p ${EXPORT_PATH}splits/$1_$3/ + split -l ${MONGO_EXPORT_SPLIT_SIZE} -a 3 ${EXPORT_PATH}$1_$3.json ${EXPORT_PATH}splits/$1_$3/ +} + +# Delete old export & ensure directory +rm -rf ${EXPORT_PATH}* +mkdir -p ${EXPORT_PATH} + +# Open SSH Tunnel +ssh -4 -M -S my-ctrl-socket -fnNT -L 27018:localhost:27017 -l ${SSH_USERNAME} ${SSH_HOST} + +# Export all Data from the Alpha to json and split them up +export_collection "badges" +export_collection "categories" +export_collection "comments" +export_collection_query "contributions" '{"type": "DELETED"}' "DELETED" +export_collection_query "contributions" '{"type": "post"}' "post" +# export_collection_query "contributions" '{"type": "cando"}' "cando" +export_collection "emotions" +# export_collection_query "follows" '{"foreignService": "organizations"}' "organizations" +export_collection_query "follows" '{"foreignService": "users"}' "users" +# export_collection "invites" +# export_collection "organizations" +# export_collection "pages" +# export_collection "projects" +# export_collection "settings" +export_collection "shouts" +# export_collection "status" +export_collection_query "users" '{"isVerified": true }' "verified" +# export_collection "userscandos" +# export_collection "usersettings" + +# Close SSH Tunnel +ssh -S my-ctrl-socket -O check -l ${SSH_USERNAME} ${SSH_HOST} +ssh -S my-ctrl-socket -O exit -l ${SSH_USERNAME} ${SSH_HOST} diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/.env b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/.env new file mode 100644 index 0000000..16220f3 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/.env @@ -0,0 +1,16 @@ +# Neo4J Settings +# NEO4J_USERNAME='neo4j' +# NEO4J_PASSWORD='letmein' + +# Import Settings +# On Windows this resolves to C:\Users\dornhoeschen\AppData\Local\Temp\mongo-export (MinGW) +IMPORT_PATH='/tmp/mongo-export/' +IMPORT_CHUNK_PATH='/tmp/mongo-export/splits/' + +IMPORT_CHUNK_PATH_CQL='/tmp/mongo-export/splits/' +# On Windows this path needs to be windows style since the cypher-shell runs native - note the forward slash +# IMPORT_CHUNK_PATH_CQL='C:/Users/dornhoeschen/AppData/Local/Temp/mongo-export/splits/' + +IMPORT_CYPHERSHELL_BIN='cypher-shell' +# On Windows use something like this +# IMPORT_CYPHERSHELL_BIN='C:\Program Files\neo4j-community\bin\cypher-shell.bat' \ No newline at end of file diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/badges/badges.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/badges/badges.cql new file mode 100644 index 0000000..adf63dc --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/badges/badges.cql @@ -0,0 +1,52 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[?] image: { +[?] path: { // Path is incorrect in Nitro - is icon the correct name for this field? +[X] type: String, +[X] required: true + }, +[ ] alt: { // If we use an image - should we not have an alt? +[ ] type: String, +[ ] required: true + } + }, +[?] status: { +[X] type: String, +[X] enum: ['permanent', 'temporary'], +[ ] default: 'permanent', // Default value is missing in Nitro +[X] required: true + }, +[?] type: { +[?] type: String, // in nitro this is a defined enum - seems good for now +[X] required: true + }, +[X] id: { +[X] type: String, +[X] required: true + }, +[?] createdAt: { +[?] type: Date, // Type is modeled as string in Nitro which is incorrect +[ ] default: Date.now // Default value is missing in Nitro + }, +[?] updatedAt: { +[?] type: Date, // Type is modeled as string in Nitro which is incorrect +[ ] default: Date.now // Default value is missing in Nitro + } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as badge +MERGE(b:Badge {id: badge._id["$oid"]}) +ON CREATE SET +b.id = badge.key, +b.type = badge.type, +b.icon = replace(badge.image.path, 'https://api-alpha.human-connection.org', ''), +b.status = badge.status, +b.createdAt = badge.createdAt.`$date`, +b.updatedAt = badge.updatedAt.`$date` +; diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/badges/delete.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/badges/delete.cql new file mode 100644 index 0000000..2a6f8c2 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/badges/delete.cql @@ -0,0 +1 @@ +MATCH (n:Badge) DETACH DELETE n; \ No newline at end of file diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/categories/categories.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/categories/categories.cql new file mode 100644 index 0000000..5d49588 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/categories/categories.cql @@ -0,0 +1,129 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[X] title: { +[X] type: String, +[X] required: true + }, +[?] slug: { +[X] type: String, +[ ] required: true, // Not required in Nitro +[ ] unique: true // Unique value is not enforced in Nitro? + }, +[?] icon: { // Nitro adds required: true +[X] type: String, +[ ] unique: true // Unique value is not enforced in Nitro? + }, +[?] createdAt: { +[?] type: Date, // Type is modeled as string in Nitro which is incorrect +[ ] default: Date.now // Default value is missing in Nitro + }, +[?] updatedAt: { +[?] type: Date, // Type is modeled as string in Nitro which is incorrect +[ ] default: Date.now // Default value is missing in Nitro + } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as category +MERGE(c:Category {id: category._id["$oid"]}) +ON CREATE SET +c.name = category.title, +c.slug = category.slug, +c.icon = category.icon, +c.createdAt = category.createdAt.`$date`, +c.updatedAt = category.updatedAt.`$date` +; + +// Transform icon names +MATCH (c:Category) +WHERE (c.icon = "categories-justforfun") +SET c.icon = 'smile' +SET c.slug = 'just-for-fun' +; + +MATCH (c:Category) +WHERE (c.icon = "categories-luck") +SET c.icon = 'heart-o' +SET c.slug = 'happiness-values' +; + +MATCH (c:Category) +WHERE (c.icon = "categories-health") +SET c.icon = 'medkit' +; + +MATCH (c:Category) +WHERE (c.icon = "categories-environment") +SET c.icon = 'tree' +; + +MATCH (c:Category) +WHERE (c.icon = "categories-animal-justice") +SET c.icon = 'paw' +SET c.slug = 'animal-protection' +; + +MATCH (c:Category) +WHERE (c.icon = "categories-human-rights") +SET c.icon = 'balance-scale' +SET c.slug = 'human-rights-justice' + +; + +MATCH (c:Category) +WHERE (c.icon = "categories-education") +SET c.icon = 'graduation-cap' +; + +MATCH (c:Category) +WHERE (c.icon = "categories-cooperation") +SET c.icon = 'users' +; + +MATCH (c:Category) +WHERE (c.icon = "categories-politics") +SET c.icon = 'university' +; + +MATCH (c:Category) +WHERE (c.icon = "categories-economy") +SET c.icon = 'money' +; + +MATCH (c:Category) +WHERE (c.icon = "categories-technology") +SET c.icon = 'flash' +; + +MATCH (c:Category) +WHERE (c.icon = "categories-internet") +SET c.icon = 'mouse-pointer' +SET c.slug = 'it-internet-data-privacy' +; + +MATCH (c:Category) +WHERE (c.icon = "categories-art") +SET c.icon = 'paint-brush' +; + +MATCH (c:Category) +WHERE (c.icon = "categories-freedom-of-speech") +SET c.icon = 'bullhorn' +SET c.slug = 'freedom-of-speech' +; + +MATCH (c:Category) +WHERE (c.icon = "categories-sustainability") +SET c.icon = 'shopping-cart' +; + +MATCH (c:Category) +WHERE (c.icon = "categories-peace") +SET c.icon = 'angellist' +SET c.slug = 'global-peace-nonviolence' +; diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/categories/delete.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/categories/delete.cql new file mode 100644 index 0000000..c06b5ef --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/categories/delete.cql @@ -0,0 +1 @@ +MATCH (n:Category) DETACH DELETE n; \ No newline at end of file diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/comments/comments.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/comments/comments.cql new file mode 100644 index 0000000..083f9f7 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/comments/comments.cql @@ -0,0 +1,67 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[?] userId: { +[X] type: String, +[ ] required: true, // Not required in Nitro +[-] index: true + }, +[?] contributionId: { +[X] type: String, +[ ] required: true, // Not required in Nitro +[-] index: true + }, +[X] content: { +[X] type: String, +[X] required: true + }, +[?] contentExcerpt: { // Generated from content +[X] type: String, +[ ] required: true // Not required in Nitro + }, +[ ] hasMore: { type: Boolean }, +[ ] upvotes: { +[ ] type: Array, +[ ] default: [] + }, +[ ] upvoteCount: { +[ ] type: Number, +[ ] default: 0 + }, +[?] deleted: { +[X] type: Boolean, +[ ] default: false, // Default value is missing in Nitro +[-] index: true + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as comment +MERGE (c:Comment {id: comment._id["$oid"]}) +ON CREATE SET +c.content = comment.content, +c.contentExcerpt = comment.contentExcerpt, +c.deleted = comment.deleted, +c.createdAt = comment.createdAt.`$date`, +c.updatedAt = comment.updatedAt.`$date`, +c.disabled = false +WITH c, comment, comment.contributionId as postId +MATCH (post:Post {id: postId}) +WITH c, post, comment.userId as userId +MATCH (author:User {id: userId}) +MERGE (c)-[:COMMENTS]->(post) +MERGE (author)-[:WROTE]->(c) +; diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/comments/delete.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/comments/delete.cql new file mode 100644 index 0000000..c4a7961 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/comments/delete.cql @@ -0,0 +1 @@ +MATCH (n:Comment) DETACH DELETE n; \ No newline at end of file diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/contributions/contributions.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/contributions/contributions.cql new file mode 100644 index 0000000..f09b5ad --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/contributions/contributions.cql @@ -0,0 +1,156 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro +[?] { //Modeled incorrect as Post +[?] userId: { +[X] type: String, +[ ] required: true, // Not required in Nitro +[-] index: true + }, +[ ] organizationId: { +[ ] type: String, +[-] index: true + }, +[X] categoryIds: { +[X] type: Array, +[-] index: true + }, +[X] title: { +[X] type: String, +[X] required: true + }, +[?] slug: { // Generated from title +[X] type: String, +[ ] required: true, // Not required in Nitro +[?] unique: true, // Unique value is not enforced in Nitro? +[-] index: true + }, +[ ] type: { // db.getCollection('contributions').distinct('type') -> 'DELETED', 'cando', 'post' +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] cando: { +[ ] difficulty: { +[ ] type: String, +[ ] enum: ['easy', 'medium', 'hard'] + }, +[ ] reasonTitle: { type: String }, +[ ] reason: { type: String } + }, +[X] content: { +[X] type: String, +[X] required: true + }, +[?] contentExcerpt: { // Generated from content +[X] type: String, +[?] required: true // Not required in Nitro + }, +[ ] hasMore: { type: Boolean }, +[X] teaserImg: { type: String }, +[ ] language: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] shoutCount: { +[ ] type: Number, +[ ] default: 0, +[-] index: true + }, +[ ] meta: { +[ ] hasVideo: { +[ ] type: Boolean, +[ ] default: false + }, +[ ] embedds: { +[ ] type: Object, +[ ] default: {} + } + }, +[?] visibility: { +[X] type: String, +[X] enum: ['public', 'friends', 'private'], +[ ] default: 'public', // Default value is missing in Nitro +[-] index: true + }, +[?] isEnabled: { +[X] type: Boolean, +[ ] default: true, // Default value is missing in Nitro +[-] index: true + }, +[?] tags: { type: Array }, // ensure this is working properly +[ ] emotions: { +[ ] type: Object, +[-] index: true, +[ ] default: { +[ ] angry: { +[ ] count: 0, +[ ] percent: 0 +[ ] }, +[ ] cry: { +[ ] count: 0, +[ ] percent: 0 +[ ] }, +[ ] surprised: { +[ ] count: 0, +[ ] percent: 0 + }, +[ ] happy: { +[ ] count: 0, +[ ] percent: 0 + }, +[ ] funny: { +[ ] count: 0, +[ ] percent: 0 + } + } + }, +[?] deleted: { // THis field is not always present in the alpha-data +[?] type: Boolean, +[ ] default: false, // Default value is missing in Nitro +[-] index: true + }, +[?] createdAt: { +[?] type: Date, // Type is modeled as string in Nitro which is incorrect +[ ] default: Date.now // Default value is missing in Nitro + }, +[?] updatedAt: { +[?] type: Date, // Type is modeled as string in Nitro which is incorrect +[ ] default: Date.now // Default value is missing in Nitro + }, +[ ] wasSeeded: { type: Boolean } + } +*/ +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as post +MERGE (p:Post {id: post._id["$oid"]}) +ON CREATE SET +p.title = post.title, +p.slug = post.slug, +p.image = replace(post.teaserImg, 'https://api-alpha.human-connection.org', ''), +p.content = post.content, +p.contentExcerpt = post.contentExcerpt, +p.visibility = toLower(post.visibility), +p.createdAt = post.createdAt.`$date`, +p.updatedAt = post.updatedAt.`$date`, +p.deleted = COALESCE(post.deleted, false), +p.disabled = COALESCE(NOT post.isEnabled, false) +WITH p, post +MATCH (u:User {id: post.userId}) +MERGE (u)-[:WROTE]->(p) +WITH p, post, post.categoryIds as categoryIds +UNWIND categoryIds AS categoryId +MATCH (c:Category {id: categoryId}) +MERGE (p)-[:CATEGORIZED]->(c) +WITH p, post.tags AS tags +UNWIND tags AS tag +WITH apoc.text.replace(tag, '[^\\p{L}0-9]', '') as tagNoSpacesAllowed +CALL apoc.when(tagNoSpacesAllowed =~ '^((\\p{L}+[\\p{L}0-9]*)|([0-9]+\\p{L}+[\\p{L}0-9]*))$', 'RETURN tagNoSpacesAllowed', '', {tagNoSpacesAllowed: tagNoSpacesAllowed}) +YIELD value as validated +WHERE validated.tagNoSpacesAllowed IS NOT NULL +MERGE (t:Tag { id: validated.tagNoSpacesAllowed, disabled: false, deleted: false }) +MERGE (p)-[:TAGGED]->(t) +; diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/contributions/delete.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/contributions/delete.cql new file mode 100644 index 0000000..70adad6 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/contributions/delete.cql @@ -0,0 +1,2 @@ +MATCH (n:Post) DETACH DELETE n; +MATCH (n:Tag) DETACH DELETE n; \ No newline at end of file diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/delete_all.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/delete_all.cql new file mode 100644 index 0000000..d018713 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/delete_all.cql @@ -0,0 +1 @@ +MATCH (n) DETACH DELETE n; \ No newline at end of file diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/emotions/delete.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/emotions/delete.cql new file mode 100644 index 0000000..18fb669 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/emotions/delete.cql @@ -0,0 +1 @@ +MATCH (u:User)-[e:EMOTED]->(c:Post) DETACH DELETE e; \ No newline at end of file diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/emotions/emotions.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/emotions/emotions.cql new file mode 100644 index 0000000..06341f2 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/emotions/emotions.cql @@ -0,0 +1,58 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[X] userId: { +[X] type: String, +[X] required: true, +[-] index: true + }, +[X] contributionId: { +[X] type: String, +[X] required: true, +[-] index: true + }, +[?] rated: { +[X] type: String, +[ ] required: true, +[?] enum: ['funny', 'happy', 'surprised', 'cry', 'angry'] + }, +[X] createdAt: { +[X] type: Date, +[X] default: Date.now + }, +[X] updatedAt: { +[X] type: Date, +[X] default: Date.now + }, +[-] wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as emotion +MATCH (u:User {id: emotion.userId}), + (c:Post {id: emotion.contributionId}) +MERGE (u)-[e:EMOTED { + id: emotion._id["$oid"], + emotion: emotion.rated, + createdAt: datetime(emotion.createdAt.`$date`), + updatedAt: datetime(emotion.updatedAt.`$date`) + }]->(c) +RETURN e; +/* + // Queries + // user sets an emotion emotion: + // MERGE (u)-[e:EMOTED {id: ..., emotion: "funny", createdAt: ..., updatedAt: ...}]->(c) + // user removes emotion + // MATCH (u)-[e:EMOTED]->(c) DELETE e + // contribution distributions over every `emotion` property value for one post + // MATCH (u:User)-[e:EMOTED]->(c:Post {id: "5a70bbc8508f5b000b443b1a"}) RETURN e.emotion,COUNT(e.emotion) + // contribution distributions over every `emotion` property value for one user (advanced - "whats the primary emotion used by the user?") + // MATCH (u:User{id:"5a663b1ac64291000bf302a1"})-[e:EMOTED]->(c:Post) RETURN e.emotion,COUNT(e.emotion) + // contribution distributions over every `emotion` property value for all posts created by one user (advanced - "how do others react to my contributions?") + // MATCH (u:User)-[e:EMOTED]->(c:Post)<-[w:WROTE]-(a:User{id:"5a663b1ac64291000bf302a1"}) RETURN e.emotion,COUNT(e.emotion) + // if we can filter the above an a variable timescale that would be great (should be possible on createdAt and updatedAt fields) +*/ \ No newline at end of file diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/follows/delete.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/follows/delete.cql new file mode 100644 index 0000000..3de01f8 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/follows/delete.cql @@ -0,0 +1 @@ +MATCH (u1:User)-[f:FOLLOWS]->(u2:User) DETACH DELETE f; \ No newline at end of file diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/follows/follows.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/follows/follows.cql new file mode 100644 index 0000000..fac858a --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/follows/follows.cql @@ -0,0 +1,36 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[?] userId: { +[-] type: String, +[ ] required: true, +[-] index: true + }, +[?] foreignId: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[?] foreignService: { // db.getCollection('follows').distinct('foreignService') returns 'organizations' and 'users' +[ ] type: String, +[ ] required: true, +[ ] index: true + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } + index: +[?] { userId: 1, foreignId: 1, foreignService: 1 },{ unique: true } // is the unique constrain modeled? +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as follow +MATCH (u1:User {id: follow.userId}), (u2:User {id: follow.foreignId}) +MERGE (u1)-[:FOLLOWS]->(u2) +; diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/import.sh b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/import.sh new file mode 100755 index 0000000..ccb22da --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/import.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +set -e + +# import .env config +set -o allexport +source $(dirname "$0")/.env +set +o allexport + +# Delete collection function defintion +function delete_collection () { + # Delete from Database + echo "Delete $2" + "${IMPORT_CYPHERSHELL_BIN}" < $(dirname "$0")/$1/delete.cql > /dev/null + # Delete index file + rm -f "${IMPORT_PATH}splits/$2.index" +} + +# Import collection function defintion +function import_collection () { + # index file of those chunks we have already imported + INDEX_FILE="${IMPORT_PATH}splits/$1.index" + # load index file + if [ -f "$INDEX_FILE" ]; then + readarray -t IMPORT_INDEX <$INDEX_FILE + else + declare -a IMPORT_INDEX + fi + # for each chunk import data + for chunk in ${IMPORT_PATH}splits/$1/* + do + CHUNK_FILE_NAME=$(basename "${chunk}") + # does the index not contain the chunk file name? + if [[ ! " ${IMPORT_INDEX[@]} " =~ " ${CHUNK_FILE_NAME} " ]]; then + # calculate the path of the chunk + export IMPORT_CHUNK_PATH_CQL_FILE="${IMPORT_CHUNK_PATH_CQL}$1/${CHUNK_FILE_NAME}" + # load the neo4j command and replace file variable with actual path + NEO4J_COMMAND="$(envsubst '${IMPORT_CHUNK_PATH_CQL_FILE}' < $(dirname "$0")/$2)" + # run the import of the chunk + echo "Import $1 ${CHUNK_FILE_NAME} (${chunk})" + echo "${NEO4J_COMMAND}" | "${IMPORT_CYPHERSHELL_BIN}" > /dev/null + # add file to array and file + IMPORT_INDEX+=("${CHUNK_FILE_NAME}") + echo "${CHUNK_FILE_NAME}" >> ${INDEX_FILE} + else + echo "Skipping $1 ${CHUNK_FILE_NAME} (${chunk})" + fi + done +} + +# Time variable +SECONDS=0 + +# Delete all Neo4J Database content +echo "Deleting Database Contents" +delete_collection "badges" "badges" +delete_collection "categories" "categories" +delete_collection "users" "users" +delete_collection "follows" "follows_users" +delete_collection "contributions" "contributions_post" +delete_collection "contributions" "contributions_cando" +delete_collection "shouts" "shouts" +delete_collection "comments" "comments" +delete_collection "emotions" "emotions" + +#delete_collection "invites" +#delete_collection "notifications" +#delete_collection "organizations" +#delete_collection "pages" +#delete_collection "projects" +#delete_collection "settings" +#delete_collection "status" +#delete_collection "systemnotifications" +#delete_collection "userscandos" +#delete_collection "usersettings" +echo "DONE" + +# Import Data +echo "Start Importing Data" +import_collection "badges" "badges/badges.cql" +import_collection "categories" "categories/categories.cql" +import_collection "users_verified" "users/users.cql" +import_collection "follows_users" "follows/follows.cql" +#import_collection "follows_organizations" "follows/follows.cql" +import_collection "contributions_post" "contributions/contributions.cql" +#import_collection "contributions_cando" "contributions/contributions.cql" +#import_collection "contributions_DELETED" "contributions/contributions.cql" +import_collection "shouts" "shouts/shouts.cql" +import_collection "comments" "comments/comments.cql" +import_collection "emotions" "emotions/emotions.cql" + +# import_collection "invites" +# import_collection "notifications" +# import_collection "organizations" +# import_collection "pages" +# import_collection "systemnotifications" +# import_collection "userscandos" +# import_collection "usersettings" + +# does only contain dummy data +# import_collection "projects" + +# does only contain alpha specifc data +# import_collection "status +# import_collection "settings"" + +echo "DONE" + +echo "Time elapsed: $SECONDS seconds" diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/invites/delete.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/invites/delete.cql new file mode 100644 index 0000000..e69de29 diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/invites/invites.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/invites/invites.cql new file mode 100644 index 0000000..f4a5bf0 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/invites/invites.cql @@ -0,0 +1,39 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] email: { +[ ] type: String, +[ ] required: true, +[-] index: true, +[ ] unique: true + }, +[ ] code: { +[ ] type: String, +[-] index: true, +[ ] required: true + }, +[ ] role: { +[ ] type: String, +[ ] enum: ['admin', 'moderator', 'manager', 'editor', 'user'], +[ ] default: 'user' + }, +[ ] invitedByUserId: { type: String }, +[ ] language: { type: String }, +[ ] badgeIds: [], +[ ] wasUsed: { +[ ] type: Boolean, +[-] index: true + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as invite; diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/notifications/delete.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/notifications/delete.cql new file mode 100644 index 0000000..e69de29 diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/notifications/notifications.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/notifications/notifications.cql new file mode 100644 index 0000000..aa6ac8e --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/notifications/notifications.cql @@ -0,0 +1,48 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] userId: { // User this notification is sent to +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] type: { +[ ] type: String, +[ ] required: true, +[ ] enum: ['comment','comment-mention','contribution-mention','following-contribution'] + }, +[ ] relatedUserId: { +[ ] type: String, +[-] index: true + }, +[ ] relatedContributionId: { +[ ] type: String, +[-] index: true + }, +[ ] relatedOrganizationId: { +[ ] type: String, +[-] index: true + }, +[ ] relatedCommentId: {type: String }, +[ ] unseen: { +[ ] type: Boolean, +[ ] default: true, +[-] index: true + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as notification; diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/organizations/delete.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/organizations/delete.cql new file mode 100644 index 0000000..e69de29 diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/organizations/organizations.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/organizations/organizations.cql new file mode 100644 index 0000000..e473e69 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/organizations/organizations.cql @@ -0,0 +1,137 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] name: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] slug: { +[ ] type: String, +[ ] required: true, +[ ] unique: true, +[-] index: true + }, +[ ] followersCounts: { +[ ] users: { +[ ] type: Number, +[ ] default: 0 + }, +[ ] organizations: { +[ ] type: Number, +[ ] default: 0 + }, +[ ] projects: { +[ ] type: Number, +[ ] default: 0 + } + }, +[ ] followingCounts: { +[ ] users: { +[ ] type: Number, +[ ] default: 0 + }, +[ ] organizations: { +[ ] type: Number, +[ ] default: 0 + }, +[ ] projects: { +[ ] type: Number, +[ ] default: 0 + } + }, +[ ] categoryIds: { +[ ] type: Array, +[ ] required: true, +[-] index: true + }, +[ ] logo: { type: String }, +[ ] coverImg: { type: String }, +[ ] userId: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] description: { +[ ] type: String, +[ ] required: true + }, +[ ] descriptionExcerpt: { type: String }, // will be generated automatically +[ ] publicEmail: { type: String }, +[ ] url: { type: String }, +[ ] type: { +[ ] type: String, +[-] index: true, +[ ] enum: ['ngo', 'npo', 'goodpurpose', 'ev', 'eva'] + }, +[ ] language: { +[ ] type: String, +[ ] required: true, +[ ] default: 'de', +[-] index: true + }, +[ ] addresses: { +[ ] type: [{ +[ ] street: { +[ ] type: String, +[ ] required: true + }, +[ ] zipCode: { +[ ] type: String, +[ ] required: true + }, +[ ] city: { +[ ] type: String, +[ ] required: true + }, +[ ] country: { +[ ] type: String, +[ ] required: true + }, +[ ] lat: { +[ ] type: Number, +[ ] required: true + }, +[ ] lng: { +[ ] type: Number, +[ ] required: true + } + }], +[ ] default: [] + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] isEnabled: { +[ ] type: Boolean, +[ ] default: false, +[-] index: true + }, +[ ] reviewedBy: { +[ ] type: String, +[ ] default: null, +[-] index: true + }, +[ ] tags: { +[ ] type: Array, +[-] index: true + }, +[ ] deleted: { +[ ] type: Boolean, +[ ] default: false, +[-] index: true + }, +[ ] wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as organisation; diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/pages/delete.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/pages/delete.cql new file mode 100644 index 0000000..e69de29 diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/pages/pages.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/pages/pages.cql new file mode 100644 index 0000000..1822313 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/pages/pages.cql @@ -0,0 +1,55 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] title: { +[ ] type: String, +[ ] required: true + }, +[ ] slug: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] type: { +[ ] type: String, +[ ] required: true, +[ ] default: 'page' + }, +[ ] key: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] content: { +[ ] type: String, +[ ] required: true + }, +[ ] language: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] active: { +[ ] type: Boolean, +[ ] default: true, +[-] index: true + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } + index: +[ ] { slug: 1, language: 1 },{ unique: true } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as page; diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/projects/delete.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/projects/delete.cql new file mode 100644 index 0000000..e69de29 diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/projects/projects.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/projects/projects.cql new file mode 100644 index 0000000..ed859c1 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/projects/projects.cql @@ -0,0 +1,44 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] name: { +[ ] type: String, +[ ] required: true + }, +[ ] slug: { type: String }, +[ ] followerIds: [], +[ ] categoryIds: { type: Array }, +[ ] logo: { type: String }, +[ ] userId: { +[ ] type: String, +[ ] required: true + }, +[ ] description: { +[ ] type: String, +[ ] required: true + }, +[ ] content: { +[ ] type: String, +[ ] required: true + }, +[ ] addresses: { +[ ] type: Array, +[ ] default: [] + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as project; diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/settings/delete.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/settings/delete.cql new file mode 100644 index 0000000..e69de29 diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/settings/settings.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/settings/settings.cql new file mode 100644 index 0000000..1d557d3 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/settings/settings.cql @@ -0,0 +1,36 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] key: { +[ ] type: String, +[ ] default: 'system', +[-] index: true, +[ ] unique: true + }, +[ ] invites: { +[ ] userCanInvite: { +[ ] type: Boolean, +[ ] required: true, +[ ] default: false + }, +[ ] maxInvitesByUser: { +[ ] type: Number, +[ ] required: true, +[ ] default: 1 + }, +[ ] onlyUserWithBadgesCanInvite: { +[ ] type: Array, +[ ] default: [] + } + }, +[ ] maintenance: false + }, { +[ ] timestamps: true + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as setting; diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/shouts/delete.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/shouts/delete.cql new file mode 100644 index 0000000..21c2e1f --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/shouts/delete.cql @@ -0,0 +1 @@ +// this is just a relation between users and contributions - no need to delete \ No newline at end of file diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/shouts/shouts.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/shouts/shouts.cql new file mode 100644 index 0000000..d370b4b --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/shouts/shouts.cql @@ -0,0 +1,36 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[?] userId: { +[X] type: String, +[ ] required: true, // Not required in Nitro +[-] index: true + }, +[?] foreignId: { +[X] type: String, +[ ] required: true, // Not required in Nitro +[-] index: true + }, +[?] foreignService: { // db.getCollection('shots').distinct('foreignService') returns 'contributions' +[X] type: String, +[ ] required: true, // Not required in Nitro +[-] index: true + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } + index: +[?] { userId: 1, foreignId: 1 },{ unique: true } // is the unique constrain modeled? +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as shout +MATCH (u:User {id: shout.userId}), (p:Post {id: shout.foreignId}) +MERGE (u)-[:SHOUTED]->(p) +; diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/status/delete.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/status/delete.cql new file mode 100644 index 0000000..e69de29 diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/status/status.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/status/status.cql new file mode 100644 index 0000000..010c2ca --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/status/status.cql @@ -0,0 +1,19 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] maintenance: { +[ ] type: Boolean, +[ ] default: false + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as status; diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications/delete.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications/delete.cql new file mode 100644 index 0000000..e69de29 diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications/systemnotifications.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications/systemnotifications.cql new file mode 100644 index 0000000..4bd33eb --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications/systemnotifications.cql @@ -0,0 +1,61 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] type: { +[ ] type: String, +[ ] default: 'info', +[ ] required: true, +[-] index: true + }, +[ ] title: { +[ ] type: String, +[ ] required: true + }, +[ ] content: { +[ ] type: String, +[ ] required: true + }, +[ ] slot: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] language: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] permanent: { +[ ] type: Boolean, +[ ] default: false + }, +[ ] requireConfirmation: { +[ ] type: Boolean, +[ ] default: false + }, +[ ] active: { +[ ] type: Boolean, +[ ] default: true, +[-] index: true + }, +[ ] totalCount: { +[ ] type: Number, +[ ] default: 0 + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as systemnotification; diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/users/delete.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/users/delete.cql new file mode 100644 index 0000000..32679f6 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/users/delete.cql @@ -0,0 +1,2 @@ +MATCH (n:User) DETACH DELETE n; +MATCH (e:EmailAddress) DETACH DELETE e; diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/users/users.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/users/users.cql new file mode 100644 index 0000000..02dff08 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/users/users.cql @@ -0,0 +1,124 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[?] email: { +[X] type: String, +[-] index: true, +[X] required: true, +[?] unique: true //unique constrain missing in Nitro + }, +[?] password: { // Not required in Alpha -> verify if always present +[X] type: String + }, +[X] name: { type: String }, +[X] slug: { +[X] type: String, +[-] index: true + }, +[ ] gender: { type: String }, +[ ] followersCounts: { +[ ] users: { +[ ] type: Number, +[ ] default: 0 + }, +[ ] organizations: { +[ ] type: Number, +[ ] default: 0 + }, +[ ] projects: { +[ ] type: Number, +[ ] default: 0 + } + }, +[ ] followingCounts: { +[ ] users: { +[ ] type: Number, +[ ] default: 0 + }, +[ ] organizations: { +[ ] type: Number, +[ ] default: 0 + }, +[ ] projects: { +[ ] type: Number, +[ ] default: 0 + } + }, +[ ] timezone: { type: String }, +[X] avatar: { type: String }, +[X] coverImg: { type: String }, +[ ] doiToken: { type: String }, +[ ] confirmedAt: { type: Date }, +[?] badgeIds: [], // Verify this is working properly +[?] deletedAt: { type: Date }, // The Date of deletion is not saved in Nitro +[?] createdAt: { +[?] type: Date, // Modeled as String in Nitro +[ ] default: Date.now // Default value is missing in Nitro + }, +[?] updatedAt: { +[?] type: Date, // Modeled as String in Nitro +[ ] default: Date.now // Default value is missing in Nitro + }, +[ ] lastActiveAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] isVerified: { type: Boolean }, +[?] role: { +[X] type: String, +[-] index: true, +[?] enum: ['admin', 'moderator', 'manager', 'editor', 'user'], // missing roles manager & editor in Nitro +[ ] default: 'user' // Default value is missing in Nitro + }, +[ ] verifyToken: { type: String }, +[ ] verifyShortToken: { type: String }, +[ ] verifyExpires: { type: Date }, +[ ] verifyChanges: { type: Object }, +[ ] resetToken: { type: String }, +[ ] resetShortToken: { type: String }, +[ ] resetExpires: { type: Date }, +[X] wasSeeded: { type: Boolean }, +[X] wasInvited: { type: Boolean }, +[ ] language: { +[ ] type: String, +[ ] default: 'en' + }, +[ ] termsAndConditionsAccepted: { type: Date }, // we display the terms and conditions on registration +[ ] systemNotificationsSeen: { +[ ] type: Array, +[ ] default: [] + } + } +*/ +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as user +MERGE(u:User {id: user._id["$oid"]}) +ON CREATE SET +u.name = user.name, +u.slug = COALESCE(user.slug, apoc.text.random(20, "[A-Za-z]")), +u.email = user.email, +u.encryptedPassword = user.password, +u.avatar = replace(user.avatar, 'https://api-alpha.human-connection.org', ''), +u.coverImg = replace(user.coverImg, 'https://api-alpha.human-connection.org', ''), +u.wasInvited = user.wasInvited, +u.wasSeeded = user.wasSeeded, +u.role = toLower(user.role), +u.createdAt = user.createdAt.`$date`, +u.updatedAt = user.updatedAt.`$date`, +u.deleted = user.deletedAt IS NOT NULL, +u.disabled = false +MERGE (e:EmailAddress { + email: user.email, + createdAt: toString(datetime()), + verifiedAt: toString(datetime()) +}) +MERGE (e)-[:BELONGS_TO]->(u) +MERGE (u)-[:PRIMARY_EMAIL]->(e) +WITH u, user, user.badgeIds AS badgeIds +UNWIND badgeIds AS badgeId +MATCH (b:Badge {id: badgeId}) +MERGE (b)-[:REWARDED]->(u) +; diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/userscandos/delete.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/userscandos/delete.cql new file mode 100644 index 0000000..e69de29 diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/userscandos/userscandos.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/userscandos/userscandos.cql new file mode 100644 index 0000000..55f58f1 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/userscandos/userscandos.cql @@ -0,0 +1,35 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] userId: { +[ ] type: String, +[ ] required: true + }, +[ ] contributionId: { +[ ] type: String, +[ ] required: true + }, +[ ] done: { +[ ] type: Boolean, +[ ] default: false + }, +[ ] doneAt: { type: Date }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } + index: +[ ] { userId: 1, contributionId: 1 },{ unique: true } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as usercando; diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/usersettings/delete.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/usersettings/delete.cql new file mode 100644 index 0000000..e69de29 diff --git a/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/usersettings/usersettings.cql b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/usersettings/usersettings.cql new file mode 100644 index 0000000..7226259 --- /dev/null +++ b/deployment/old/legacy-migration/maintenance-worker/migration/neo4j/usersettings/usersettings.cql @@ -0,0 +1,43 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] userId: { +[ ] type: String, +[ ] required: true, +[ ] unique: true + }, +[ ] blacklist: { +[ ] type: Array, +[ ] default: [] + }, +[ ] uiLanguage: { +[ ] type: String, +[ ] required: true + }, +[ ] contentLanguages: { +[ ] type: Array, +[ ] default: [] + }, +[ ] filter: { +[ ] categoryIds: { +[ ] type: Array, +[ ] index: true + }, +[ ] emotions: { +[ ] type: Array, +[ ] index: true + } + }, +[ ] hideUsersWithoutTermsOfUseSigniture: {type: Boolean}, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as usersetting; diff --git a/deployment/old/mailserver/Deployment.yaml b/deployment/old/mailserver/Deployment.yaml new file mode 100644 index 0000000..a36e165 --- /dev/null +++ b/deployment/old/mailserver/Deployment.yaml @@ -0,0 +1,40 @@ +{{- if .Values.developmentMailserverDomain }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-mailserver + labels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/name: ocelot-social + app.kubernetes.io/version: {{ .Chart.AppVersion }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +spec: + replicas: 1 + minReadySeconds: 15 + progressDeadlineSeconds: 60 + selector: + matchLabels: + ocelot.social/selector: deployment-mailserver + template: + metadata: + labels: + ocelot.social/selector: deployment-mailserver + name: mailserver + spec: + containers: + - name: mailserver + image: djfarrelly/maildev + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - containerPort: 80 + - containerPort: 25 + envFrom: + - configMapRef: + name: {{ .Release.Name }}-configmap + - secretRef: + name: {{ .Release.Name }}-secrets + restartPolicy: Always + terminationGracePeriodSeconds: 30 +status: {} +{{- end}} \ No newline at end of file diff --git a/deployment/old/mailserver/README.md b/deployment/old/mailserver/README.md new file mode 100644 index 0000000..ed9292d --- /dev/null +++ b/deployment/old/mailserver/README.md @@ -0,0 +1,18 @@ +# Development Mail Server + +You can deploy a fake smtp server which captures all send mails and displays +them in a web interface. The [sample configuration](../templates/configmap.template.yaml) +is assuming such a dummy server in the `SMTP_HOST` configuration and points to +a cluster-internal SMTP server. + +To deploy the SMTP server just uncomment the relevant code in the +[ingress server configuration](../../https/templates/ingress.template.yaml) and +run the following: + +```bash +# in folder deployment/ocelot-social +$ kubectl apply -f mailserver/ +``` + +You might need to refresh the TLS secret to enable HTTPS on the publicly +available web interface. diff --git a/deployment/old/mailserver/Service.yaml b/deployment/old/mailserver/Service.yaml new file mode 100644 index 0000000..bba7349 --- /dev/null +++ b/deployment/old/mailserver/Service.yaml @@ -0,0 +1,22 @@ +{{- if .Values.developmentMailserverDomain }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-mailserver + labels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/name: ocelot-social + app.kubernetes.io/version: {{ .Chart.AppVersion }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +spec: + ports: + - name: web + port: 80 + targetPort: 80 + - name: smtp + port: 25 + targetPort: 25 + selector: + ocelot.social/selector: deployment-mailserver +{{- end}} \ No newline at end of file diff --git a/deployment/old/mailserver/ingress.yaml b/deployment/old/mailserver/ingress.yaml new file mode 100644 index 0000000..1ea9c58 --- /dev/null +++ b/deployment/old/mailserver/ingress.yaml @@ -0,0 +1,42 @@ +kind: Ingress +apiVersion: networking.k8s.io/v1 +metadata: + name: ingress-{{ .Release.Name }}-webapp + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/component: "ingress webapp" + app.kubernetes.io/part-of: "{{ .Chart.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + annotations: + kubernetes.io/ingress.class: "nginx" + cert-manager.io/cluster-issuer: {{ .Values.LETSENCRYPT.ISSUER }} + nginx.ingress.kubernetes.io/proxy-body-size: {{ .Values.NGINX.PROXY_BODY_SIZE }} +spec: + tls: + - hosts: + - {{ .Values.LETSENCRYPT.DOMAIN }} + secretName: tls + rules: + - host: {{ .Values.LETSENCRYPT.DOMAIN }} + http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + service: + name: {{ .Release.Name }}-webapp + port: + number: 3000 + +#{{- if .Values.developmentMailserverDomain }} +# - host: {{ .Values.developmentMailserverDomain }} +# http: +# paths: +# - path: / +# backend: +# serviceName: {{ .Release.Name }}-mailserver +# servicePort: 80 +#{{- end }} diff --git a/deployment/old/monitoring/README.md b/deployment/old/monitoring/README.md new file mode 100644 index 0000000..46dfb03 --- /dev/null +++ b/deployment/old/monitoring/README.md @@ -0,0 +1,43 @@ +# Metrics + +You can optionally setup [prometheus](https://prometheus.io/) and +[grafana](https://grafana.com/) for metrics. + +We follow this tutorial [here](https://medium.com/@chris_linguine/how-to-monitor-your-kubernetes-cluster-with-prometheus-and-grafana-2d5704187fc8): + +```bash +kubectl proxy # proxy to your kubernetes dashboard + +helm repo list +# If using helm v3, the stable repository is not set, so you need to manually add it. +helm repo add stable https://kubernetes-charts.storage.googleapis.com +# Create a monitoring namespace for your cluster +kubectl create namespace monitoring +helm --namespace monitoring install prometheus stable/prometheus +kubectl -n monitoring get pods # look for 'server' +kubectl port-forward -n monitoring 9090 +# You can now see your prometheus server on: http://localhost:9090 + +# Make sure you are in folder `deployment/` +kubectl apply -f monitoring/grafana/config.yml +helm --namespace monitoring install grafana stable/grafana -f monitoring/grafana/values.yml +# Get the admin password for grafana from your kubernetes dashboard. +kubectl --namespace monitoring port-forward 3000 +# You can now see your grafana dashboard on: http://localhost:3000 +# Login with user 'admin' and the password you just looked up. +# In your dashboard import this dashboard: +# https://grafana.com/grafana/dashboards/1860 +# Enter ID 180 and choose "Prometheus" as datasource. +# You got metrics! +``` + +Now you should see something like this: + +![Grafana dashboard](./grafana/metrics.png) + +You can set up a grafana dashboard, by visiting https://grafana.com/dashboards, finding one that is suitable and copying it's id. +You then go to the left hand menu in localhost, choose `Dashboard` > `Manage` > `Import` +Paste in the id, click `Load`, select `Prometheus` for the data source, and click `Import` + +When you just installed prometheus and grafana, the data will not be available +immediately, so wait for a couple of minutes and reload. diff --git a/deployment/old/monitoring/grafana/config.yml b/deployment/old/monitoring/grafana/config.yml new file mode 100644 index 0000000..a338e34 --- /dev/null +++ b/deployment/old/monitoring/grafana/config.yml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-grafana-datasource + namespace: monitoring + labels: + grafana_datasource: '1' +data: + datasource.yaml: |- + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus-server.monitoring.svc.cluster.local diff --git a/deployment/old/monitoring/grafana/metrics.png b/deployment/old/monitoring/grafana/metrics.png new file mode 100644 index 0000000..cc68f1b Binary files /dev/null and b/deployment/old/monitoring/grafana/metrics.png differ diff --git a/deployment/old/monitoring/grafana/values.yml b/deployment/old/monitoring/grafana/values.yml new file mode 100644 index 0000000..02004cc --- /dev/null +++ b/deployment/old/monitoring/grafana/values.yml @@ -0,0 +1,4 @@ +sidecar: + datasources: + enabled: true + label: grafana_datasource diff --git a/deployment/old/volumes/README.md b/deployment/old/volumes/README.md new file mode 100644 index 0000000..1d84968 --- /dev/null +++ b/deployment/old/volumes/README.md @@ -0,0 +1,37 @@ +# Persistent Volumes + +At the moment, the application needs two persistent volumes: + +* The `/data/` folder where `neo4j` stores its database and +* the folder `/develop-backend/public/uploads` where the backend stores uploads, in case you don't use Digital Ocean Spaces (an AWS S3 bucket) for this purpose. + +As a matter of precaution, the persistent volume claims that setup these volumes +live in a separate folder. You don't want to accidently loose all your data in +your database by running + +```sh +kubectl delete -f ocelot-social/ +``` + +or do you? + +## Create Persistent Volume Claims + +Run the following: + +```sh +# in folder deployments/ +$ kubectl apply -f volumes +persistentvolumeclaim/neo4j-data-claim created +persistentvolumeclaim/uploads-claim created +``` + +## Backup And Restore + +We tested a couple of options how to do disaster recovery in kubernetes. First, +there is the [offline backup strategy](./neo4j-offline-backup/README.md) of the +community edition of Neo4J, which you can also run on a local installation. +Kubernetes also offers so-called [volume snapshots](./volume-snapshots/README.md). +Changing the [reclaim policy](./reclaim-policy/README.md) of your persistent +volumes might be an additional safety measure. Finally, there is also a +kubernetes specific disaster recovery tool called [Velero](./velero/README.md). diff --git a/deployment/old/volumes/neo4j-offline-backup/README.md b/deployment/old/volumes/neo4j-offline-backup/README.md new file mode 100644 index 0000000..7c34aa7 --- /dev/null +++ b/deployment/old/volumes/neo4j-offline-backup/README.md @@ -0,0 +1,88 @@ +# Backup (offline) + +This tutorial explains how to carry out an offline backup of your Neo4J +database in a kubernetes cluster. + +An offline backup requires the Neo4J database to be stopped. Read +[the docs](https://neo4j.com/docs/operations-manual/current/tools/dump-load/). +Neo4J also offers online backups but this is available in enterprise edition +only. + +The tricky part is to stop the Neo4J database *without* stopping the container. +Neo4J's docker container image starts `neo4j` by default, so we have to override +this command with sth. that keeps the container spinning but does not terminate +it. + +## Stop and Restart Neo4J Database in Kubernetes + +[This tutorial](http://bigdatums.net/2017/11/07/how-to-keep-docker-containers-running/) +explains how to keep a docker container running. For kubernetes, the way to +override the docker image `CMD` is explained [here](https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#define-a-command-and-arguments-when-you-create-a-pod). + +So, all we have to do is edit the kubernetes deployment of our Neo4J database +and set a custom `command` every time we have to carry out tasks like backup, +restore, seed etc. + +First bring the application into [maintenance mode](https://github.com/Ocelot-Social-Community/Ocelot-Social/blob/master/deployment/ocelot-social/maintenance/README.md) to ensure there are no +database connections left and nobody can access the application. + +Run the following: + +```sh +$ kubectl -n ocelot-social edit deployment develop-neo4j +``` + +Add the following to `spec.template.spec.containers`: + +```sh +["tail", "-f", "/dev/null"] +``` + +and write the file which will update the deployment. + +The command `tail -f /dev/null` is the equivalent of *sleep forever*. It is a +hack to keep the container busy and to prevent its shutdown. It will also +override the default `neo4j` command and the kubernetes pod will not start the +database. + +Now perform your tasks! + +When you're done, edit the deployment again and remove the `command`. Write the +file and trigger an update of the deployment. + +## Create a Backup in Kubernetes + +First stop your Neo4J database, see above. Then: + +```sh +$ kubectl -n ocelot-social get pods +# Copy the ID of the pod running Neo4J. +$ kubectl -n ocelot-social exec -it bash +# Once you're in the pod, dump the db to a file e.g. `/root/neo4j-backup`. +> neo4j-admin dump --to=/root/neo4j-backup +> exit +# Download the file from the pod to your computer. +$ kubectl cp human-connection/:/root/neo4j-backup ./neo4j-backup +``` + +Revert your changes to deployment `develop-neo4j` which will restart the database. + +## Restore a Backup in Kubernetes + +First stop your Neo4J database. Then: + +```sh +$ kubectl -n ocelot-social get pods +# Copy the ID of the pod running Neo4J. +# Then upload your local backup to the pod. Note that once the pod gets deleted +# e.g. if you change the deployment, the backup file is gone with it. +$ kubectl cp ./neo4j-backup human-connection/:/root/ +$ kubectl -n ocelot-social exec -it bash +# Once you're in the pod restore the backup and overwrite the default database +# called `graph.db` with `--force`. +# This will delete all existing data in database `graph.db`! +> neo4j-admin load --from=/root/neo4j-backup --force +> exit +``` + +Revert your changes to deployment `develop-neo4j` which will restart the database. diff --git a/deployment/old/volumes/neo4j-online-backup/README.md b/deployment/old/volumes/neo4j-online-backup/README.md new file mode 100644 index 0000000..602bbd5 --- /dev/null +++ b/deployment/old/volumes/neo4j-online-backup/README.md @@ -0,0 +1,59 @@ +# Backup (online) + +## Online backups are only avaible with a Neo4j Enterprise and a license, see https://neo4j.com/licensing/ for the different licenses available + +This tutorial explains how to carry out an online backup of your Neo4J +database in a kubernetes cluster. + +One of the benefits of doing an online backup is that the Neo4j database does not need to be stopped, so there is no downtime. Read [the docs](https://neo4j.com/docs/operations-manual/current/backup/performing/) + +To use Neo4j Enterprise you must add this line to your configmap, if using, or your deployment `develop-neo4j` env. + +```sh +NEO4J_ACCEPT_LICENSE_AGREEMENT: "yes" +``` + +## Create a Backup in Kubernetes + +```sh +# Backup the database with one command, this will get the develop-neo4j pod, ssh into it, and run the backup command +$ kubectl -n=human-connection exec -it $(kubectl -n=human-connection get pods | grep develop-neo4j | awk '{ print $1 }') -- neo4j-admin backup --backup-dir=/var/lib/neo4j --name=neo4j-backup +# Download the file from the pod to your computer. +$ kubectl cp human-connection/$(kubectl -n=human-connection get pods | grep develop-neo4j | awk '{ print $1 }'):/var/lib/neo4j/neo4j-backup ./neo4j-backup/ +``` + +You should now have a backup of the database locally. If you want, you can simulate disaster recovery by sshing into the develop-neo4j pod, deleting all data and restoring from backup + +## Disaster where database data is gone somehow + +```sh +$ kubectl -n=human-connection exec -it $(kubectl -n=human-connection get pods | grep develop-neo4j |awk '{ print $1 }') bash +# Enter cypher-shell +$ cypher-shell +# Delete all data +> MATCH (n) DETACH DELETE (n); + +> exit +``` + +## Restore a backup in Kubernetes + +Restoration must be done while the database is not running, see [our docs](https://docs.human-connection.org/human-connection/deployment/volumes/neo4j-offline-backup#stop-and-restart-neo-4-j-database-in-kubernetes) for how to stop the database, but keep the container running + +After, you have stopped the database, and have the pod running, you can restore the database by running these commands: + +```sh +$ kubectl -n ocelot-social get pods +# Copy the ID of the pod running Neo4J. +# Then upload your local backup to the pod. Note that once the pod gets deleted +# e.g. if you change the deployment, the backup file is gone with it. +$ kubectl cp ./neo4j-backup/ human-connection/:/root/ +$ kubectl -n ocelot-social exec -it bash +# Once you're in the pod restore the backup and overwrite the default database +# called `graph.db` with `--force`. +# This will delete all existing data in database `graph.db`! +> neo4j-admin restore --from=/root/neo4j-backup --force +> exit +``` + +Revert your changes to deployment `develop-neo4j` which will restart the database. diff --git a/deployment/old/volumes/uploads.yaml b/deployment/old/volumes/uploads.yaml new file mode 100644 index 0000000..45e1292 --- /dev/null +++ b/deployment/old/volumes/uploads.yaml @@ -0,0 +1,12 @@ +--- + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: uploads-claim + namespace: ocelot-social + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "10Gi" diff --git a/deployment/old/volumes/velero/README.md b/deployment/old/volumes/velero/README.md new file mode 100644 index 0000000..5b8fc9d --- /dev/null +++ b/deployment/old/volumes/velero/README.md @@ -0,0 +1,112 @@ +# Velero + +{% hint style="danger" %} +I tried Velero and it did not work reliably all the time. Sometimes the +kubernetes cluster crashes during recovery or data is not fully recovered. + +Feel free to test it out and update this documentation once you feel that it's +working reliably. It is very likely that Digital Ocean had some bugs when I +tried out the steps below. +{% endhint %} + +We use [velero](https://github.com/heptio/velero) for on premise backups, we +tested on version `v0.11.0`, you can find their +documentation [here](https://heptio.github.io/velero/v0.11.0/). + +Our kubernets configurations adds some annotations to pods. The annotations +define the important persistent volumes that need to be backed up. Velero will +pick them up and store the volumes in the same cluster but in another namespace +`velero`. + +## Prequisites + +You have to install the binary `velero` on your computer and get a tarball of +the latest release. We use `v0.11.0` so visit the +[release](https://github.com/heptio/velero/releases/tag/v0.11.0) page and +download and extract e.g. [velero-v0.11.0-linux-arm64.tar.gz](https://github.com/heptio/velero/releases/download/v0.11.0/velero-v0.11.0-linux-amd64.tar.gz). + + +## Setup Velero Namespace + +Follow their [getting started](https://heptio.github.io/velero/v0.11.0/get-started) +instructions to setup the Velero namespace. We use +[Minio](https://docs.min.io/docs/deploy-minio-on-kubernetes) and +[restic](https://github.com/restic/restic), so check out Velero's instructions +how to setup [restic](https://heptio.github.io/velero/v0.11.0/restic): + +```sh +# run from the extracted folder of the tarball +$ kubectl apply -f config/common/00-prereqs.yaml +$ kubectl apply -f config/minio/ +``` + +Once completed, you should see the namespace in your kubernetes dashboard. + +## Manually Create an On-Premise Backup + +When you create your deployments for Human Connection the required annotations +should already be in place. So when you create a backup of namespace +`human-connection`: + +```sh +$ velero backup create hc-backup --include-namespaces=human-connection +``` + +That should backup your persistent volumes, too. When you enter: + +```sh +$ velero backup describe hc-backup --details +``` + +You should see the persistent volumes at the end of the log: + +```sh +.... + +Restic Backups: + Completed: + human-connection/develop-backend-5b6dd96d6b-q77n6: uploads + human-connection/develop-neo4j-686d768598-z2vhh: neo4j-data +``` + +## Simulate a Disaster + +Feel free to try out if you loose any data when you simulate a disaster and try +to restore the namespace from the backup: + +```sh +$ kubectl delete namespace human-connection +``` + +Wait until the wrongdoing has completed, then: +```sh +$ velero restore create --from-backup hc-backup +``` + +Now, I keep my fingers crossed that everything comes back again. If not, I feel +very sorry for you. + + +## Schedule a Regular Backup + +Check out the [docs](https://heptio.github.io/velero/v0.11.0/get-started). You +can create a regular schedule e.g. with: + +```sh +$ velero schedule create hc-weekly-backup --schedule="@weekly" --include-namespaces=human-connection +``` + +Inspect the created backups: + +```sh +$ velero schedule get +NAME STATUS CREATED SCHEDULE BACKUP TTL LAST BACKUP SELECTOR +hc-weekly-backup Enabled 2019-05-08 17:51:31 +0200 CEST @weekly 720h0m0s 6s ago + +$ velero backup get +NAME STATUS CREATED EXPIRES STORAGE LOCATION SELECTOR +hc-weekly-backup-20190508155132 Completed 2019-05-08 17:51:32 +0200 CEST 29d default + +$ velero backup describe hc-weekly-backup-20190508155132 --details +# see if the persistent volumes are backed up +``` diff --git a/deployment/old/volumes/volume-snapshots/README.md b/deployment/old/volumes/volume-snapshots/README.md new file mode 100644 index 0000000..cc66ae4 --- /dev/null +++ b/deployment/old/volumes/volume-snapshots/README.md @@ -0,0 +1,50 @@ +# Kubernetes Volume Snapshots + +It is possible to backup persistent volumes through volume snapshots. This is +especially handy if you don't want to stop the database to create an [offline +backup](../neo4j-offline-backup/README.md) thus having a downtime. + +Kubernetes announced this feature in a [blog post](https://kubernetes.io/blog/2018/10/09/introducing-volume-snapshot-alpha-for-kubernetes/). Please make yourself familiar with it before you continue. + +## Create a Volume Snapshot + +There is an example in this folder how you can e.g. create a volume snapshot for +the persistent volume claim `neo4j-data-claim`: + +```sh +# in folder deployment/volumes/volume-snapshots/ +kubectl apply -f snapshot.yaml +``` + +If you are on Digital Ocean the volume snapshot should show up in the Web UI: + +![Digital Ocean Web UI showing a volume snapshot](./digital-ocean-volume-snapshots.png) + +## Provision a Volume based on a Snapshot + +Edit your persistent volume claim configuration and add a `dataSource` pointing +to your volume snapshot. [The blog post](https://kubernetes.io/blog/2018/10/09/introducing-volume-snapshot-alpha-for-kubernetes/) has an example in section "Provision a new volume from a snapshot with +Kubernetes". + +There is also an example in this folder how the configuration could look like. +If you apply the configuration new persistent volume claim will be provisioned +with the data from the volume snapshot: + +``` +# in folder deployment/volumes/volume-snapshots/ +kubectl apply -f neo4j-data.yaml +``` + +## Data Consistency Warning + +Note that volume snapshots do not guarantee data consistency. Quote from the +[blog post](https://kubernetes.io/blog/2018/10/09/introducing-volume-snapshot-alpha-for-kubernetes/): + +> Please note that the alpha release of Kubernetes Snapshot does not provide +> any consistency guarantees. You have to prepare your application (pause +> application, freeze filesystem etc.) before taking the snapshot for data +> consistency. + +In case of Neo4J this probably means that enterprise edition is required which +supports [online backups](https://neo4j.com/docs/operations-manual/current/backup/). + diff --git a/deployment/old/volumes/volume-snapshots/backen-uploads-snapshot.yaml b/deployment/old/volumes/volume-snapshots/backen-uploads-snapshot.yaml new file mode 100644 index 0000000..697346c --- /dev/null +++ b/deployment/old/volumes/volume-snapshots/backen-uploads-snapshot.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: snapshot.storage.k8s.io/v1beta1 +kind: VolumeSnapshot +metadata: + name: YYYY-MM-DD-uploads-snapshot +spec: + source: + persistentVolumeClaimName: volume-claim-ocelot-uploads diff --git a/deployment/old/volumes/volume-snapshots/digital-ocean-volume-snapshots.png b/deployment/old/volumes/volume-snapshots/digital-ocean-volume-snapshots.png new file mode 100644 index 0000000..cb65996 Binary files /dev/null and b/deployment/old/volumes/volume-snapshots/digital-ocean-volume-snapshots.png differ diff --git a/deployment/old/volumes/volume-snapshots/neo4j-data-provision-snapshot.yaml b/deployment/old/volumes/volume-snapshots/neo4j-data-provision-snapshot.yaml new file mode 100644 index 0000000..cd8552b --- /dev/null +++ b/deployment/old/volumes/volume-snapshots/neo4j-data-provision-snapshot.yaml @@ -0,0 +1,18 @@ +--- + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: neo4j-data-claim + namespace: ocelot-social + labels: + app: ocelot-social + spec: + dataSource: + name: neo4j-data-snapshot + kind: VolumeSnapshot + apiGroup: snapshot.storage.k8s.io + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/deployment/old/volumes/volume-snapshots/neo4j-data-snapshot.yaml b/deployment/old/volumes/volume-snapshots/neo4j-data-snapshot.yaml new file mode 100644 index 0000000..1053d01 --- /dev/null +++ b/deployment/old/volumes/volume-snapshots/neo4j-data-snapshot.yaml @@ -0,0 +1,12 @@ +--- + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: neo4j-data-claim + namespace: ocelot-social + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "10Gi" # see requirements for Neo4j v3.5.14 https://neo4j.com/docs/operations-manual/3.5/installation/requirements/