From c240fe32adb9cdff2471bee32dd58357d36c22e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20Hu=C3=9F?= Date: Fri, 1 Dec 2023 14:38:42 +0100 Subject: [PATCH 1/9] Add more detailed logs for the scripts - Ignore 'backup-cron-job.log' files. --- .gitignore | 1 + deployment/scripts/cluster.backup.sh | 3 +++ deployment/scripts/cluster.maintenance.sh | 2 ++ deployment/scripts/cluster.neo4j.sh | 4 ++++ 4 files changed, 10 insertions(+) diff --git a/.gitignore b/.gitignore index 928dae262..a0a08dcaa 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ yarn-error.log* .yarn-integrity .eslintcache kubeconfig.yaml +backup-cron-job.log node_modules/ cypress/videos diff --git a/deployment/scripts/cluster.backup.sh b/deployment/scripts/cluster.backup.sh index 93c29dd92..7da493e4b 100755 --- a/deployment/scripts/cluster.backup.sh +++ b/deployment/scripts/cluster.backup.sh @@ -27,13 +27,16 @@ mkdir -p ${BACKUP_FOLDER} ${SCRIPT_DIR}/cluster.neo4j.sh maintenance on # database backup +echo "Dumping database ..." kubectl --kubeconfig=${KUBECONFIG} -n default exec -it \ $(kubectl --kubeconfig=${KUBECONFIG} -n default get pods | grep ocelot-neo4j | awk '{ print $1 }') \ -- neo4j-admin dump --database=neo4j --to=/var/lib/neo4j/$BACKUP_DATE-neo4j-dump # copy neo4j backup to local drive +echo "Coping database ..." kubectl --kubeconfig=${KUBECONFIG} cp \ default/$(kubectl --kubeconfig=${KUBECONFIG} -n default get pods | grep ocelot-neo4j |awk '{ print $1 }'):/var/lib/neo4j/$BACKUP_DATE-neo4j-dump $BACKUP_FOLDER/neo4j-dump # copy image data +echo "Coping public uploads ..." kubectl --kubeconfig=${KUBECONFIG} cp \ default/$(kubectl --kubeconfig=${KUBECONFIG} -n default get pods | grep ocelot-backend |awk '{ print $1 }'):/app/public/uploads $BACKUP_FOLDER/public-uploads diff --git a/deployment/scripts/cluster.maintenance.sh b/deployment/scripts/cluster.maintenance.sh index b64994e9b..33c52254d 100755 --- a/deployment/scripts/cluster.maintenance.sh +++ b/deployment/scripts/cluster.maintenance.sh @@ -16,9 +16,11 @@ KUBECONFIG=${KUBECONFIG:-${SCRIPT_DIR}/../configurations/${CONFIGURATION}/kubeco case $1 in on) + echo "Network maintenance: on" kubectl --kubeconfig=${KUBECONFIG} patch ingress ingress-ocelot-webapp --type merge --patch-file ${SCRIPT_DIR}/../src/kubernetes/patches/patch.ingress.maintenance.on.yaml ;; off) + echo "Network maintenance: off" kubectl --kubeconfig=${KUBECONFIG} patch ingress ingress-ocelot-webapp --type merge --patch-file ${SCRIPT_DIR}/../src/kubernetes/patches/patch.ingress.maintenance.off.yaml ;; *) diff --git a/deployment/scripts/cluster.neo4j.sh b/deployment/scripts/cluster.neo4j.sh index 41ebe4227..ae539b314 100755 --- a/deployment/scripts/cluster.neo4j.sh +++ b/deployment/scripts/cluster.neo4j.sh @@ -21,20 +21,24 @@ case $1 in ${SCRIPT_DIR}/cluster.maintenance.sh on # set Neo4j in offline mode (maintenance) + echo "Neo4j maintenance: on" kubectl --kubeconfig=${KUBECONFIG} get deployment ocelot-neo4j -o json \ | jq '.spec.template.spec.containers[] += {"command": ["tail", "-f", "/dev/null"]}' \ | kubectl --kubeconfig=${KUBECONFIG} apply -f - # wait for the container to restart + echo "Wait 60s ..." sleep 60 ;; off) # set Neo4j in online mode + echo "Neo4j maintenance: off" kubectl --kubeconfig=${KUBECONFIG} get deployment ocelot-neo4j -o json \ | jq 'del(.spec.template.spec.containers[].command)' \ | kubectl --kubeconfig=${KUBECONFIG} apply -f - # wait for the container to restart + echo "Wait 60s ..." sleep 60 # maintenance mode off From 4351c9e0dba77f7a73949a7ea17d038f34ba53b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20Hu=C3=9F?= Date: Fri, 1 Dec 2023 14:40:26 +0100 Subject: [PATCH 2/9] Refine script readme --- deployment/deployment.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/deployment/deployment.md b/deployment/deployment.md index ebcc2f6ca..88b4f39de 100644 --- a/deployment/deployment.md +++ b/deployment/deployment.md @@ -109,7 +109,7 @@ The backups will be saved into your networks folders `backup` folder in a new fo ⚠️ *Attention: Please check carefully whether really the oldest backups have been deleted. As shells on different systems behave differently with regard to the commands used in this script.* -Install automated backups by a cron job. +Install automated backups by a [cron job](https://en.wikipedia.org/wiki/Cron). Be aware of having the bash shell installed to run the script. The environment variables for the automated backups are described above. @@ -131,3 +131,7 @@ In the editor add the line: ``` This way the terminal output is written into a log file named `backup-cron-job.log` located in the deployment folder. + +Be aware that the server datetime can differ from your local time. +Especially by the change between summer and winter time, because servers usually have UTC. +Find out the actual difference by running the command `date` on your server. From 7d4a75a94c31b00c1af40cd271c3e82b5293341c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20Hu=C3=9F?= Date: Fri, 1 Dec 2023 15:02:25 +0100 Subject: [PATCH 3/9] Refine checking of definition of variables --- deployment/scripts/cluster.backup.sh | 2 +- deployment/scripts/cluster.maintenance.sh | 2 +- deployment/scripts/cluster.neo4j.sh | 2 +- deployment/scripts/clusters.backup-multiple-servers.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deployment/scripts/cluster.backup.sh b/deployment/scripts/cluster.backup.sh index 7da493e4b..2d3bf156e 100755 --- a/deployment/scripts/cluster.backup.sh +++ b/deployment/scripts/cluster.backup.sh @@ -9,7 +9,7 @@ SCRIPT_PATH=$(realpath $0) SCRIPT_DIR=$(dirname $SCRIPT_PATH) # check CONFIGURATION -if [[ -z ${CONFIGURATION} ]]; then +if [[ -z "$CONFIGURATION" ]] || [[ $CONFIGURATION == "" ]]; then echo "!!! You must provide a CONFIGURATION via environment variable !!!" exit 1 fi diff --git a/deployment/scripts/cluster.maintenance.sh b/deployment/scripts/cluster.maintenance.sh index 33c52254d..780b81bd6 100755 --- a/deployment/scripts/cluster.maintenance.sh +++ b/deployment/scripts/cluster.maintenance.sh @@ -5,7 +5,7 @@ SCRIPT_PATH=$(realpath $0) SCRIPT_DIR=$(dirname $SCRIPT_PATH) # check CONFIGURATION -if [ -z ${CONFIGURATION} ]; then +if [[ -z "$CONFIGURATION" ]] || [[ $CONFIGURATION == "" ]]; then echo "You must provide a `CONFIGURATION` via environment variable" exit 1 fi diff --git a/deployment/scripts/cluster.neo4j.sh b/deployment/scripts/cluster.neo4j.sh index ae539b314..3b284c0b7 100755 --- a/deployment/scripts/cluster.neo4j.sh +++ b/deployment/scripts/cluster.neo4j.sh @@ -5,7 +5,7 @@ SCRIPT_PATH=$(realpath $0) SCRIPT_DIR=$(dirname $SCRIPT_PATH) # check CONFIGURATION -if [ -z ${CONFIGURATION} ]; then +if [[ -z "$CONFIGURATION" ]] || [[ $CONFIGURATION == "" ]]; then echo "You must provide a `CONFIGURATION` via environment variable" exit 1 fi diff --git a/deployment/scripts/clusters.backup-multiple-servers.sh b/deployment/scripts/clusters.backup-multiple-servers.sh index 3dfcb1d79..e81165457 100755 --- a/deployment/scripts/clusters.backup-multiple-servers.sh +++ b/deployment/scripts/clusters.backup-multiple-servers.sh @@ -17,7 +17,7 @@ source ${SCRIPT_DIR}/../.env set +a # check BACKUP_CONFIGURATIONS -if [[ -z ${BACKUP_CONFIGURATIONS} ]]; then +if [[ -z "$BACKUP_CONFIGURATIONS" ]] || [[ $BACKUP_CONFIGURATIONS == "" ]]; then #%! echo "You must provide a BACKUP_CONFIGURATIONS via environment variable" printf "!!! You must provide a BACKUP_CONFIGURATIONS via environment variable !!!\n" exit 1 From 1dc871d3069917d9e4720f093aa320368012aa58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20Hu=C3=9F?= Date: Fri, 1 Dec 2023 15:06:38 +0100 Subject: [PATCH 4/9] Add database name as variable to script --- deployment/scripts/cluster.backup.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/deployment/scripts/cluster.backup.sh b/deployment/scripts/cluster.backup.sh index 2d3bf156e..88fdb7c6d 100755 --- a/deployment/scripts/cluster.backup.sh +++ b/deployment/scripts/cluster.backup.sh @@ -14,6 +14,13 @@ if [[ -z "$CONFIGURATION" ]] || [[ $CONFIGURATION == "" ]]; then exit 1 fi +# check DATABASE_NAME or set default +if [[ -z "$DATABASE_NAME" ]] || [[ $DATABASE_NAME == "" ]]; then + DATABASE_NAME="neo4j" + printf "Set satabase default name.\n" +fi +printf "Database name: '%s'\n" $DATABASE_NAME + # configuration KUBECONFIG=${KUBECONFIG:-${SCRIPT_DIR}/../configurations/${CONFIGURATION}/kubeconfig.yaml} BACKUP_DATE=$(date "+%F_%H-%M-%S") @@ -30,7 +37,7 @@ ${SCRIPT_DIR}/cluster.neo4j.sh maintenance on echo "Dumping database ..." kubectl --kubeconfig=${KUBECONFIG} -n default exec -it \ $(kubectl --kubeconfig=${KUBECONFIG} -n default get pods | grep ocelot-neo4j | awk '{ print $1 }') \ - -- neo4j-admin dump --database=neo4j --to=/var/lib/neo4j/$BACKUP_DATE-neo4j-dump + -- neo4j-admin dump --database=$DATABASE_NAME --to=/var/lib/neo4j/$BACKUP_DATE-neo4j-dump # copy neo4j backup to local drive echo "Coping database ..." kubectl --kubeconfig=${KUBECONFIG} cp \ From 64ac68bd01b687ce83dabc6a8390793e21e40cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20Hu=C3=9F?= Date: Tue, 12 Dec 2023 15:43:47 +0100 Subject: [PATCH 5/9] Add 'NEO4J_dbms_default__database' to 'docker-compose.yml' as commented example --- docker-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 78cee69b3..7954e2bab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -111,10 +111,12 @@ services: # settings reference: https://neo4j.com/docs/operations-manual/4.4/docker/ref-settings/ # TODO: This sounds scary for a production environment - NEO4J_AUTH=none + # - NEO4J_dbms_default__database=graph.db + # - NEO4J_dbms_default__database=neo4j - NEO4J_dbms_security_procedures_unrestricted=algo.*,apoc.* - NEO4J_dbms_allow__format__migration=true - NEO4J_dbms_allow__upgrade=true - # Uncomment following line for Neo4j Enterprise version instead of Community version + # uncomment following line for Neo4j Enterprise version instead of Community version # TODO: clarify if that is the only thing needed to unlock the Enterprise version # - NEO4J_ACCEPT_LICENSE_AGREEMENT=yes # TODO: Remove the playground from production From e4fabc47a0e54b1b65e1b90fc0df5ad951c4d67d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20Hu=C3=9F?= Date: Tue, 12 Dec 2023 15:46:24 +0100 Subject: [PATCH 6/9] Add 'Database Management Commands' to Neo4j readme - Fix of some headings and designations. --- neo4j/README.md | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/neo4j/README.md b/neo4j/README.md index 9b8a460ec..d9e510d8f 100644 --- a/neo4j/README.md +++ b/neo4j/README.md @@ -1,4 +1,4 @@ -# Neo4J +# Neo4j 4.4 Human Connection is a social network. Using a graph based database which can model nodes and edges natively - a network - feels like an obvious choice. We @@ -16,8 +16,7 @@ docker-compose up ``` You can access Neo4J through [http://localhost:7474/](http://localhost:7474/) -for an interactive cypher shell and a visualization of the graph. - +for an interactive Cypher shell and a visualization of the graph. ## Installation Without Docker @@ -51,7 +50,7 @@ in `backend/.env`. Start Neo4J and confirm the database is running at [http://localhost:7474](http://localhost:7474). -## Operations on Neo4j +## Operations on Neo4j 4.4 ### Docker or Docker Compose @@ -143,7 +142,7 @@ $ kubectl -n default exec -it $(kubectl -n default get pods | grep ocelot-backen ***Cypher commands to show indexes and constraints*** ```bash -# in browser command line or cypher shell +# in browser command line or Cypher shell # show all indexes and constraints $ :schema @@ -158,7 +157,7 @@ $ CALL db.constraints(); ***Cypher commands to create and drop indexes and constraints*** ```bash -# in browser command line or cypher shell +# in browser command line or Cypher shell # create indexes $ CALL db.index.fulltext.createNodeIndex("post_fulltext_search",["Post"],["title", "content"]); @@ -171,3 +170,21 @@ $ DROP CONSTRAINT ON ( image:Image ) ASSERT image.url IS UNIQUE # drop all indexes and constraints $ CALL apoc.schema.assert({},{},true) YIELD label, key RETURN * ; ``` + +### Database Management Commands + +***Cypher commands to manage databases*** + +```bash +# in browser command line or Cypher shell + +# show the default database +$ SHOW DEFAULT DATABASE +# show all databases +$ SHOW DATABASES +``` + +To set the default database by configuration, use `NEO4J_dbms_default__database` as an environment variable when starting Neo4j 4.4, see [Docker specific configuration settings](). + +If a database with this name does not exist, an empty database with this name is created and all other databases remain. +You can switch back to an existing database without damaging it. From c931c27a106a78952b8f3e7d46c94a3f8e2d27a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20Hu=C3=9F?= Date: Tue, 12 Dec 2023 15:52:34 +0100 Subject: [PATCH 7/9] Add description option '--database=' to dumping or loading database --- neo4j/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/neo4j/README.md b/neo4j/README.md index d9e510d8f..f5be62eb8 100644 --- a/neo4j/README.md +++ b/neo4j/README.md @@ -74,6 +74,8 @@ neo4j% exit $ docker cp :/var/lib/neo4j/neo4j-dump /$(date +%F)-neo4j-dump ``` +If you need a specific database name, add the option `--database=` to the command `neo4j-admin dump`. + ### Import Neo4j Dump To import a dump into Neo4j running in a Docker container: @@ -92,6 +94,8 @@ neo4j% neo4j-admin load --expand-commands --database=neo4j --from /var/lib/neo4j neo4j% exit ``` +If you need a specific database name, add the option `--database=` to the command `neo4j-admin load`. + ## Commands Here we describe some rarely used Cypher commands for Neo4j that are needed from time to time: From af1de3d4a27450cae35f7bc9a3d99213879a9cf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20Hu=C3=9F?= Date: Tue, 12 Dec 2023 15:53:54 +0100 Subject: [PATCH 8/9] Revert "Add database name as variable to script" This reverts commit 1dc871d3069917d9e4720f093aa320368012aa58. --- deployment/scripts/cluster.backup.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/deployment/scripts/cluster.backup.sh b/deployment/scripts/cluster.backup.sh index 88fdb7c6d..2d3bf156e 100755 --- a/deployment/scripts/cluster.backup.sh +++ b/deployment/scripts/cluster.backup.sh @@ -14,13 +14,6 @@ if [[ -z "$CONFIGURATION" ]] || [[ $CONFIGURATION == "" ]]; then exit 1 fi -# check DATABASE_NAME or set default -if [[ -z "$DATABASE_NAME" ]] || [[ $DATABASE_NAME == "" ]]; then - DATABASE_NAME="neo4j" - printf "Set satabase default name.\n" -fi -printf "Database name: '%s'\n" $DATABASE_NAME - # configuration KUBECONFIG=${KUBECONFIG:-${SCRIPT_DIR}/../configurations/${CONFIGURATION}/kubeconfig.yaml} BACKUP_DATE=$(date "+%F_%H-%M-%S") @@ -37,7 +30,7 @@ ${SCRIPT_DIR}/cluster.neo4j.sh maintenance on echo "Dumping database ..." kubectl --kubeconfig=${KUBECONFIG} -n default exec -it \ $(kubectl --kubeconfig=${KUBECONFIG} -n default get pods | grep ocelot-neo4j | awk '{ print $1 }') \ - -- neo4j-admin dump --database=$DATABASE_NAME --to=/var/lib/neo4j/$BACKUP_DATE-neo4j-dump + -- neo4j-admin dump --database=neo4j --to=/var/lib/neo4j/$BACKUP_DATE-neo4j-dump # copy neo4j backup to local drive echo "Coping database ..." kubectl --kubeconfig=${KUBECONFIG} cp \ From 0b8407ccff5d90ec0cfe7d450ebba683bfe15674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20Hu=C3=9F?= Date: Tue, 12 Dec 2023 16:13:46 +0100 Subject: [PATCH 9/9] Remove '--database=neo4j' and '--expand-commands' from all the database commands --- deployment/scripts/cluster.backup.sh | 2 +- deployment/src/kubernetes/Backup.md | 4 +++- neo4j/README.md | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/deployment/scripts/cluster.backup.sh b/deployment/scripts/cluster.backup.sh index 2d3bf156e..784bb1844 100755 --- a/deployment/scripts/cluster.backup.sh +++ b/deployment/scripts/cluster.backup.sh @@ -30,7 +30,7 @@ ${SCRIPT_DIR}/cluster.neo4j.sh maintenance on echo "Dumping database ..." kubectl --kubeconfig=${KUBECONFIG} -n default exec -it \ $(kubectl --kubeconfig=${KUBECONFIG} -n default get pods | grep ocelot-neo4j | awk '{ print $1 }') \ - -- neo4j-admin dump --database=neo4j --to=/var/lib/neo4j/$BACKUP_DATE-neo4j-dump + -- neo4j-admin dump --to=/var/lib/neo4j/$BACKUP_DATE-neo4j-dump # copy neo4j backup to local drive echo "Coping database ..." kubectl --kubeconfig=${KUBECONFIG} cp \ diff --git a/deployment/src/kubernetes/Backup.md b/deployment/src/kubernetes/Backup.md index 9f41ee7f4..bfae50776 100644 --- a/deployment/src/kubernetes/Backup.md +++ b/deployment/src/kubernetes/Backup.md @@ -137,7 +137,7 @@ $ kubectl -n default exec -it $(kubectl -n default get pods | grep ocelot-neo4j # bash: enter bash of Neo4j $ kubectl -n default exec -it $(kubectl -n default get pods | grep ocelot-neo4j | awk '{ print $1 }') -- bash # generate Dump -neo4j% neo4j-admin dump --database=neo4j --to=/var/lib/neo4j/$(date +%F)-neo4j-dump +neo4j% neo4j-admin dump --to=/var/lib/neo4j/$(date +%F)-neo4j-dump # exit bash neo4j% exit @@ -145,6 +145,8 @@ neo4j% exit $ kubectl -n default exec -it $(kubectl -n default get pods | grep ocelot-neo4j | awk '{ print $1 }') -- ls ``` +If you need a specific database name, add the option `--database=` to the command `neo4j-admin dump`. + Lets copy the dump backup ```bash diff --git a/neo4j/README.md b/neo4j/README.md index f5be62eb8..fa2773d6c 100644 --- a/neo4j/README.md +++ b/neo4j/README.md @@ -67,7 +67,7 @@ To create a dump in Neo4j running in a Docker container: # connect to the Docker containers Neo4j terminal $ docker exec -it neo4j bash # generate Dump -neo4j% neo4j-admin dump --database=neo4j --to=/var/lib/neo4j/$(date +%F)-neo4j-dump +neo4j% neo4j-admin dump --to=/var/lib/neo4j/$(date +%F)-neo4j-dump # exit bash neo4j% exit # copy the dump out of the running Docker container @@ -89,7 +89,7 @@ $ docker cp /neo4j-dump :/var/lib # connect to the Docker containers Neo4j terminal $ docker exec -it neo4j bash # to load the dump into the database we need the following command in this terminal -neo4j% neo4j-admin load --expand-commands --database=neo4j --from /var/lib/neo4j/$(date +%F)-neo4j-dump --force +neo4j% neo4j-admin load --from /var/lib/neo4j/$(date +%F)-neo4j-dump --force # leave the terminal by entering neo4j% exit ```