From ce664040c60dfb0fa4d84cb457c3be74ddcf58b5 Mon Sep 17 00:00:00 2001 From: roschaefer Date: Sun, 19 Jan 2020 17:51:27 +0100 Subject: [PATCH] docs(backend): How to create & run data migrations --- backend/README.md | 32 +++++++++++ neo4j/README.md | 14 ----- ...ge_disabled_relationship_to_report_node.sh | 55 ------------------- .../db_manipulation/add_image_aspect_ratio.sh | 22 -------- ...ge_disabled_relationship_to_report_node.sh | 51 ----------------- 5 files changed, 32 insertions(+), 142 deletions(-) delete mode 100755 neo4j/change_disabled_relationship_to_report_node.sh delete mode 100755 neo4j/db_manipulation/add_image_aspect_ratio.sh delete mode 100755 neo4j/db_manipulation/change_disabled_relationship_to_report_node.sh diff --git a/backend/README.md b/backend/README.md index c42c5ac85..7fd49faf8 100644 --- a/backend/README.md +++ b/backend/README.md @@ -111,6 +111,38 @@ $ yarn run db:reset {% endtab %} {% endtabs %} +### Data migrations + +Although Neo4J is schema-less,you might find yourself in a situation in which +you have to migrate your data e.g. because your data modeling has changed. + +{% tabs %} +{% tab title="Docker" %} +Generate a data migration file: +```bash +$ docker-compose exec backend yarn run db:migrate:create your_data_migration +# Edit the file in ./src/db/migrations/ +``` + +To run the migration: +```bash +$ docker-compose exec backend yarn run db:migrate up +``` +{% endtab %} +{% tab title="Without Docker" %} +Generate a data migration file: +```bash +$ yarn run db:migrate:create your_data_migration +# Edit the file in ./src/db/migrations/ +``` + +To run the migration: +```bash +$ yarn run db:migrate up +``` +{% endtab %} +{% endtabs %} + # Testing **Beware**: We have no multiple database setup at the moment. We clean the diff --git a/neo4j/README.md b/neo4j/README.md index 5df01cc71..a4242b512 100644 --- a/neo4j/README.md +++ b/neo4j/README.md @@ -36,20 +36,6 @@ Then make sure to allow Apoc procedures by adding the following line to your Neo ``` dbms.security.procedures.unrestricted=apoc.* ``` -### Database Indices and Constraints - -If you have `cypher-shell` available with your local installation of neo4j you -can run: - -```bash -# in folder neo4j/ -$ cp .env.template .env -$ ./db_setup.sh -``` - -Otherwise, if you don't have `cypher-shell` available, copy the cypher -statements [from the `db_setup.sh` script](https://github.com/Human-Connection/Human-Connection/blob/master/neo4j/db_setup.sh) and paste the scripts into your -[database browser frontend](http://localhost:7474). ### Alternatives diff --git a/neo4j/change_disabled_relationship_to_report_node.sh b/neo4j/change_disabled_relationship_to_report_node.sh deleted file mode 100755 index 2f44b8e59..000000000 --- a/neo4j/change_disabled_relationship_to_report_node.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -ENV_FILE=$(dirname "$0")/.env -[[ -f "$ENV_FILE" ]] && source "$ENV_FILE" - -if [ -z "$NEO4J_USERNAME" ] || [ -z "$NEO4J_PASSWORD" ]; then - echo "Please set NEO4J_USERNAME and NEO4J_PASSWORD environment variables." - echo "Database manipulation is not possible without connecting to the database." - echo "E.g. you could \`cp .env.template .env\` unless you run the script in a docker container" -fi - -until echo 'RETURN "Connection successful" as info;' | cypher-shell -do - echo "Connecting to neo4j failed, trying again..." - sleep 1 -done - -echo " -// convert old DISABLED to new REVIEWED-Report-BELONGS_TO structure -MATCH (moderator:User)-[disabled:DISABLED]->(disabledResource) -WHERE disabledResource:User OR disabledResource:Comment OR disabledResource:Post -DELETE disabled -CREATE (moderator)-[review:REVIEWED]->(report:Report)-[:BELONGS_TO]->(disabledResource) -SET review.createdAt = toString(datetime()), review.updatedAt = review.createdAt, review.disable = true -SET report.id = randomUUID(), report.createdAt = toString(datetime()), report.updatedAt = report.createdAt, report.rule = 'latestReviewUpdatedAtRules', report.closed = false - -// if disabledResource has no filed report, then create a moderators default filed report -WITH moderator, disabledResource, report -OPTIONAL MATCH (disabledResourceReporter:User)-[existingFiledReport:FILED]->(disabledResource) -FOREACH(disabledResource IN CASE WHEN existingFiledReport IS NULL THEN [1] ELSE [] END | - CREATE (moderator)-[addModeratorReport:FILED]->(report) - SET addModeratorReport.createdAt = toString(datetime()), addModeratorReport.reasonCategory = 'other', addModeratorReport.reasonDescription = 'Old DISABLED relations didn't enforce mandatory reporting !!! Created automatically to ensure database consistency! Creation date is when the database manipulation happened.' -) -FOREACH(disabledResource IN CASE WHEN existingFiledReport IS NOT NULL THEN [1] ELSE [] END | - CREATE (disabledResourceReporter)-[moveModeratorReport:FILED]->(report) - SET moveModeratorReport = existingFiledReport - DELETE existingFiledReport -) - -RETURN disabledResource {.id}; -" | cypher-shell - -echo " -// for FILED resources without DISABLED relation which are handled above, create new FILED-Report-BELONGS_TO structure -MATCH (reporter:User)-[oldReport:REPORTED]->(notDisabledResource) -WHERE notDisabledResource:User OR notDisabledResource:Comment OR notDisabledResource:Post -MERGE (report:Report)-[:BELONGS_TO]->(notDisabledResource) -ON CREATE SET report.id = randomUUID(), report.createdAt = toString(datetime()), report.updatedAt = report.createdAt, report.rule = 'latestReviewUpdatedAtRules', report.closed = false -CREATE (reporter)-[filed:FILED]->(report) -SET report = oldReport -DELETE oldReport - -RETURN notDisabledResource {.id}; -" | cypher-shell - diff --git a/neo4j/db_manipulation/add_image_aspect_ratio.sh b/neo4j/db_manipulation/add_image_aspect_ratio.sh deleted file mode 100755 index 8e2a16a01..000000000 --- a/neo4j/db_manipulation/add_image_aspect_ratio.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -if [ -z "$NEO4J_USERNAME" ] || [ -z "$NEO4J_PASSWORD" ]; then - echo "Please set NEO4J_USERNAME and NEO4J_PASSWORD environment variables." - echo "Database manipulation is not possible without connecting to the database." - echo "E.g. you could \`cp .env.template .env\` unless you run the script in a docker container" -fi - -until echo 'RETURN "Connection successful" as info;' | cypher-shell -do - echo "Connecting to neo4j failed, trying again..." - sleep 1 -done - -echo " - CALL apoc.periodic.iterate(' - CALL apoc.load.csv("out.csv") yield map as row return row - ',' - MATCH (post:Post) where post.image = row.image - set post.imageAspectRatio = row.aspectRatio - ', {batchSize:10000, iterateList:true, parallel:true}); -" | cypher-shell diff --git a/neo4j/db_manipulation/change_disabled_relationship_to_report_node.sh b/neo4j/db_manipulation/change_disabled_relationship_to_report_node.sh deleted file mode 100755 index e611382f0..000000000 --- a/neo4j/db_manipulation/change_disabled_relationship_to_report_node.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -ENV_FILE=$(dirname "$0")/.env -[[ -f "$ENV_FILE" ]] && source "$ENV_FILE" - -if [ -z "$NEO4J_USERNAME" ] || [ -z "$NEO4J_PASSWORD" ]; then - echo "Please set NEO4J_USERNAME and NEO4J_PASSWORD environment variables." - echo "Database manipulation is not possible without connecting to the database." - echo "E.g. you could \`cp .env.template .env\` unless you run the script in a docker container" -fi - -until echo 'RETURN "Connection successful" as info;' | cypher-shell -do - echo "Connecting to neo4j failed, trying again..." - sleep 1 -done - -echo " - :begin - MATCH(user)-[reported:REPORTED]->(resource) - WITH reported, resource, COLLECT(user) as users - MERGE(report:Report)-[:BELONGS_TO]->(resource) - SET report.id = randomUUID(), report.createdAt = toString(datetime()), report.updatedAt = report.createdAt, report.rule = 'latestReviewUpdatedAtRules', report.closed = false - WITH report, users, reported - UNWIND users as user - MERGE (user)-[filed:FILED]->(report) - SET filed = reported - DELETE reported; - - MATCH(moderator)-[disabled:DISABLED]->(resource) - MATCH(report:Report)-[:BELONGS_TO]->(resource) - WITH disabled, resource, COLLECT(moderator) as moderators, report - DELETE disabled - WITH report, moderators, disabled - UNWIND moderators as moderator - MERGE (moderator)-[review:REVIEWED {disable: true}]->(report) - SET review.createdAt = toString(datetime()), review.updatedAt = review.createdAt, review.disable = true; - - MATCH(moderator)-[disabled:DISABLED]->(resource) - WITH disabled, resource, COLLECT(moderator) as moderators - MERGE(report:Report)-[:BELONGS_TO]->(resource) - SET report.id = randomUUID(), report.createdAt = toString(datetime()), report.updatedAt = report.createdAt, report.rule = 'latestReviewUpdatedAtRules', report.closed = false - DELETE disabled - WITH report, moderators, disabled - UNWIND moderators as moderator - MERGE(moderator)-[filed:FILED]->(report) - SET filed.createdAt = toString(datetime()), filed.reasonCategory = 'other', filed.reasonDescription = 'Old DISABLED relations didn\'t enforce mandatory reporting !!! Created automatically to ensure database consistency! Creation date is when the database manipulation happened.' - MERGE (moderator)-[review:REVIEWED {disable: true}]->(report) - SET review.createdAt = toString(datetime()), review.updatedAt = review.createdAt, review.disable = true; - :commit -" | cypher-shell \ No newline at end of file