From c0e2675912b9c5cf1bdbc97ead9eab13f8c5aac7 Mon Sep 17 00:00:00 2001 From: roschaefer Date: Mon, 13 Jan 2020 23:25:07 +0100 Subject: [PATCH 01/12] refactor(neo4j) Setup constraints with neode --- backend/README.md | 21 ++++++++++ backend/package.json | 4 +- backend/src/migration/migrate.js | 0 backend/src/migration/setup.js | 7 ++++ backend/src/models/Category.js | 2 +- backend/src/models/Post.js | 2 +- backend/src/models/UnverifiedEmailAddress.js | 2 +- backend/src/models/User.js | 2 +- neo4j/Dockerfile | 2 - neo4j/README.md | 9 ----- neo4j/db_setup.sh | 41 -------------------- 11 files changed, 35 insertions(+), 57 deletions(-) create mode 100644 backend/src/migration/migrate.js create mode 100644 backend/src/migration/setup.js delete mode 100755 neo4j/db_setup.sh diff --git a/backend/README.md b/backend/README.md index 14e6d0ddd..5474c30d7 100644 --- a/backend/README.md +++ b/backend/README.md @@ -53,6 +53,27 @@ can issue GraphQL requests or access GraphQL Playground in the browser. ![GraphQL Playground](../.gitbook/assets/graphql-playground.png) +### Database Indices and Constraints + +Database indices and constraints need to be created when the database and the +backend is running: + +{% tabs %} +{% tab title="Docker" %} +```bash +docker-compose exec backend yarn run db:setup +``` +{% endtab %} + +{% tab title="Without Docker" %} +```bash +# in folder backend/ +# make sure your database is running on http://localhost:7474/browser/ +yarn run db:setup +``` +{% endtab %} +{% endtabs %} + #### Seed Database diff --git a/backend/package.json b/backend/package.json index cd8b3ea3f..acd0a9549 100644 --- a/backend/package.json +++ b/backend/package.json @@ -11,7 +11,9 @@ "lint": "eslint src --config .eslintrc.js", "test": "jest --forceExit --detectOpenHandles --runInBand", "db:reset": "babel-node src/seed/reset-db.js", - "db:seed": "babel-node src/seed/seed-db.js" + "db:seed": "babel-node src/seed/seed-db.js", + "db:setup": "babel-node src/migration/setup.js", + "db:migrate": "babel-node src/migration/migrate.js" }, "author": "Human Connection gGmbH", "license": "MIT", diff --git a/backend/src/migration/migrate.js b/backend/src/migration/migrate.js new file mode 100644 index 000000000..e69de29bb diff --git a/backend/src/migration/setup.js b/backend/src/migration/setup.js new file mode 100644 index 000000000..33667ff1b --- /dev/null +++ b/backend/src/migration/setup.js @@ -0,0 +1,7 @@ +import { getNeode } from '../bootstrap/neo4j' + +(async() => { + await getNeode().schema.install() + console.log('Schema installed!') + process.exit(0) +})() diff --git a/backend/src/models/Category.js b/backend/src/models/Category.js index faf5f189f..223bb4f87 100644 --- a/backend/src/models/Category.js +++ b/backend/src/models/Category.js @@ -3,7 +3,7 @@ import uuid from 'uuid/v4' export default { id: { type: 'string', primary: true, default: uuid }, name: { type: 'string', required: true, default: false }, - slug: { type: 'string' }, + slug: { type: 'string', unique: 'true' }, icon: { type: 'string', required: true, default: false }, createdAt: { type: 'string', isoDate: true, default: () => new Date().toISOString() }, updatedAt: { diff --git a/backend/src/models/Post.js b/backend/src/models/Post.js index fd1e5b2ac..c29036009 100644 --- a/backend/src/models/Post.js +++ b/backend/src/models/Post.js @@ -11,7 +11,7 @@ export default { direction: 'in', }, title: { type: 'string', disallow: [null], min: 3 }, - slug: { type: 'string', allow: [null] }, + slug: { type: 'string', allow: [null], unique: 'true', }, content: { type: 'string', disallow: [null], min: 3 }, contentExcerpt: { type: 'string', allow: [null] }, image: { type: 'string', allow: [null] }, diff --git a/backend/src/models/UnverifiedEmailAddress.js b/backend/src/models/UnverifiedEmailAddress.js index 489e8517a..c582ed011 100644 --- a/backend/src/models/UnverifiedEmailAddress.js +++ b/backend/src/models/UnverifiedEmailAddress.js @@ -1,5 +1,5 @@ export default { - email: { type: 'string', primary: true, lowercase: true, email: true }, + email: { type: 'string', lowercase: true, email: true }, createdAt: { type: 'string', isoDate: true, default: () => new Date().toISOString() }, nonce: { type: 'string', token: true }, belongsTo: { diff --git a/backend/src/models/User.js b/backend/src/models/User.js index fc352dccc..049611eb7 100644 --- a/backend/src/models/User.js +++ b/backend/src/models/User.js @@ -4,7 +4,7 @@ export default { id: { type: 'string', primary: true, default: uuid }, // TODO: should be type: 'uuid' but simplified for our tests actorId: { type: 'string', allow: [null] }, name: { type: 'string', disallow: [null], min: 3 }, - slug: { type: 'string', regex: /^[a-z0-9_-]+$/, lowercase: true }, + slug: { type: 'string', unique: 'true', regex: /^[a-z0-9_-]+$/, lowercase: true }, encryptedPassword: 'string', avatar: { type: 'string', allow: [null] }, coverImg: { type: 'string', allow: [null] }, diff --git a/neo4j/Dockerfile b/neo4j/Dockerfile index 22dabe114..b068b22b2 100644 --- a/neo4j/Dockerfile +++ b/neo4j/Dockerfile @@ -4,7 +4,5 @@ LABEL Description="Neo4J database of the Social Network Human-Connection.org wit ARG BUILD_COMMIT ENV BUILD_COMMIT=$BUILD_COMMIT -COPY db_setup.sh /usr/local/bin/db_setup - RUN apt-get update && apt-get -y install wget htop RUN wget https://github.com/neo4j-contrib/neo4j-apoc-procedures/releases/download/3.5.0.4/apoc-3.5.0.4-all.jar -P plugins/ diff --git a/neo4j/README.md b/neo4j/README.md index fe8825734..5df01cc71 100644 --- a/neo4j/README.md +++ b/neo4j/README.md @@ -18,15 +18,6 @@ docker-compose up You can access Neo4J through [http://localhost:7474/](http://localhost:7474/) for an interactive cypher shell and a visualization of the graph. -### Database Indices and Constraints - -Database indices and constraints need to be created when the database is -running. So start the container with the command above and run: - -```bash -docker-compose exec neo4j db_setup -``` - ## Installation without Docker diff --git a/neo4j/db_setup.sh b/neo4j/db_setup.sh deleted file mode 100755 index b7562d0c9..000000000 --- a/neo4j/db_setup.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -ENV_FILE=$(dirname "$0")/.env -[[ -f "$ENV_FILE" ]] && source "$ENV_FILE" -if [ -z "$NEO4J_USERNAME" ] || [ -z "$NEO4J_PASSWORD" ]; then - echo "Please set NEO4J_USERNAME and NEO4J_PASSWORD environment variables." - echo "Setting up database constraints and indexes will probably fail because of authentication errors." - echo "E.g. you could \`cp .env.template .env\` unless you run the script in a docker container" -fi - -until echo 'RETURN "Connection successful" as info;' | cypher-shell -do - echo "Connecting to neo4j failed, trying again..." - sleep 1 -done - -echo ' -RETURN "Here is a list of indexes and constraints BEFORE THE SETUP:" as info; -CALL db.indexes(); -' | cypher-shell - -echo ' -CALL db.index.fulltext.createNodeIndex("post_fulltext_search",["Post"],["title", "content"]); -CALL db.index.fulltext.createNodeIndex("user_fulltext_search",["User"],["name", "slug"]); -CREATE CONSTRAINT ON (p:Post) ASSERT p.id IS UNIQUE; -CREATE CONSTRAINT ON (c:Comment) ASSERT c.id IS UNIQUE; -CREATE CONSTRAINT ON (c:Category) ASSERT c.id IS UNIQUE; -CREATE CONSTRAINT ON (u:User) ASSERT u.id IS UNIQUE; -CREATE CONSTRAINT ON (t:Tag) ASSERT t.id IS UNIQUE; - -CREATE CONSTRAINT ON (p:Post) ASSERT p.slug IS UNIQUE; -CREATE CONSTRAINT ON (c:Category) ASSERT c.slug IS UNIQUE; -CREATE CONSTRAINT ON (u:User) ASSERT u.slug IS UNIQUE; - -CREATE CONSTRAINT ON (e:EmailAddress) ASSERT e.email IS UNIQUE; -' | cypher-shell - -echo ' -RETURN "Setting up all the indexes and constraints seems to have been successful. Here is a list AFTER THE SETUP:" as info; -CALL db.indexes(); -' | cypher-shell From 30268dec0943b4d8ad46418fca98bcb2fe75030b Mon Sep 17 00:00:00 2001 From: roschaefer Date: Tue, 14 Jan 2020 01:47:03 +0100 Subject: [PATCH 02/12] build(deps): Add `migrate` for neo4j data migrations Implement a migration to merge duplicate user accounts with reactive programming. Those duplicate user accounts existed, because around 40 users have decided to register again while we experienced a bug related to normalized emails in our database. --- backend/migrations/1579387929122-foo.js | 69 +++++++++++++++++ backend/package.json | 6 +- backend/src/db/migrationTemplate.js | 7 ++ .../src/{migration/setup.js => db/setup.sj} | 0 backend/src/migration/migrate.js | 0 backend/yarn.lock | 74 +++++++++++++++++-- 6 files changed, 146 insertions(+), 10 deletions(-) create mode 100644 backend/migrations/1579387929122-foo.js create mode 100644 backend/src/db/migrationTemplate.js rename backend/src/{migration/setup.js => db/setup.sj} (100%) delete mode 100644 backend/src/migration/migrate.js diff --git a/backend/migrations/1579387929122-foo.js b/backend/migrations/1579387929122-foo.js new file mode 100644 index 000000000..9d23051ac --- /dev/null +++ b/backend/migrations/1579387929122-foo.js @@ -0,0 +1,69 @@ +import { throwError, of, concat } from 'rxjs' +import { tap, flatMap, mergeMap, map, catchError, filter } from 'rxjs/operators' +import CONFIG from '../src/config' +import { getNeode, getDriver } from '../src/bootstrap/neo4j' +import normalizeEmail from '../src/schema/resolvers//helpers/normalizeEmail' + + +export function up (next) { + const driver = getDriver() + const rxSession = driver.rxSession() + rxSession + .beginTransaction() + .pipe( + flatMap(txc => + concat( + txc + .run("MATCH (email:EmailAddress) RETURN email {.email}") + .records() + .pipe( + map(record => { + const { email } = record.get('email') + const normalizedEmail = normalizeEmail(email) + return { email, normalizedEmail } + }), + filter(({email, normalizedEmail}) => email !== normalizedEmail), + mergeMap(({email, normalizedEmail})=> { + return txc + .run(` + MATCH (oldUser:User)-[:PRIMARY_EMAIL]->(oldEmail:EmailAddress {email: $email}), (oldUser)-[previousRelationship]-(oldEmail) + MATCH (user:User)-[:PRIMARY_EMAIL]->(email:EmailAddress {email: $normalizedEmail}) + DELETE previousRelationship + WITH oldUser, oldEmail, user, email + CALL apoc.refactor.mergeNodes([user, oldUser], { properties: 'discard', mergeRels: true }) YIELD node as mergedUser + CALL apoc.refactor.mergeNodes([email, oldEmail], { properties: 'discard', mergeRels: true }) YIELD node as mergedEmail + RETURN user {.*}, email {.*} + `, { email, normalizedEmail }) + .records() + .pipe( + map(r => ({ + oldEmail: email, + email: r.get('email'), + user: r.get('user'), + })) + ) + }), + ), + txc.commit(), + ).pipe(catchError(err => txc.rollback().pipe(throwError(err)))) + ) + ) + .subscribe({ + next: ({ user, email, oldUser, oldEmail }) => console.log(` + Merged: + ============================= + userId: ${user.id} + email: ${oldEmail} => ${email.email} + ============================= + `), + complete: () => { + console.log('Merging of duplicate users completed') + next() + }, + error: error => throw new Error(error) + }) +} + +export function down () { + throw new Error("Irreversible migration") +} diff --git a/backend/package.json b/backend/package.json index acd0a9549..4d23406c7 100644 --- a/backend/package.json +++ b/backend/package.json @@ -12,8 +12,9 @@ "test": "jest --forceExit --detectOpenHandles --runInBand", "db:reset": "babel-node src/seed/reset-db.js", "db:seed": "babel-node src/seed/seed-db.js", - "db:setup": "babel-node src/migration/setup.js", - "db:migrate": "babel-node src/migration/migrate.js" + "db:setup": "babel-node src/db/setup.js", + "db:migrate:create": "migrate create --template-file./src/db/migrationTemplate.js", + "db:migrate": "migrate --compiler 'js:@babel/register'" }, "author": "Human Connection gGmbH", "license": "MIT", @@ -80,6 +81,7 @@ "metascraper-url": "^5.10.3", "metascraper-video": "^5.10.3", "metascraper-youtube": "^5.10.5", + "migrate": "^1.6.2", "minimatch": "^3.0.4", "mustache": "^4.0.0", "neo4j-driver": "^4.0.1", diff --git a/backend/src/db/migrationTemplate.js b/backend/src/db/migrationTemplate.js new file mode 100644 index 000000000..dff6564ef --- /dev/null +++ b/backend/src/db/migrationTemplate.js @@ -0,0 +1,7 @@ +export function up (next) { + next() +} + +export function down (next) { + next() +} diff --git a/backend/src/migration/setup.js b/backend/src/db/setup.sj similarity index 100% rename from backend/src/migration/setup.js rename to backend/src/db/setup.sj diff --git a/backend/src/migration/migrate.js b/backend/src/migration/migrate.js deleted file mode 100644 index e69de29bb..000000000 diff --git a/backend/yarn.lock b/backend/yarn.lock index 8c67f5a30..db1062af0 100644 --- a/backend/yarn.lock +++ b/backend/yarn.lock @@ -1680,6 +1680,11 @@ ansi-regex@^4.0.0, ansi-regex@^4.1.0: resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-4.1.0.tgz#8b9f8f08cf1acb843756a839ca8c7e3168c51997" integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg== +ansi-styles@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" + integrity sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4= + ansi-styles@^3.1.0, ansi-styles@^3.2.0, ansi-styles@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" @@ -2492,6 +2497,17 @@ chalk@2.3.0: escape-string-regexp "^1.0.5" supports-color "^4.0.0" +chalk@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" + integrity sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg= + dependencies: + ansi-styles "^2.2.1" + escape-string-regexp "^1.0.2" + has-ansi "^2.0.0" + strip-ansi "^3.0.0" + supports-color "^2.0.0" + chalk@^2.0.0, chalk@^2.0.1, chalk@^2.1.0, chalk@^2.4.1, chalk@^2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" @@ -2704,6 +2720,11 @@ combined-stream@^1.0.6, combined-stream@~1.0.6: dependencies: delayed-stream "~1.0.0" +commander@^2.9.0, commander@~2.20.3: + version "2.20.3" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" + integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== + commander@^3.0.1: version "3.0.2" resolved "https://registry.yarnpkg.com/commander/-/commander-3.0.2.tgz#6837c3fb677ad9933d1cfba42dd14d5117d6b39e" @@ -2714,11 +2735,6 @@ commander@^4.0.1: resolved "https://registry.yarnpkg.com/commander/-/commander-4.0.1.tgz#b67622721785993182e807f4883633e6401ba53c" integrity sha512-IPF4ouhCP+qdlcmCedhxX4xiGBPyigb8v5NeUp+0LyhwLgxMqyp3S0vl7TAPfS/hiP7FC3caI/PB9lTmP8r1NA== -commander@~2.20.3: - version "2.20.3" - resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" - integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== - commondir@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b" @@ -3001,6 +3017,11 @@ date-fns@2.9.0: resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.9.0.tgz#d0b175a5c37ed5f17b97e2272bbc1fa5aec677d2" integrity sha512-khbFLu/MlzLjEzy9Gh8oY1hNt/Dvxw3J6Rbc28cVoYWQaC1S3YI4xwkF9ZWcjDLscbZlY9hISMr66RFzZagLsA== +dateformat@^2.0.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-2.2.0.tgz#4065e2013cf9fb916ddfd82efb506ad4c6769062" + integrity sha1-QGXiATz5+5Ft39gu+1Bq1MZ2kGI= + dayjs@^1.8.19: version "1.8.19" resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.8.19.tgz#5117dc390d8f8e586d53891dbff3fa308f51abfe" @@ -3423,7 +3444,7 @@ escape-html@~1.0.3: resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" integrity sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg= -escape-string-regexp@^1.0.5: +escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ= @@ -3834,7 +3855,6 @@ extsprintf@^1.2.0: faker@Marak/faker.js#master: version "4.1.0" - uid "3b2fa4aebccee52ae1bafc15d575061fb30c3cf1" resolved "https://codeload.github.com/Marak/faker.js/tar.gz/3b2fa4aebccee52ae1bafc15d575061fb30c3cf1" fast-deep-equal@^2.0.1: @@ -4387,6 +4407,13 @@ har-validator@~5.1.0: ajv "^6.5.5" har-schema "^2.0.0" +has-ansi@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" + integrity sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE= + dependencies: + ansi-regex "^2.0.0" + has-flag@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-2.0.0.tgz#e8207af1cc7b30d446cc70b734b5e8be18f88d51" @@ -6119,6 +6146,20 @@ micromatch@^4.0.2: braces "^3.0.1" picomatch "^2.0.5" +migrate@^1.6.2: + version "1.6.2" + resolved "https://registry.yarnpkg.com/migrate/-/migrate-1.6.2.tgz#8970d596780553fe9f545bdf83806df8473f025b" + integrity sha512-XAFab+ArPTo9BHzmihKjsZ5THKRryenA+lwob0R+ax0hLDs7YzJFJT5YZE3gtntZgzdgcuFLs82EJFB/Dssr+g== + dependencies: + chalk "^1.1.3" + commander "^2.9.0" + dateformat "^2.0.0" + dotenv "^4.0.0" + inherits "^2.0.3" + minimatch "^3.0.3" + mkdirp "^0.5.1" + slug "^0.9.2" + mime-db@1.40.0: version "1.40.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.40.0.tgz#a65057e998db090f732a68f6c276d387d4126c32" @@ -6163,7 +6204,7 @@ mimic-response@^1.0.0, mimic-response@^1.0.1: resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b" integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ== -minimatch@^3.0.4: +minimatch@^3.0.3, minimatch@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== @@ -7704,6 +7745,11 @@ serve-static@1.14.1: version "1.14.1" resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.14.1.tgz#666e636dc4f010f7ef29970a88a674320898b2f9" integrity sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg== + dependencies: + encodeurl "~1.0.2" + escape-html "~1.0.3" + parseurl "~1.3.3" + send "0.17.1" set-blocking@^2.0.0, set-blocking@~2.0.0: version "2.0.0" @@ -7784,6 +7830,13 @@ slice-ansi@^2.1.0: astral-regex "^1.0.0" is-fullwidth-code-point "^2.0.0" +slug@^0.9.2: + version "0.9.4" + resolved "https://registry.yarnpkg.com/slug/-/slug-0.9.4.tgz#fad5f1ef33150830c7688cd8500514576eccabd8" + integrity sha512-3YHq0TeJ4+AIFbJm+4UWSQs5A1mmeWOTQqydW3OoPmQfNKxlO96NDRTIrp+TBkmvEsEFrd+Z/LXw8OD/6OlZ5g== + dependencies: + unicode ">= 0.3.1" + slug@~2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/slug/-/slug-2.1.0.tgz#293f8d53de7e55c15871846fd1bc36114841a8c7" @@ -8151,6 +8204,11 @@ supertest@~4.0.2: methods "^1.1.2" superagent "^3.8.3" +supports-color@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" + integrity sha1-U10EXOa2Nj+kARcIRimZXp3zJMc= + supports-color@^4.0.0: version "4.5.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-4.5.0.tgz#be7a0de484dec5c5cddf8b3d59125044912f635b" From daf2c40caec80587354cb1670a28b950afe181bf Mon Sep 17 00:00:00 2001 From: roschaefer Date: Sun, 19 Jan 2020 13:13:52 +0100 Subject: [PATCH 03/12] Implement Neo4J store for `migrate` --- backend/package.json | 8 ++- backend/src/db/migrate/store.js | 57 +++++++++++++++++++ backend/src/db/migrate/template.js | 7 +++ backend/src/db/migrationTemplate.js | 7 --- ...87929122-merge_duplicate_user_accounts.js} | 14 ++++- backend/src/db/{setup.sj => setup.js} | 0 backend/src/models/Post.js | 2 +- backend/yarn.lock | 2 +- 8 files changed, 82 insertions(+), 15 deletions(-) create mode 100644 backend/src/db/migrate/store.js create mode 100644 backend/src/db/migrate/template.js delete mode 100644 backend/src/db/migrationTemplate.js rename backend/{migrations/1579387929122-foo.js => src/db/migrations/1579387929122-merge_duplicate_user_accounts.js} (77%) rename backend/src/db/{setup.sj => setup.js} (100%) diff --git a/backend/package.json b/backend/package.json index 4d23406c7..bbdf951ad 100644 --- a/backend/package.json +++ b/backend/package.json @@ -13,8 +13,10 @@ "db:reset": "babel-node src/seed/reset-db.js", "db:seed": "babel-node src/seed/seed-db.js", "db:setup": "babel-node src/db/setup.js", - "db:migrate:create": "migrate create --template-file./src/db/migrationTemplate.js", - "db:migrate": "migrate --compiler 'js:@babel/register'" + "__migrate": "migrate --compiler 'js:@babel/register' --migrations-dir ./src/db/migrations", + "db:migrate": "yarn run __migrate --store ./src/db/migrate/store.js", + "db:migrate:create": "yarn run __migrate --template-file ./src/db/migrate/template.js create", + "production:db:migrate": "migrate --migrations-dir ./dist/db/migrations --store ./dist/db/migrate/store.js" }, "author": "Human Connection gGmbH", "license": "MIT", @@ -106,7 +108,7 @@ "@babel/node": "~7.8.3", "@babel/plugin-proposal-throw-expressions": "^7.8.3", "@babel/preset-env": "~7.8.3", - "@babel/register": "~7.8.3", + "@babel/register": "^7.8.3", "apollo-server-testing": "~2.9.16", "babel-core": "~7.0.0-0", "babel-eslint": "~10.0.3", diff --git a/backend/src/db/migrate/store.js b/backend/src/db/migrate/store.js new file mode 100644 index 000000000..68bf4e8d9 --- /dev/null +++ b/backend/src/db/migrate/store.js @@ -0,0 +1,57 @@ +import { getDriver } from '../../bootstrap/neo4j' + +class Store { + async load(fn) { + const driver = getDriver() + const session = driver.session() + const readTxResultPromise = session.readTransaction(async txc => { + const result = await txc.run( + 'MATCH (migration:Migration) RETURN migration {.*} ORDER BY migration.timestamp DESC', + ) + return result.records.map(r => r.get('migration')) + }) + try { + const migrations = await readTxResultPromise + if (migrations.length <= 0) { + // eslint-disable-next-line no-console + console.log( + "No migrations found in database. If it's the first time you run migrations, then this is normal.", + ) + return fn(null, {}) + } + const [{ title: lastRun }] = migrations + fn(null, { lastRun, migrations }) + } catch (error) { + console.log(error) // eslint-disable-line no-console + } finally { + session.close() + } + } + + async save(set, fn) { + const driver = getDriver() + const session = driver.session() + const { migrations } = set + const writeTxResultPromise = session.writeTransaction(txc => { + return Promise.all( + migrations.map(migration => { + const { title, description, timestamp } = migration + const properties = { title, description, timestamp } + return txc.run('CREATE (migration:Migration) SET migration += $properties', { + properties, + }) + }), + ) + }) + try { + await writeTxResultPromise + } catch (error) { + console.log(error) // eslint-disable-line no-console + } finally { + session.close() + fn() + } + } +} + +module.exports = Store diff --git a/backend/src/db/migrate/template.js b/backend/src/db/migrate/template.js new file mode 100644 index 000000000..941f2a9e3 --- /dev/null +++ b/backend/src/db/migrate/template.js @@ -0,0 +1,7 @@ +export function up(next) { + next() +} + +export function down(next) { + next() +} diff --git a/backend/src/db/migrationTemplate.js b/backend/src/db/migrationTemplate.js deleted file mode 100644 index dff6564ef..000000000 --- a/backend/src/db/migrationTemplate.js +++ /dev/null @@ -1,7 +0,0 @@ -export function up (next) { - next() -} - -export function down (next) { - next() -} diff --git a/backend/migrations/1579387929122-foo.js b/backend/src/db/migrations/1579387929122-merge_duplicate_user_accounts.js similarity index 77% rename from backend/migrations/1579387929122-foo.js rename to backend/src/db/migrations/1579387929122-merge_duplicate_user_accounts.js index 9d23051ac..7d2abcdeb 100644 --- a/backend/migrations/1579387929122-foo.js +++ b/backend/src/db/migrations/1579387929122-merge_duplicate_user_accounts.js @@ -1,10 +1,18 @@ import { throwError, of, concat } from 'rxjs' import { tap, flatMap, mergeMap, map, catchError, filter } from 'rxjs/operators' -import CONFIG from '../src/config' -import { getNeode, getDriver } from '../src/bootstrap/neo4j' -import normalizeEmail from '../src/schema/resolvers//helpers/normalizeEmail' +import CONFIG from '../../src/config' +import { getNeode, getDriver } from '../../src/bootstrap/neo4j' +import normalizeEmail from '../../src/schema/resolvers//helpers/normalizeEmail' +export const description = ` + This migration merges duplicate :User and :EmailAddress nodes. It became + necessary after we implemented the email normalization but forgot to migrate + the existing data. Some (40) users decided to just register with a new account + but the same email address. On signup our backend would normalize the email, + which is good, but would also keep the existing unnormalized email address. + This led to about 40 duplicate user and email address nodes in our database. +` export function up (next) { const driver = getDriver() const rxSession = driver.rxSession() diff --git a/backend/src/db/setup.sj b/backend/src/db/setup.js similarity index 100% rename from backend/src/db/setup.sj rename to backend/src/db/setup.js diff --git a/backend/src/models/Post.js b/backend/src/models/Post.js index c29036009..e2e153a1b 100644 --- a/backend/src/models/Post.js +++ b/backend/src/models/Post.js @@ -11,7 +11,7 @@ export default { direction: 'in', }, title: { type: 'string', disallow: [null], min: 3 }, - slug: { type: 'string', allow: [null], unique: 'true', }, + slug: { type: 'string', allow: [null], unique: 'true' }, content: { type: 'string', disallow: [null], min: 3 }, contentExcerpt: { type: 'string', allow: [null] }, image: { type: 'string', allow: [null] }, diff --git a/backend/yarn.lock b/backend/yarn.lock index db1062af0..2c1b2efee 100644 --- a/backend/yarn.lock +++ b/backend/yarn.lock @@ -751,7 +751,7 @@ levenary "^1.1.0" semver "^5.5.0" -"@babel/register@^7.8.3", "@babel/register@~7.8.3": +"@babel/register@^7.8.3": version "7.8.3" resolved "https://registry.yarnpkg.com/@babel/register/-/register-7.8.3.tgz#5d5d30cfcc918437535d724b8ac1e4a60c5db1f8" integrity sha512-t7UqebaWwo9nXWClIPLPloa5pN33A2leVs8Hf0e9g9YwUP8/H9NeR7DJU+4CXo23QtjChQv5a3DjEtT83ih1rg== From b063847849a84db885337dc8e84e75ddaf87011f Mon Sep 17 00:00:00 2001 From: roschaefer Date: Sun, 19 Jan 2020 23:53:36 +0100 Subject: [PATCH 04/12] refactor: Make `db:setup` init stage of `migrate` --- .travis.yml | 2 +- backend/README.md | 6 +- backend/package.json | 6 +- .../src/activitypub/routes/webfinger.spec.js | 4 +- backend/src/{seed/reset-db.js => db/clean.js} | 2 +- backend/src/db/migrate/store.js | 10 +- ...387929122-merge_duplicate_user_accounts.js | 89 ++++++------ backend/src/{bootstrap => db}/neo4j.js | 0 backend/src/{seed/seed-db.js => db/seed.js} | 4 +- backend/src/db/setup.js | 7 - backend/src/{seed => }/factories/badges.js | 0 .../src/{seed => }/factories/categories.js | 0 backend/src/{seed => }/factories/comments.js | 0 backend/src/{seed => }/factories/donations.js | 0 .../{seed => }/factories/emailAddresses.js | 0 backend/src/{seed => }/factories/index.js | 2 +- backend/src/{seed => }/factories/locations.js | 0 backend/src/{seed => }/factories/posts.js | 0 backend/src/{seed => }/factories/reports.js | 0 .../src/{seed => }/factories/socialMedia.js | 0 backend/src/{seed => }/factories/tags.js | 0 .../factories/unverifiedEmailAddresses.js | 0 backend/src/{seed => }/factories/users.js | 2 +- backend/src/jest/helpers.js | 5 - backend/src/jwt/decode.spec.js | 4 +- .../hashtags/hashtagsMiddleware.spec.js | 4 +- .../notificationsMiddleware.spec.js | 4 +- .../src/middleware/orderByMiddleware.spec.js | 4 +- .../src/middleware/permissionsMiddleware.js | 2 +- .../middleware/permissionsMiddleware.spec.js | 4 +- .../src/middleware/slugifyMiddleware.spec.js | 4 +- .../softDelete/softDeleteMiddleware.spec.js | 4 +- .../validation/validationMiddleware.spec.js | 4 +- backend/src/models/User.spec.js | 4 +- backend/src/schema/resolvers/comments.spec.js | 4 +- .../src/schema/resolvers/donations.spec.js | 4 +- backend/src/schema/resolvers/emails.spec.js | 4 +- backend/src/schema/resolvers/follow.js | 2 +- backend/src/schema/resolvers/follow.spec.js | 4 +- .../src/schema/resolvers/locations.spec.js | 4 +- .../src/schema/resolvers/moderation.spec.js | 4 +- .../schema/resolvers/notifications.spec.js | 4 +- .../schema/resolvers/passwordReset.spec.js | 4 +- backend/src/schema/resolvers/posts.spec.js | 4 +- backend/src/schema/resolvers/registration.js | 2 +- .../src/schema/resolvers/registration.spec.js | 4 +- backend/src/schema/resolvers/reports.spec.js | 4 +- backend/src/schema/resolvers/rewards.js | 2 +- backend/src/schema/resolvers/rewards.spec.js | 4 +- backend/src/schema/resolvers/shout.spec.js | 4 +- backend/src/schema/resolvers/socialMedia.js | 2 +- .../src/schema/resolvers/socialMedia.spec.js | 4 +- .../src/schema/resolvers/statistics.spec.js | 4 +- .../src/schema/resolvers/user_management.js | 2 +- .../schema/resolvers/user_management.spec.js | 4 +- backend/src/schema/resolvers/users.js | 2 +- backend/src/schema/resolvers/users.spec.js | 4 +- .../resolvers/users/blockedUsers.spec.js | 4 +- .../schema/resolvers/users/location.spec.js | 4 +- backend/src/seed/seed-helpers.js | 134 ------------------ backend/src/server.js | 2 +- backend/test/features/support/steps.js | 2 +- cypress/support/factories.js | 4 +- features/support/steps.js | 2 +- 64 files changed, 139 insertions(+), 270 deletions(-) rename backend/src/{seed/reset-db.js => db/clean.js} (91%) rename backend/src/{bootstrap => db}/neo4j.js (100%) rename backend/src/{seed/seed-db.js => db/seed.js} (99%) delete mode 100644 backend/src/db/setup.js rename backend/src/{seed => }/factories/badges.js (100%) rename backend/src/{seed => }/factories/categories.js (100%) rename backend/src/{seed => }/factories/comments.js (100%) rename backend/src/{seed => }/factories/donations.js (100%) rename backend/src/{seed => }/factories/emailAddresses.js (100%) rename backend/src/{seed => }/factories/index.js (96%) rename backend/src/{seed => }/factories/locations.js (100%) rename backend/src/{seed => }/factories/posts.js (100%) rename backend/src/{seed => }/factories/reports.js (100%) rename backend/src/{seed => }/factories/socialMedia.js (100%) rename backend/src/{seed => }/factories/tags.js (100%) rename backend/src/{seed => }/factories/unverifiedEmailAddresses.js (100%) rename backend/src/{seed => }/factories/users.js (95%) delete mode 100644 backend/src/jest/helpers.js delete mode 100644 backend/src/seed/seed-helpers.js diff --git a/.travis.yml b/.travis.yml index 19ba3ff9d..110655c79 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ before_script: - docker-compose -f docker-compose.yml -f docker-compose.build-and-test.yml build # just tagging, just be quite fast - docker-compose -f docker-compose.yml -f docker-compose.build-and-test.yml up -d - wait-on http://localhost:7474 - - docker-compose -f docker-compose.yml -f docker-compose.build-and-test.yml exec neo4j db_setup + - docker-compose -f docker-compose.yml -f docker-compose.build-and-test.yml exec backend yarn run db:migrate init script: - export CYPRESS_RETRIES=1 diff --git a/backend/README.md b/backend/README.md index 5474c30d7..c42c5ac85 100644 --- a/backend/README.md +++ b/backend/README.md @@ -61,7 +61,7 @@ backend is running: {% tabs %} {% tab title="Docker" %} ```bash -docker-compose exec backend yarn run db:setup +docker-compose exec backend yarn run db:migrate init ``` {% endtab %} @@ -69,7 +69,7 @@ docker-compose exec backend yarn run db:setup ```bash # in folder backend/ # make sure your database is running on http://localhost:7474/browser/ -yarn run db:setup +yarn run db:migrate init ``` {% endtab %} {% endtabs %} @@ -94,7 +94,7 @@ $ docker-compose exec backend yarn run db:reset # you could also wipe out your neo4j database and delete all volumes with: $ docker-compose down -v # if container is not running, run this command to set up your database indeces and contstraints -$ docker-compose run neo4j db_setup +$ docker-compose run backend yarn run db:migrate init ``` {% endtab %} diff --git a/backend/package.json b/backend/package.json index bbdf951ad..3c9fda287 100644 --- a/backend/package.json +++ b/backend/package.json @@ -10,9 +10,9 @@ "dev:debug": "nodemon --exec babel-node --inspect=0.0.0.0:9229 src/ -e js,gql", "lint": "eslint src --config .eslintrc.js", "test": "jest --forceExit --detectOpenHandles --runInBand", - "db:reset": "babel-node src/seed/reset-db.js", - "db:seed": "babel-node src/seed/seed-db.js", - "db:setup": "babel-node src/db/setup.js", + "db:clean": "babel-node src/db/clean.js", + "db:reset": "yarn run db:clean", + "db:seed": "babel-node src/db/seed.js", "__migrate": "migrate --compiler 'js:@babel/register' --migrations-dir ./src/db/migrations", "db:migrate": "yarn run __migrate --store ./src/db/migrate/store.js", "db:migrate:create": "yarn run __migrate --template-file ./src/db/migrate/template.js create", diff --git a/backend/src/activitypub/routes/webfinger.spec.js b/backend/src/activitypub/routes/webfinger.spec.js index 4e9b2196d..06ca4577d 100644 --- a/backend/src/activitypub/routes/webfinger.spec.js +++ b/backend/src/activitypub/routes/webfinger.spec.js @@ -1,6 +1,6 @@ import { handler } from './webfinger' -import Factory from '../../seed/factories' -import { getDriver } from '../../bootstrap/neo4j' +import Factory from '../../factories' +import { getDriver } from '../../db/neo4j' let resource, res, json, status, contentType diff --git a/backend/src/seed/reset-db.js b/backend/src/db/clean.js similarity index 91% rename from backend/src/seed/reset-db.js rename to backend/src/db/clean.js index 125d135d8..cbb1412e2 100644 --- a/backend/src/seed/reset-db.js +++ b/backend/src/db/clean.js @@ -1,4 +1,4 @@ -import { cleanDatabase } from './factories' +import { cleanDatabase } from '../factories' if (process.env.NODE_ENV === 'production') { throw new Error(`You cannot clean the database in production environment!`) diff --git a/backend/src/db/migrate/store.js b/backend/src/db/migrate/store.js index 68bf4e8d9..9984b3971 100644 --- a/backend/src/db/migrate/store.js +++ b/backend/src/db/migrate/store.js @@ -1,6 +1,14 @@ -import { getDriver } from '../../bootstrap/neo4j' +import { getDriver, getNeode } from '../../db/neo4j' class Store { + async init(fn) { + const neode = getNeode() + await getNeode().schema.install() + // eslint-disable-next-line no-console + console.log('Successfully created database indices and constraints!') + neode.driver.close() + } + async load(fn) { const driver = getDriver() const session = driver.session() diff --git a/backend/src/db/migrations/1579387929122-merge_duplicate_user_accounts.js b/backend/src/db/migrations/1579387929122-merge_duplicate_user_accounts.js index 7d2abcdeb..9dbc8ad05 100644 --- a/backend/src/db/migrations/1579387929122-merge_duplicate_user_accounts.js +++ b/backend/src/db/migrations/1579387929122-merge_duplicate_user_accounts.js @@ -1,8 +1,7 @@ -import { throwError, of, concat } from 'rxjs' -import { tap, flatMap, mergeMap, map, catchError, filter } from 'rxjs/operators' -import CONFIG from '../../src/config' -import { getNeode, getDriver } from '../../src/bootstrap/neo4j' -import normalizeEmail from '../../src/schema/resolvers//helpers/normalizeEmail' +import { throwError, concat } from 'rxjs' +import { flatMap, mergeMap, map, catchError, filter } from 'rxjs/operators' +import { getDriver } from '../neo4j' +import normalizeEmail from '../../schema/resolvers//helpers/normalizeEmail' export const description = ` This migration merges duplicate :User and :EmailAddress nodes. It became @@ -13,7 +12,7 @@ export const description = ` This led to about 40 duplicate user and email address nodes in our database. ` -export function up (next) { +export function up(next) { const driver = getDriver() const rxSession = driver.rxSession() rxSession @@ -22,18 +21,19 @@ export function up (next) { flatMap(txc => concat( txc - .run("MATCH (email:EmailAddress) RETURN email {.email}") - .records() - .pipe( - map(record => { - const { email } = record.get('email') - const normalizedEmail = normalizeEmail(email) - return { email, normalizedEmail } - }), - filter(({email, normalizedEmail}) => email !== normalizedEmail), - mergeMap(({email, normalizedEmail})=> { - return txc - .run(` + .run('MATCH (email:EmailAddress) RETURN email {.email}') + .records() + .pipe( + map(record => { + const { email } = record.get('email') + const normalizedEmail = normalizeEmail(email) + return { email, normalizedEmail } + }), + filter(({ email, normalizedEmail }) => email !== normalizedEmail), + mergeMap(({ email, normalizedEmail }) => { + return txc + .run( + ` MATCH (oldUser:User)-[:PRIMARY_EMAIL]->(oldEmail:EmailAddress {email: $email}), (oldUser)-[previousRelationship]-(oldEmail) MATCH (user:User)-[:PRIMARY_EMAIL]->(email:EmailAddress {email: $normalizedEmail}) DELETE previousRelationship @@ -41,37 +41,44 @@ export function up (next) { CALL apoc.refactor.mergeNodes([user, oldUser], { properties: 'discard', mergeRels: true }) YIELD node as mergedUser CALL apoc.refactor.mergeNodes([email, oldEmail], { properties: 'discard', mergeRels: true }) YIELD node as mergedEmail RETURN user {.*}, email {.*} - `, { email, normalizedEmail }) - .records() - .pipe( - map(r => ({ - oldEmail: email, - email: r.get('email'), - user: r.get('user'), - })) - ) - }), - ), + `, + { email, normalizedEmail }, + ) + .records() + .pipe( + map(r => ({ + oldEmail: email, + email: r.get('email'), + user: r.get('user'), + })), + ) + }), + ), txc.commit(), - ).pipe(catchError(err => txc.rollback().pipe(throwError(err)))) - ) + ).pipe(catchError(err => txc.rollback().pipe(throwError(err)))), + ), ) .subscribe({ - next: ({ user, email, oldUser, oldEmail }) => console.log(` - Merged: - ============================= - userId: ${user.id} - email: ${oldEmail} => ${email.email} - ============================= - `), + next: ({ user, email, oldUser, oldEmail }) => + // eslint-disable-next-line no-console + console.log(` + Merged: + ============================= + userId: ${user.id} + email: ${oldEmail} => ${email.email} + ============================= + `), complete: () => { + // eslint-disable-next-line no-console console.log('Merging of duplicate users completed') next() }, - error: error => throw new Error(error) + error: error => { + throw new Error(error) + }, }) } -export function down () { - throw new Error("Irreversible migration") +export function down() { + throw new Error('Irreversible migration') } diff --git a/backend/src/bootstrap/neo4j.js b/backend/src/db/neo4j.js similarity index 100% rename from backend/src/bootstrap/neo4j.js rename to backend/src/db/neo4j.js diff --git a/backend/src/seed/seed-db.js b/backend/src/db/seed.js similarity index 99% rename from backend/src/seed/seed-db.js rename to backend/src/db/seed.js index 4178169bb..f3c46b0d4 100644 --- a/backend/src/seed/seed-db.js +++ b/backend/src/db/seed.js @@ -2,8 +2,8 @@ import faker from 'faker' import sample from 'lodash/sample' import { createTestClient } from 'apollo-server-testing' import createServer from '../server' -import Factory from './factories' -import { getNeode, getDriver } from '../bootstrap/neo4j' +import Factory from '../factories' +import { getNeode, getDriver } from '../db/neo4j' import { gql } from '../helpers/jest' const languages = ['de', 'en', 'es', 'fr', 'it', 'pt', 'pl'] diff --git a/backend/src/db/setup.js b/backend/src/db/setup.js deleted file mode 100644 index 33667ff1b..000000000 --- a/backend/src/db/setup.js +++ /dev/null @@ -1,7 +0,0 @@ -import { getNeode } from '../bootstrap/neo4j' - -(async() => { - await getNeode().schema.install() - console.log('Schema installed!') - process.exit(0) -})() diff --git a/backend/src/seed/factories/badges.js b/backend/src/factories/badges.js similarity index 100% rename from backend/src/seed/factories/badges.js rename to backend/src/factories/badges.js diff --git a/backend/src/seed/factories/categories.js b/backend/src/factories/categories.js similarity index 100% rename from backend/src/seed/factories/categories.js rename to backend/src/factories/categories.js diff --git a/backend/src/seed/factories/comments.js b/backend/src/factories/comments.js similarity index 100% rename from backend/src/seed/factories/comments.js rename to backend/src/factories/comments.js diff --git a/backend/src/seed/factories/donations.js b/backend/src/factories/donations.js similarity index 100% rename from backend/src/seed/factories/donations.js rename to backend/src/factories/donations.js diff --git a/backend/src/seed/factories/emailAddresses.js b/backend/src/factories/emailAddresses.js similarity index 100% rename from backend/src/seed/factories/emailAddresses.js rename to backend/src/factories/emailAddresses.js diff --git a/backend/src/seed/factories/index.js b/backend/src/factories/index.js similarity index 96% rename from backend/src/seed/factories/index.js rename to backend/src/factories/index.js index ff6a52a76..c3ab14f64 100644 --- a/backend/src/seed/factories/index.js +++ b/backend/src/factories/index.js @@ -1,4 +1,4 @@ -import { getDriver, getNeode } from '../../bootstrap/neo4j' +import { getDriver, getNeode } from '../db/neo4j' const factories = { Badge: require('./badges.js').default, diff --git a/backend/src/seed/factories/locations.js b/backend/src/factories/locations.js similarity index 100% rename from backend/src/seed/factories/locations.js rename to backend/src/factories/locations.js diff --git a/backend/src/seed/factories/posts.js b/backend/src/factories/posts.js similarity index 100% rename from backend/src/seed/factories/posts.js rename to backend/src/factories/posts.js diff --git a/backend/src/seed/factories/reports.js b/backend/src/factories/reports.js similarity index 100% rename from backend/src/seed/factories/reports.js rename to backend/src/factories/reports.js diff --git a/backend/src/seed/factories/socialMedia.js b/backend/src/factories/socialMedia.js similarity index 100% rename from backend/src/seed/factories/socialMedia.js rename to backend/src/factories/socialMedia.js diff --git a/backend/src/seed/factories/tags.js b/backend/src/factories/tags.js similarity index 100% rename from backend/src/seed/factories/tags.js rename to backend/src/factories/tags.js diff --git a/backend/src/seed/factories/unverifiedEmailAddresses.js b/backend/src/factories/unverifiedEmailAddresses.js similarity index 100% rename from backend/src/seed/factories/unverifiedEmailAddresses.js rename to backend/src/factories/unverifiedEmailAddresses.js diff --git a/backend/src/seed/factories/users.js b/backend/src/factories/users.js similarity index 95% rename from backend/src/seed/factories/users.js rename to backend/src/factories/users.js index d56c42d0a..57f69b76b 100644 --- a/backend/src/seed/factories/users.js +++ b/backend/src/factories/users.js @@ -1,6 +1,6 @@ import faker from 'faker' import uuid from 'uuid/v4' -import encryptPassword from '../../helpers/encryptPassword' +import encryptPassword from '../helpers/encryptPassword' import slugify from 'slug' export default function create() { diff --git a/backend/src/jest/helpers.js b/backend/src/jest/helpers.js deleted file mode 100644 index 201d68c14..000000000 --- a/backend/src/jest/helpers.js +++ /dev/null @@ -1,5 +0,0 @@ -//* This is a fake ES2015 template string, just to benefit of syntax -// highlighting of `gql` template strings in certain editors. -export function gql(strings) { - return strings.join('') -} diff --git a/backend/src/jwt/decode.spec.js b/backend/src/jwt/decode.spec.js index 7aa703d97..71444a3e5 100644 --- a/backend/src/jwt/decode.spec.js +++ b/backend/src/jwt/decode.spec.js @@ -1,5 +1,5 @@ -import Factory from '../seed/factories/index' -import { getDriver, getNeode } from '../bootstrap/neo4j' +import Factory from '../factories/index' +import { getDriver, getNeode } from '../db/neo4j' import decode from './decode' const factory = Factory() diff --git a/backend/src/middleware/hashtags/hashtagsMiddleware.spec.js b/backend/src/middleware/hashtags/hashtagsMiddleware.spec.js index 0fa1e2dc5..2247e692d 100644 --- a/backend/src/middleware/hashtags/hashtagsMiddleware.spec.js +++ b/backend/src/middleware/hashtags/hashtagsMiddleware.spec.js @@ -1,7 +1,7 @@ import { gql } from '../../helpers/jest' -import Factory from '../../seed/factories' +import Factory from '../../factories' import { createTestClient } from 'apollo-server-testing' -import { getNeode, getDriver } from '../../bootstrap/neo4j' +import { getNeode, getDriver } from '../../db/neo4j' import createServer from '../../server' let server diff --git a/backend/src/middleware/notifications/notificationsMiddleware.spec.js b/backend/src/middleware/notifications/notificationsMiddleware.spec.js index c5f5990d3..136388b88 100644 --- a/backend/src/middleware/notifications/notificationsMiddleware.spec.js +++ b/backend/src/middleware/notifications/notificationsMiddleware.spec.js @@ -1,7 +1,7 @@ import { gql } from '../../helpers/jest' -import Factory from '../../seed/factories' +import Factory from '../../factories' import { createTestClient } from 'apollo-server-testing' -import { getNeode, getDriver } from '../../bootstrap/neo4j' +import { getNeode, getDriver } from '../../db/neo4j' import createServer from '../../server' let server, query, mutate, notifiedUser, authenticatedUser diff --git a/backend/src/middleware/orderByMiddleware.spec.js b/backend/src/middleware/orderByMiddleware.spec.js index 129f3a8b4..8d92a5b5d 100644 --- a/backend/src/middleware/orderByMiddleware.spec.js +++ b/backend/src/middleware/orderByMiddleware.spec.js @@ -1,6 +1,6 @@ import { gql } from '../helpers/jest' -import Factory from '../seed/factories' -import { getNeode, getDriver } from '../bootstrap/neo4j' +import Factory from '../factories' +import { getNeode, getDriver } from '../db/neo4j' import { createTestClient } from 'apollo-server-testing' import createServer from '../server' diff --git a/backend/src/middleware/permissionsMiddleware.js b/backend/src/middleware/permissionsMiddleware.js index a4c41871f..3e5bbd6e9 100644 --- a/backend/src/middleware/permissionsMiddleware.js +++ b/backend/src/middleware/permissionsMiddleware.js @@ -1,5 +1,5 @@ import { rule, shield, deny, allow, or } from 'graphql-shield' -import { getNeode } from '../bootstrap/neo4j' +import { getNeode } from '../db/neo4j' import CONFIG from '../config' const debug = !!CONFIG.DEBUG diff --git a/backend/src/middleware/permissionsMiddleware.spec.js b/backend/src/middleware/permissionsMiddleware.spec.js index 60aff961d..a4f13ea0c 100644 --- a/backend/src/middleware/permissionsMiddleware.spec.js +++ b/backend/src/middleware/permissionsMiddleware.spec.js @@ -1,8 +1,8 @@ import { createTestClient } from 'apollo-server-testing' import createServer from '../server' -import Factory from '../seed/factories' +import Factory from '../factories' import { gql } from '../helpers/jest' -import { getDriver, getNeode } from '../bootstrap/neo4j' +import { getDriver, getNeode } from '../db/neo4j' const factory = Factory() const instance = getNeode() diff --git a/backend/src/middleware/slugifyMiddleware.spec.js b/backend/src/middleware/slugifyMiddleware.spec.js index 1c2e59317..cf9f0941c 100644 --- a/backend/src/middleware/slugifyMiddleware.spec.js +++ b/backend/src/middleware/slugifyMiddleware.spec.js @@ -1,6 +1,6 @@ -import Factory from '../seed/factories' +import Factory from '../factories' import { gql } from '../helpers/jest' -import { getNeode, getDriver } from '../bootstrap/neo4j' +import { getNeode, getDriver } from '../db/neo4j' import createServer from '../server' import { createTestClient } from 'apollo-server-testing' diff --git a/backend/src/middleware/softDelete/softDeleteMiddleware.spec.js b/backend/src/middleware/softDelete/softDeleteMiddleware.spec.js index b7c16dfd3..6e1735af2 100644 --- a/backend/src/middleware/softDelete/softDeleteMiddleware.spec.js +++ b/backend/src/middleware/softDelete/softDeleteMiddleware.spec.js @@ -1,6 +1,6 @@ -import Factory from '../../seed/factories' +import Factory from '../../factories' import { gql } from '../../helpers/jest' -import { getNeode, getDriver } from '../../bootstrap/neo4j' +import { getNeode, getDriver } from '../../db/neo4j' import createServer from '../../server' import { createTestClient } from 'apollo-server-testing' diff --git a/backend/src/middleware/validation/validationMiddleware.spec.js b/backend/src/middleware/validation/validationMiddleware.spec.js index d093f939a..38cd010b4 100644 --- a/backend/src/middleware/validation/validationMiddleware.spec.js +++ b/backend/src/middleware/validation/validationMiddleware.spec.js @@ -1,6 +1,6 @@ import { gql } from '../../helpers/jest' -import Factory from '../../seed/factories' -import { getNeode, getDriver } from '../../bootstrap/neo4j' +import Factory from '../../factories' +import { getNeode, getDriver } from '../../db/neo4j' import { createTestClient } from 'apollo-server-testing' import createServer from '../../server' diff --git a/backend/src/models/User.spec.js b/backend/src/models/User.spec.js index 433cc5a6f..7bdde7014 100644 --- a/backend/src/models/User.spec.js +++ b/backend/src/models/User.spec.js @@ -1,5 +1,5 @@ -import Factory from '../seed/factories' -import { getNeode } from '../bootstrap/neo4j' +import Factory from '../factories' +import { getNeode } from '../db/neo4j' const factory = Factory() const neode = getNeode() diff --git a/backend/src/schema/resolvers/comments.spec.js b/backend/src/schema/resolvers/comments.spec.js index f96a60514..9877161db 100644 --- a/backend/src/schema/resolvers/comments.spec.js +++ b/backend/src/schema/resolvers/comments.spec.js @@ -1,8 +1,8 @@ -import Factory from '../../seed/factories' +import Factory from '../../factories' import { gql } from '../../helpers/jest' import { createTestClient } from 'apollo-server-testing' import createServer from '../../server' -import { getNeode, getDriver } from '../../bootstrap/neo4j' +import { getNeode, getDriver } from '../../db/neo4j' const driver = getDriver() const neode = getNeode() diff --git a/backend/src/schema/resolvers/donations.spec.js b/backend/src/schema/resolvers/donations.spec.js index d8dd5db06..c382eb475 100644 --- a/backend/src/schema/resolvers/donations.spec.js +++ b/backend/src/schema/resolvers/donations.spec.js @@ -1,7 +1,7 @@ import { createTestClient } from 'apollo-server-testing' -import Factory from '../../seed/factories' +import Factory from '../../factories' import { gql } from '../../helpers/jest' -import { getNeode, getDriver } from '../../bootstrap/neo4j' +import { getNeode, getDriver } from '../../db/neo4j' import createServer from '../../server' let mutate, query, authenticatedUser, variables diff --git a/backend/src/schema/resolvers/emails.spec.js b/backend/src/schema/resolvers/emails.spec.js index 82ce43337..97a1f0c29 100644 --- a/backend/src/schema/resolvers/emails.spec.js +++ b/backend/src/schema/resolvers/emails.spec.js @@ -1,6 +1,6 @@ -import Factory from '../../seed/factories' +import Factory from '../../factories' import { gql } from '../../helpers/jest' -import { getDriver, getNeode } from '../../bootstrap/neo4j' +import { getDriver, getNeode } from '../../db/neo4j' import createServer from '../../server' import { createTestClient } from 'apollo-server-testing' diff --git a/backend/src/schema/resolvers/follow.js b/backend/src/schema/resolvers/follow.js index 0416fe3d2..80cce8400 100644 --- a/backend/src/schema/resolvers/follow.js +++ b/backend/src/schema/resolvers/follow.js @@ -1,4 +1,4 @@ -import { getNeode } from '../../bootstrap/neo4j' +import { getNeode } from '../../db/neo4j' const neode = getNeode() diff --git a/backend/src/schema/resolvers/follow.spec.js b/backend/src/schema/resolvers/follow.spec.js index ff884666e..ad836a461 100644 --- a/backend/src/schema/resolvers/follow.spec.js +++ b/backend/src/schema/resolvers/follow.spec.js @@ -1,6 +1,6 @@ import { createTestClient } from 'apollo-server-testing' -import Factory from '../../seed/factories' -import { getDriver, getNeode } from '../../bootstrap/neo4j' +import Factory from '../../factories' +import { getDriver, getNeode } from '../../db/neo4j' import createServer from '../../server' import { gql } from '../../helpers/jest' diff --git a/backend/src/schema/resolvers/locations.spec.js b/backend/src/schema/resolvers/locations.spec.js index f4a846afd..aba11f9bc 100644 --- a/backend/src/schema/resolvers/locations.spec.js +++ b/backend/src/schema/resolvers/locations.spec.js @@ -1,6 +1,6 @@ -import Factory from '../../seed/factories' +import Factory from '../../factories' import { gql } from '../../helpers/jest' -import { getNeode, getDriver } from '../../bootstrap/neo4j' +import { getNeode, getDriver } from '../../db/neo4j' import createServer from '../../server' import { createTestClient } from 'apollo-server-testing' diff --git a/backend/src/schema/resolvers/moderation.spec.js b/backend/src/schema/resolvers/moderation.spec.js index f76cbdf46..cd502be75 100644 --- a/backend/src/schema/resolvers/moderation.spec.js +++ b/backend/src/schema/resolvers/moderation.spec.js @@ -1,7 +1,7 @@ import { createTestClient } from 'apollo-server-testing' -import Factory from '../../seed/factories' +import Factory from '../../factories' import { gql } from '../../helpers/jest' -import { getNeode, getDriver } from '../../bootstrap/neo4j' +import { getNeode, getDriver } from '../../db/neo4j' import createServer from '../../server' const factory = Factory() diff --git a/backend/src/schema/resolvers/notifications.spec.js b/backend/src/schema/resolvers/notifications.spec.js index 89bbd2528..a5c46e930 100644 --- a/backend/src/schema/resolvers/notifications.spec.js +++ b/backend/src/schema/resolvers/notifications.spec.js @@ -1,6 +1,6 @@ -import Factory from '../../seed/factories' +import Factory from '../../factories' import { gql } from '../../helpers/jest' -import { getDriver } from '../../bootstrap/neo4j' +import { getDriver } from '../../db/neo4j' import { createTestClient } from 'apollo-server-testing' import createServer from '../.././server' diff --git a/backend/src/schema/resolvers/passwordReset.spec.js b/backend/src/schema/resolvers/passwordReset.spec.js index be3c8c085..d7b3a0157 100644 --- a/backend/src/schema/resolvers/passwordReset.spec.js +++ b/backend/src/schema/resolvers/passwordReset.spec.js @@ -1,6 +1,6 @@ -import Factory from '../../seed/factories' +import Factory from '../../factories' import { gql } from '../../helpers/jest' -import { getNeode, getDriver } from '../../bootstrap/neo4j' +import { getNeode, getDriver } from '../../db/neo4j' import createPasswordReset from './helpers/createPasswordReset' import createServer from '../../server' import { createTestClient } from 'apollo-server-testing' diff --git a/backend/src/schema/resolvers/posts.spec.js b/backend/src/schema/resolvers/posts.spec.js index dcbd16d5d..71d1aa359 100644 --- a/backend/src/schema/resolvers/posts.spec.js +++ b/backend/src/schema/resolvers/posts.spec.js @@ -1,7 +1,7 @@ import { createTestClient } from 'apollo-server-testing' -import Factory from '../../seed/factories' +import Factory from '../../factories' import { gql } from '../../helpers/jest' -import { getNeode, getDriver } from '../../bootstrap/neo4j' +import { getNeode, getDriver } from '../../db/neo4j' import createServer from '../../server' const driver = getDriver() diff --git a/backend/src/schema/resolvers/registration.js b/backend/src/schema/resolvers/registration.js index e03f294cd..1e7708395 100644 --- a/backend/src/schema/resolvers/registration.js +++ b/backend/src/schema/resolvers/registration.js @@ -1,5 +1,5 @@ import { UserInputError } from 'apollo-server' -import { getNeode } from '../../bootstrap/neo4j' +import { getNeode } from '../../db/neo4j' import fileUpload from './fileUpload' import encryptPassword from '../../helpers/encryptPassword' import generateNonce from './helpers/generateNonce' diff --git a/backend/src/schema/resolvers/registration.spec.js b/backend/src/schema/resolvers/registration.spec.js index 8f3a7ac39..23b1f9d2a 100644 --- a/backend/src/schema/resolvers/registration.spec.js +++ b/backend/src/schema/resolvers/registration.spec.js @@ -1,6 +1,6 @@ -import Factory from '../../seed/factories' +import Factory from '../../factories' import { gql } from '../../helpers/jest' -import { getDriver, getNeode } from '../../bootstrap/neo4j' +import { getDriver, getNeode } from '../../db/neo4j' import createServer from '../../server' import { createTestClient } from 'apollo-server-testing' diff --git a/backend/src/schema/resolvers/reports.spec.js b/backend/src/schema/resolvers/reports.spec.js index 8b1bb925d..7f827b111 100644 --- a/backend/src/schema/resolvers/reports.spec.js +++ b/backend/src/schema/resolvers/reports.spec.js @@ -1,8 +1,8 @@ import { createTestClient } from 'apollo-server-testing' import createServer from '../.././server' -import Factory from '../../seed/factories' +import Factory from '../../factories' import { gql } from '../../helpers/jest' -import { getDriver, getNeode } from '../../bootstrap/neo4j' +import { getDriver, getNeode } from '../../db/neo4j' const factory = Factory() const instance = getNeode() diff --git a/backend/src/schema/resolvers/rewards.js b/backend/src/schema/resolvers/rewards.js index 44bdab770..311cfd2e6 100644 --- a/backend/src/schema/resolvers/rewards.js +++ b/backend/src/schema/resolvers/rewards.js @@ -1,4 +1,4 @@ -import { getNeode } from '../../bootstrap/neo4j' +import { getNeode } from '../../db/neo4j' import { UserInputError } from 'apollo-server' const neode = getNeode() diff --git a/backend/src/schema/resolvers/rewards.spec.js b/backend/src/schema/resolvers/rewards.spec.js index e6f67ecab..fe2807f25 100644 --- a/backend/src/schema/resolvers/rewards.spec.js +++ b/backend/src/schema/resolvers/rewards.spec.js @@ -1,7 +1,7 @@ import { createTestClient } from 'apollo-server-testing' -import Factory from '../../seed/factories' +import Factory from '../../factories' import { gql } from '../../helpers/jest' -import { getNeode, getDriver } from '../../bootstrap/neo4j' +import { getNeode, getDriver } from '../../db/neo4j' import createServer from '../../server' const factory = Factory() diff --git a/backend/src/schema/resolvers/shout.spec.js b/backend/src/schema/resolvers/shout.spec.js index e747946aa..104a28399 100644 --- a/backend/src/schema/resolvers/shout.spec.js +++ b/backend/src/schema/resolvers/shout.spec.js @@ -1,7 +1,7 @@ import { createTestClient } from 'apollo-server-testing' -import Factory from '../../seed/factories' +import Factory from '../../factories' import { gql } from '../../helpers/jest' -import { getNeode, getDriver } from '../../bootstrap/neo4j' +import { getNeode, getDriver } from '../../db/neo4j' import createServer from '../../server' let mutate, query, authenticatedUser, variables diff --git a/backend/src/schema/resolvers/socialMedia.js b/backend/src/schema/resolvers/socialMedia.js index c206778e5..c5b9dcd91 100644 --- a/backend/src/schema/resolvers/socialMedia.js +++ b/backend/src/schema/resolvers/socialMedia.js @@ -1,4 +1,4 @@ -import { getNeode } from '../../bootstrap/neo4j' +import { getNeode } from '../../db/neo4j' import Resolver from './helpers/Resolver' const neode = getNeode() diff --git a/backend/src/schema/resolvers/socialMedia.spec.js b/backend/src/schema/resolvers/socialMedia.spec.js index 8f6d91d43..f292b58a0 100644 --- a/backend/src/schema/resolvers/socialMedia.spec.js +++ b/backend/src/schema/resolvers/socialMedia.spec.js @@ -1,8 +1,8 @@ import { createTestClient } from 'apollo-server-testing' import createServer from '../../server' -import Factory from '../../seed/factories' +import Factory from '../../factories' import { gql } from '../../helpers/jest' -import { getNeode, getDriver } from '../../bootstrap/neo4j' +import { getNeode, getDriver } from '../../db/neo4j' const driver = getDriver() const factory = Factory() diff --git a/backend/src/schema/resolvers/statistics.spec.js b/backend/src/schema/resolvers/statistics.spec.js index 48baf00cd..e2b9dafe4 100644 --- a/backend/src/schema/resolvers/statistics.spec.js +++ b/backend/src/schema/resolvers/statistics.spec.js @@ -1,7 +1,7 @@ import { createTestClient } from 'apollo-server-testing' -import Factory from '../../seed/factories' +import Factory from '../../factories' import { gql } from '../../helpers/jest' -import { getNeode, getDriver } from '../../bootstrap/neo4j' +import { getNeode, getDriver } from '../../db/neo4j' import createServer from '../../server' let query, authenticatedUser diff --git a/backend/src/schema/resolvers/user_management.js b/backend/src/schema/resolvers/user_management.js index 4d035d9fa..4d40a6f63 100644 --- a/backend/src/schema/resolvers/user_management.js +++ b/backend/src/schema/resolvers/user_management.js @@ -1,7 +1,7 @@ import encode from '../../jwt/encode' import bcrypt from 'bcryptjs' import { AuthenticationError } from 'apollo-server' -import { getNeode } from '../../bootstrap/neo4j' +import { getNeode } from '../../db/neo4j' import normalizeEmail from './helpers/normalizeEmail' import log from './helpers/databaseLogger' diff --git a/backend/src/schema/resolvers/user_management.spec.js b/backend/src/schema/resolvers/user_management.spec.js index 3527e5dc2..5e7043e74 100644 --- a/backend/src/schema/resolvers/user_management.spec.js +++ b/backend/src/schema/resolvers/user_management.spec.js @@ -1,11 +1,11 @@ import jwt from 'jsonwebtoken' import CONFIG from './../../config' -import Factory from '../../seed/factories' +import Factory from '../../factories' import { gql } from '../../helpers/jest' import { createTestClient } from 'apollo-server-testing' import createServer, { context } from '../../server' import encode from '../../jwt/encode' -import { getNeode } from '../../bootstrap/neo4j' +import { getNeode } from '../../db/neo4j' const factory = Factory() const neode = getNeode() diff --git a/backend/src/schema/resolvers/users.js b/backend/src/schema/resolvers/users.js index 0b3f13631..6183511f1 100644 --- a/backend/src/schema/resolvers/users.js +++ b/backend/src/schema/resolvers/users.js @@ -1,6 +1,6 @@ import { neo4jgraphql } from 'neo4j-graphql-js' import fileUpload from './fileUpload' -import { getNeode } from '../../bootstrap/neo4j' +import { getNeode } from '../../db/neo4j' import { UserInputError, ForbiddenError } from 'apollo-server' import Resolver from './helpers/Resolver' import log from './helpers/databaseLogger' diff --git a/backend/src/schema/resolvers/users.spec.js b/backend/src/schema/resolvers/users.spec.js index 5d1ebd8e2..cfd84fcf7 100644 --- a/backend/src/schema/resolvers/users.spec.js +++ b/backend/src/schema/resolvers/users.spec.js @@ -1,6 +1,6 @@ -import Factory from '../../seed/factories' +import Factory from '../../factories' import { gql } from '../../helpers/jest' -import { getNeode, getDriver } from '../../bootstrap/neo4j' +import { getNeode, getDriver } from '../../db/neo4j' import createServer from '../../server' import { createTestClient } from 'apollo-server-testing' diff --git a/backend/src/schema/resolvers/users/blockedUsers.spec.js b/backend/src/schema/resolvers/users/blockedUsers.spec.js index 11bcb823d..bec8b59d0 100644 --- a/backend/src/schema/resolvers/users/blockedUsers.spec.js +++ b/backend/src/schema/resolvers/users/blockedUsers.spec.js @@ -1,8 +1,8 @@ import { createTestClient } from 'apollo-server-testing' import createServer from '../../../server' -import Factory from '../../../seed/factories' +import Factory from '../../../factories' import { gql } from '../../../helpers/jest' -import { getNeode, getDriver } from '../../../bootstrap/neo4j' +import { getNeode, getDriver } from '../../../db/neo4j' const driver = getDriver() const factory = Factory() diff --git a/backend/src/schema/resolvers/users/location.spec.js b/backend/src/schema/resolvers/users/location.spec.js index 59d093afb..f7315174c 100644 --- a/backend/src/schema/resolvers/users/location.spec.js +++ b/backend/src/schema/resolvers/users/location.spec.js @@ -1,6 +1,6 @@ import { gql } from '../../../helpers/jest' -import Factory from '../../../seed/factories' -import { getNeode, getDriver } from '../../../bootstrap/neo4j' +import Factory from '../../../factories' +import { getNeode, getDriver } from '../../../db/neo4j' import { createTestClient } from 'apollo-server-testing' import createServer from '../../../server' diff --git a/backend/src/seed/seed-helpers.js b/backend/src/seed/seed-helpers.js deleted file mode 100644 index 913ca1d54..000000000 --- a/backend/src/seed/seed-helpers.js +++ /dev/null @@ -1,134 +0,0 @@ -const _ = require('lodash') -const faker = require('faker') -const unsplashTopics = [ - 'love', - 'family', - 'spring', - 'business', - 'nature', - 'travel', - 'happy', - 'landscape', - 'health', - 'friends', - 'computer', - 'autumn', - 'space', - 'animal', - 'smile', - 'face', - 'people', - 'portrait', - 'amazing', -] -let unsplashTopicsTmp = [] - -const ngoLogos = [ - 'http://www.fetchlogos.com/wp-content/uploads/2015/11/Girl-Scouts-Of-The-Usa-Logo.jpg', - 'http://logos.textgiraffe.com/logos/logo-name/Ngo-designstyle-friday-m.png', - 'http://seeklogo.com/images/N/ngo-logo-BD53A3E024-seeklogo.com.png', - 'https://dcassetcdn.com/design_img/10133/25833/25833_303600_10133_image.jpg', - 'https://cdn.tutsplus.com/vector/uploads/legacy/articles/08bad_ngologos/20.jpg', - 'https://cdn.tutsplus.com/vector/uploads/legacy/articles/08bad_ngologos/33.jpg', - null, -] - -const difficulties = ['easy', 'medium', 'hard'] - -export default { - randomItem: (items, filter) => { - const ids = filter - ? Object.keys(items).filter(id => { - return filter(items[id]) - }) - : _.keys(items) - const randomIds = _.shuffle(ids) - return items[randomIds.pop()] - }, - randomItems: (items, key = 'id', min = 1, max = 1) => { - const randomIds = _.shuffle(_.keys(items)) - const res = [] - - const count = _.random(min, max) - - for (let i = 0; i < count; i++) { - let r = items[randomIds.pop()][key] - if (key === 'id') { - r = r.toString() - } - res.push(r) - } - return res - }, - random: items => { - return _.shuffle(items).pop() - }, - randomDifficulty: () => { - return _.shuffle(difficulties).pop() - }, - randomLogo: () => { - return _.shuffle(ngoLogos).pop() - }, - randomUnsplashUrl: () => { - if (Math.random() < 0.6) { - // do not attach images in 60 percent of the cases (faster seeding) - return - } - if (unsplashTopicsTmp.length < 2) { - unsplashTopicsTmp = _.shuffle(unsplashTopics) - } - return ( - 'https://source.unsplash.com/daily?' + unsplashTopicsTmp.pop() + ',' + unsplashTopicsTmp.pop() - ) - }, - randomCategories: (seederstore, allowEmpty = false) => { - let count = Math.round(Math.random() * 3) - if (allowEmpty === false && count === 0) { - count = 1 - } - const categorieIds = _.shuffle(_.keys(seederstore.categories)) - const ids = [] - for (let i = 0; i < count; i++) { - ids.push(categorieIds.pop()) - } - return ids - }, - randomAddresses: () => { - const count = Math.round(Math.random() * 3) - const addresses = [] - for (let i = 0; i < count; i++) { - addresses.push({ - city: faker.address.city(), - zipCode: faker.address.zipCode(), - street: faker.address.streetAddress(), - country: faker.address.countryCode(), - lat: 54.032726 - Math.random() * 10, - lng: 6.558838 + Math.random() * 10, - }) - } - return addresses - }, - /** - * Get array of ids from the given seederstore items after mapping them by the key in the values - * - * @param items items from the seederstore - * @param values values for which you need the ids - * @param key the field key that is represented in the values (slug, name, etc.) - */ - mapIdsByKey: (items, values, key) => { - const res = [] - values.forEach(value => { - res.push(_.find(items, [key, value]).id.toString()) - }) - return res - }, - genInviteCode: () => { - const chars = '23456789abcdefghkmnpqrstuvwxyzABCDEFGHJKLMNPRSTUVWXYZ' - let code = '' - for (let i = 0; i < 8; i++) { - const n = _.random(0, chars.length - 1) - code += chars.substr(n, 1) - } - return code - }, -} diff --git a/backend/src/server.js b/backend/src/server.js index bd9973a39..02e166b71 100644 --- a/backend/src/server.js +++ b/backend/src/server.js @@ -3,7 +3,7 @@ import helmet from 'helmet' import { ApolloServer } from 'apollo-server-express' import CONFIG from './config' import middleware from './middleware' -import { getNeode, getDriver } from './bootstrap/neo4j' +import { getNeode, getDriver } from './db/neo4j' import decode from './jwt/decode' import schema from './schema' import webfinger from './activitypub/routes/webfinger' diff --git a/backend/test/features/support/steps.js b/backend/test/features/support/steps.js index 73d059348..70802f4e2 100644 --- a/backend/test/features/support/steps.js +++ b/backend/test/features/support/steps.js @@ -3,7 +3,7 @@ import { Given, When, Then, AfterAll } from 'cucumber' import { expect } from 'chai' // import { client } from '../../../src/activitypub/apollo-client' import { GraphQLClient } from 'graphql-request' -import Factory from '../../../src/seed/factories' +import Factory from '../../../src/factories' const debug = require('debug')('ea:test:steps') const factory = Factory() diff --git a/cypress/support/factories.js b/cypress/support/factories.js index e0b6210d8..1b76a1a01 100644 --- a/cypress/support/factories.js +++ b/cypress/support/factories.js @@ -1,5 +1,5 @@ -import Factory from '../../backend/src/seed/factories' -import { getDriver, getNeode } from '../../backend/src/bootstrap/neo4j' +import Factory from '../../backend/src/factories' +import { getDriver, getNeode } from '../../backend/src/db/neo4j' const neo4jConfigs = { uri: Cypress.env('NEO4J_URI'), diff --git a/features/support/steps.js b/features/support/steps.js index 923dc9766..71f493834 100644 --- a/features/support/steps.js +++ b/features/support/steps.js @@ -1,6 +1,6 @@ // features/support/steps.js import { Given, When, Then, After, AfterAll } from 'cucumber' -import Factory from '../../backend/src/seed/factories' +import Factory from '../../backend/src/factories' import dotenv from 'dotenv' import expect from 'expect' From ce664040c60dfb0fa4d84cb457c3be74ddcf58b5 Mon Sep 17 00:00:00 2001 From: roschaefer Date: Sun, 19 Jan 2020 17:51:27 +0100 Subject: [PATCH 05/12] docs(backend): How to create & run data migrations --- backend/README.md | 32 +++++++++++ neo4j/README.md | 14 ----- ...ge_disabled_relationship_to_report_node.sh | 55 ------------------- .../db_manipulation/add_image_aspect_ratio.sh | 22 -------- ...ge_disabled_relationship_to_report_node.sh | 51 ----------------- 5 files changed, 32 insertions(+), 142 deletions(-) delete mode 100755 neo4j/change_disabled_relationship_to_report_node.sh delete mode 100755 neo4j/db_manipulation/add_image_aspect_ratio.sh delete mode 100755 neo4j/db_manipulation/change_disabled_relationship_to_report_node.sh diff --git a/backend/README.md b/backend/README.md index c42c5ac85..7fd49faf8 100644 --- a/backend/README.md +++ b/backend/README.md @@ -111,6 +111,38 @@ $ yarn run db:reset {% endtab %} {% endtabs %} +### Data migrations + +Although Neo4J is schema-less,you might find yourself in a situation in which +you have to migrate your data e.g. because your data modeling has changed. + +{% tabs %} +{% tab title="Docker" %} +Generate a data migration file: +```bash +$ docker-compose exec backend yarn run db:migrate:create your_data_migration +# Edit the file in ./src/db/migrations/ +``` + +To run the migration: +```bash +$ docker-compose exec backend yarn run db:migrate up +``` +{% endtab %} +{% tab title="Without Docker" %} +Generate a data migration file: +```bash +$ yarn run db:migrate:create your_data_migration +# Edit the file in ./src/db/migrations/ +``` + +To run the migration: +```bash +$ yarn run db:migrate up +``` +{% endtab %} +{% endtabs %} + # Testing **Beware**: We have no multiple database setup at the moment. We clean the diff --git a/neo4j/README.md b/neo4j/README.md index 5df01cc71..a4242b512 100644 --- a/neo4j/README.md +++ b/neo4j/README.md @@ -36,20 +36,6 @@ Then make sure to allow Apoc procedures by adding the following line to your Neo ``` dbms.security.procedures.unrestricted=apoc.* ``` -### Database Indices and Constraints - -If you have `cypher-shell` available with your local installation of neo4j you -can run: - -```bash -# in folder neo4j/ -$ cp .env.template .env -$ ./db_setup.sh -``` - -Otherwise, if you don't have `cypher-shell` available, copy the cypher -statements [from the `db_setup.sh` script](https://github.com/Human-Connection/Human-Connection/blob/master/neo4j/db_setup.sh) and paste the scripts into your -[database browser frontend](http://localhost:7474). ### Alternatives diff --git a/neo4j/change_disabled_relationship_to_report_node.sh b/neo4j/change_disabled_relationship_to_report_node.sh deleted file mode 100755 index 2f44b8e59..000000000 --- a/neo4j/change_disabled_relationship_to_report_node.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -ENV_FILE=$(dirname "$0")/.env -[[ -f "$ENV_FILE" ]] && source "$ENV_FILE" - -if [ -z "$NEO4J_USERNAME" ] || [ -z "$NEO4J_PASSWORD" ]; then - echo "Please set NEO4J_USERNAME and NEO4J_PASSWORD environment variables." - echo "Database manipulation is not possible without connecting to the database." - echo "E.g. you could \`cp .env.template .env\` unless you run the script in a docker container" -fi - -until echo 'RETURN "Connection successful" as info;' | cypher-shell -do - echo "Connecting to neo4j failed, trying again..." - sleep 1 -done - -echo " -// convert old DISABLED to new REVIEWED-Report-BELONGS_TO structure -MATCH (moderator:User)-[disabled:DISABLED]->(disabledResource) -WHERE disabledResource:User OR disabledResource:Comment OR disabledResource:Post -DELETE disabled -CREATE (moderator)-[review:REVIEWED]->(report:Report)-[:BELONGS_TO]->(disabledResource) -SET review.createdAt = toString(datetime()), review.updatedAt = review.createdAt, review.disable = true -SET report.id = randomUUID(), report.createdAt = toString(datetime()), report.updatedAt = report.createdAt, report.rule = 'latestReviewUpdatedAtRules', report.closed = false - -// if disabledResource has no filed report, then create a moderators default filed report -WITH moderator, disabledResource, report -OPTIONAL MATCH (disabledResourceReporter:User)-[existingFiledReport:FILED]->(disabledResource) -FOREACH(disabledResource IN CASE WHEN existingFiledReport IS NULL THEN [1] ELSE [] END | - CREATE (moderator)-[addModeratorReport:FILED]->(report) - SET addModeratorReport.createdAt = toString(datetime()), addModeratorReport.reasonCategory = 'other', addModeratorReport.reasonDescription = 'Old DISABLED relations didn't enforce mandatory reporting !!! Created automatically to ensure database consistency! Creation date is when the database manipulation happened.' -) -FOREACH(disabledResource IN CASE WHEN existingFiledReport IS NOT NULL THEN [1] ELSE [] END | - CREATE (disabledResourceReporter)-[moveModeratorReport:FILED]->(report) - SET moveModeratorReport = existingFiledReport - DELETE existingFiledReport -) - -RETURN disabledResource {.id}; -" | cypher-shell - -echo " -// for FILED resources without DISABLED relation which are handled above, create new FILED-Report-BELONGS_TO structure -MATCH (reporter:User)-[oldReport:REPORTED]->(notDisabledResource) -WHERE notDisabledResource:User OR notDisabledResource:Comment OR notDisabledResource:Post -MERGE (report:Report)-[:BELONGS_TO]->(notDisabledResource) -ON CREATE SET report.id = randomUUID(), report.createdAt = toString(datetime()), report.updatedAt = report.createdAt, report.rule = 'latestReviewUpdatedAtRules', report.closed = false -CREATE (reporter)-[filed:FILED]->(report) -SET report = oldReport -DELETE oldReport - -RETURN notDisabledResource {.id}; -" | cypher-shell - diff --git a/neo4j/db_manipulation/add_image_aspect_ratio.sh b/neo4j/db_manipulation/add_image_aspect_ratio.sh deleted file mode 100755 index 8e2a16a01..000000000 --- a/neo4j/db_manipulation/add_image_aspect_ratio.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -if [ -z "$NEO4J_USERNAME" ] || [ -z "$NEO4J_PASSWORD" ]; then - echo "Please set NEO4J_USERNAME and NEO4J_PASSWORD environment variables." - echo "Database manipulation is not possible without connecting to the database." - echo "E.g. you could \`cp .env.template .env\` unless you run the script in a docker container" -fi - -until echo 'RETURN "Connection successful" as info;' | cypher-shell -do - echo "Connecting to neo4j failed, trying again..." - sleep 1 -done - -echo " - CALL apoc.periodic.iterate(' - CALL apoc.load.csv("out.csv") yield map as row return row - ',' - MATCH (post:Post) where post.image = row.image - set post.imageAspectRatio = row.aspectRatio - ', {batchSize:10000, iterateList:true, parallel:true}); -" | cypher-shell diff --git a/neo4j/db_manipulation/change_disabled_relationship_to_report_node.sh b/neo4j/db_manipulation/change_disabled_relationship_to_report_node.sh deleted file mode 100755 index e611382f0..000000000 --- a/neo4j/db_manipulation/change_disabled_relationship_to_report_node.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -ENV_FILE=$(dirname "$0")/.env -[[ -f "$ENV_FILE" ]] && source "$ENV_FILE" - -if [ -z "$NEO4J_USERNAME" ] || [ -z "$NEO4J_PASSWORD" ]; then - echo "Please set NEO4J_USERNAME and NEO4J_PASSWORD environment variables." - echo "Database manipulation is not possible without connecting to the database." - echo "E.g. you could \`cp .env.template .env\` unless you run the script in a docker container" -fi - -until echo 'RETURN "Connection successful" as info;' | cypher-shell -do - echo "Connecting to neo4j failed, trying again..." - sleep 1 -done - -echo " - :begin - MATCH(user)-[reported:REPORTED]->(resource) - WITH reported, resource, COLLECT(user) as users - MERGE(report:Report)-[:BELONGS_TO]->(resource) - SET report.id = randomUUID(), report.createdAt = toString(datetime()), report.updatedAt = report.createdAt, report.rule = 'latestReviewUpdatedAtRules', report.closed = false - WITH report, users, reported - UNWIND users as user - MERGE (user)-[filed:FILED]->(report) - SET filed = reported - DELETE reported; - - MATCH(moderator)-[disabled:DISABLED]->(resource) - MATCH(report:Report)-[:BELONGS_TO]->(resource) - WITH disabled, resource, COLLECT(moderator) as moderators, report - DELETE disabled - WITH report, moderators, disabled - UNWIND moderators as moderator - MERGE (moderator)-[review:REVIEWED {disable: true}]->(report) - SET review.createdAt = toString(datetime()), review.updatedAt = review.createdAt, review.disable = true; - - MATCH(moderator)-[disabled:DISABLED]->(resource) - WITH disabled, resource, COLLECT(moderator) as moderators - MERGE(report:Report)-[:BELONGS_TO]->(resource) - SET report.id = randomUUID(), report.createdAt = toString(datetime()), report.updatedAt = report.createdAt, report.rule = 'latestReviewUpdatedAtRules', report.closed = false - DELETE disabled - WITH report, moderators, disabled - UNWIND moderators as moderator - MERGE(moderator)-[filed:FILED]->(report) - SET filed.createdAt = toString(datetime()), filed.reasonCategory = 'other', filed.reasonDescription = 'Old DISABLED relations didn\'t enforce mandatory reporting !!! Created automatically to ensure database consistency! Creation date is when the database manipulation happened.' - MERGE (moderator)-[review:REVIEWED {disable: true}]->(report) - SET review.createdAt = toString(datetime()), review.updatedAt = review.createdAt, review.disable = true; - :commit -" | cypher-shell \ No newline at end of file From 98a4521ecc36cf44deebb2ba02fbbdf96d53b8c2 Mon Sep 17 00:00:00 2001 From: roschaefer Date: Mon, 20 Jan 2020 11:28:36 +0100 Subject: [PATCH 06/12] Add back missing search constraint --- backend/src/db/migrate/store.js | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/backend/src/db/migrate/store.js b/backend/src/db/migrate/store.js index 9984b3971..b2d65a0f2 100644 --- a/backend/src/db/migrate/store.js +++ b/backend/src/db/migrate/store.js @@ -3,10 +3,27 @@ import { getDriver, getNeode } from '../../db/neo4j' class Store { async init(fn) { const neode = getNeode() - await getNeode().schema.install() + const { driver } = neode + const session = driver.session() // eslint-disable-next-line no-console - console.log('Successfully created database indices and constraints!') - neode.driver.close() + const writeTxResultPromise = session.writeTransaction(async txc => { + await txc.run('CALL apoc.schema.assert({},{},true)') // drop all indices + return Promise.all([ + 'CALL db.index.fulltext.createNodeIndex("post_fulltext_search",["Post"],["title", "content"])', + 'CALL db.index.fulltext.createNodeIndex("user_fulltext_search",["User"],["name", "slug"])' + ].map(statement => txc.run(statement))) + }) + try { + await writeTxResultPromise + await getNeode().schema.install() + console.log('Successfully created database indices and constraints!') + } catch (error) { + console.log(error) // eslint-disable-line no-console + } finally { + session.close() + driver.close() + fn() + } } async load(fn) { From a86b26a756d47629c73adfb0e142f0d3b476dc35 Mon Sep 17 00:00:00 2001 From: roschaefer Date: Mon, 20 Jan 2020 15:22:51 +0100 Subject: [PATCH 07/12] Various fixes for data migrations * Add unique index for `Migration`s * Fix proper use of `next` callback. First argument is potential error. * Update migration template --- backend/package.json | 8 +++--- backend/src/db/migrate/store.js | 28 +++++++++++-------- backend/src/db/migrate/template.js | 28 +++++++++++++++++-- ...387929122-merge_duplicate_user_accounts.js | 6 ++-- backend/src/models/Migration.js | 5 ++++ backend/src/models/index.js | 1 + 6 files changed, 56 insertions(+), 20 deletions(-) create mode 100644 backend/src/models/Migration.js diff --git a/backend/package.json b/backend/package.json index 3c9fda287..b87091a1b 100644 --- a/backend/package.json +++ b/backend/package.json @@ -4,8 +4,10 @@ "description": "GraphQL Backend for Human Connection", "main": "src/index.js", "scripts": { - "build": "babel src/ -d dist/ --copy-files", + "__migrate": "migrate --compiler 'js:@babel/register' --migrations-dir ./src/db/migrations", + "prod:migrate": "migrate --migrations-dir ./dist/db/migrations --store ./dist/db/migrate/store.js", "start": "node dist/", + "build": "babel src/ -d dist/ --copy-files", "dev": "nodemon --exec babel-node src/ -e js,gql", "dev:debug": "nodemon --exec babel-node --inspect=0.0.0.0:9229 src/ -e js,gql", "lint": "eslint src --config .eslintrc.js", @@ -13,10 +15,8 @@ "db:clean": "babel-node src/db/clean.js", "db:reset": "yarn run db:clean", "db:seed": "babel-node src/db/seed.js", - "__migrate": "migrate --compiler 'js:@babel/register' --migrations-dir ./src/db/migrations", "db:migrate": "yarn run __migrate --store ./src/db/migrate/store.js", - "db:migrate:create": "yarn run __migrate --template-file ./src/db/migrate/template.js create", - "production:db:migrate": "migrate --migrations-dir ./dist/db/migrations --store ./dist/db/migrate/store.js" + "db:migrate:create": "yarn run __migrate --template-file ./src/db/migrate/template.js create" }, "author": "Human Connection gGmbH", "license": "MIT", diff --git a/backend/src/db/migrate/store.js b/backend/src/db/migrate/store.js index b2d65a0f2..97b057dac 100644 --- a/backend/src/db/migrate/store.js +++ b/backend/src/db/migrate/store.js @@ -1,32 +1,36 @@ import { getDriver, getNeode } from '../../db/neo4j' class Store { - async init(fn) { + async init(next) { const neode = getNeode() const { driver } = neode const session = driver.session() // eslint-disable-next-line no-console const writeTxResultPromise = session.writeTransaction(async txc => { await txc.run('CALL apoc.schema.assert({},{},true)') // drop all indices - return Promise.all([ - 'CALL db.index.fulltext.createNodeIndex("post_fulltext_search",["Post"],["title", "content"])', - 'CALL db.index.fulltext.createNodeIndex("user_fulltext_search",["User"],["name", "slug"])' - ].map(statement => txc.run(statement))) + return Promise.all( + [ + 'CALL db.index.fulltext.createNodeIndex("post_fulltext_search",["Post"],["title", "content"])', + 'CALL db.index.fulltext.createNodeIndex("user_fulltext_search",["User"],["name", "slug"])', + ].map(statement => txc.run(statement)), + ) }) try { await writeTxResultPromise await getNeode().schema.install() + // eslint-disable-next-line no-console console.log('Successfully created database indices and constraints!') + next() } catch (error) { console.log(error) // eslint-disable-line no-console + next(error, null) } finally { session.close() driver.close() - fn() } } - async load(fn) { + async load(next) { const driver = getDriver() const session = driver.session() const readTxResultPromise = session.readTransaction(async txc => { @@ -42,18 +46,19 @@ class Store { console.log( "No migrations found in database. If it's the first time you run migrations, then this is normal.", ) - return fn(null, {}) + return next(null, {}) } const [{ title: lastRun }] = migrations - fn(null, { lastRun, migrations }) + next(null, { lastRun, migrations }) } catch (error) { console.log(error) // eslint-disable-line no-console + next(error) } finally { session.close() } } - async save(set, fn) { + async save(set, next) { const driver = getDriver() const session = driver.session() const { migrations } = set @@ -70,11 +75,12 @@ class Store { }) try { await writeTxResultPromise + next() } catch (error) { console.log(error) // eslint-disable-line no-console + next(error) } finally { session.close() - fn() } } } diff --git a/backend/src/db/migrate/template.js b/backend/src/db/migrate/template.js index 941f2a9e3..b8511e9bb 100644 --- a/backend/src/db/migrate/template.js +++ b/backend/src/db/migrate/template.js @@ -1,7 +1,31 @@ +import { getDriver } from '../../db/neo4j' + +export const description = '' + export function up(next) { - next() + const driver = getDriver() + const session = driver.session() + try { + // Implement your migration here. + next() + } catch (err) { + next(err) + } finally { + session.close() + driver.close() + } } export function down(next) { - next() + const driver = getDriver() + const session = driver.session() + try { + // Rollback your migration here. + next() + } catch (err) { + next(err) + } finally { + session.close() + driver.close() + } } diff --git a/backend/src/db/migrations/1579387929122-merge_duplicate_user_accounts.js b/backend/src/db/migrations/1579387929122-merge_duplicate_user_accounts.js index 9dbc8ad05..ec38befc5 100644 --- a/backend/src/db/migrations/1579387929122-merge_duplicate_user_accounts.js +++ b/backend/src/db/migrations/1579387929122-merge_duplicate_user_accounts.js @@ -74,11 +74,11 @@ export function up(next) { next() }, error: error => { - throw new Error(error) + next(new Error(error), null) }, }) } -export function down() { - throw new Error('Irreversible migration') +export function down(next) { + next(new Error('Irreversible migration')) } diff --git a/backend/src/models/Migration.js b/backend/src/models/Migration.js new file mode 100644 index 000000000..e36d10ac3 --- /dev/null +++ b/backend/src/models/Migration.js @@ -0,0 +1,5 @@ +export default { + title: { type: 'string', primary: true, token: true }, + description: { type: 'string' }, + timestamp: { type: 'number', unique: true }, +} diff --git a/backend/src/models/index.js b/backend/src/models/index.js index 047ace67c..dbb6a927e 100644 --- a/backend/src/models/index.js +++ b/backend/src/models/index.js @@ -13,4 +13,5 @@ export default { Location: require('./Location.js').default, Donations: require('./Donations.js').default, Report: require('./Report.js').default, + Migration: require('./Migration.js').default, } From 561889c53007dad8ffb91430e6cd9b18af251d46 Mon Sep 17 00:00:00 2001 From: mattwr18 Date: Wed, 22 Jan 2020 13:19:38 +0100 Subject: [PATCH 08/12] Add migration to merge duplicate Locations - having duplicate Location nodes in the production database blocks us from adding a unique constraint, so that Locations are not created which have the same id. --- backend/src/db/migrate/store.js | 21 +++-- ...87929111-merge_duplicate_location_nodes.js | 77 +++++++++++++++++++ 2 files changed, 91 insertions(+), 7 deletions(-) create mode 100644 backend/src/db/migrations/1579387929111-merge_duplicate_location_nodes.js diff --git a/backend/src/db/migrate/store.js b/backend/src/db/migrate/store.js index 97b057dac..81df47708 100644 --- a/backend/src/db/migrate/store.js +++ b/backend/src/db/migrate/store.js @@ -35,7 +35,7 @@ class Store { const session = driver.session() const readTxResultPromise = session.readTransaction(async txc => { const result = await txc.run( - 'MATCH (migration:Migration) RETURN migration {.*} ORDER BY migration.timestamp DESC', + 'MATCH (migration:Migration) RETURN migration {.*} ORDER BY migration.createdAt DESC', ) return result.records.map(r => r.get('migration')) }) @@ -64,12 +64,19 @@ class Store { const { migrations } = set const writeTxResultPromise = session.writeTransaction(txc => { return Promise.all( - migrations.map(migration => { - const { title, description, timestamp } = migration - const properties = { title, description, timestamp } - return txc.run('CREATE (migration:Migration) SET migration += $properties', { - properties, - }) + migrations.map(async migration => { + const { title, description } = migration + const properties = { title, description } + const migrationResult = await txc.run( + ` + MERGE (migration:Migration { title: $properties.title }) + ON CREATE SET migration += $properties, + migration.createdAt = toString(datetime()) + RETURN migration + `, + { properties }, + ) + return migrationResult }), ) }) diff --git a/backend/src/db/migrations/1579387929111-merge_duplicate_location_nodes.js b/backend/src/db/migrations/1579387929111-merge_duplicate_location_nodes.js new file mode 100644 index 000000000..390bfd935 --- /dev/null +++ b/backend/src/db/migrations/1579387929111-merge_duplicate_location_nodes.js @@ -0,0 +1,77 @@ +import { throwError, concat } from 'rxjs' +import { flatMap, mergeMap, map, catchError } from 'rxjs/operators' +import { getDriver } from '../neo4j' + +export const description = ` + This migration merges duplicate :Location nodes. It became + necessary after we realized that we had not set up constraints for Location.id in production. +` +export function up(next) { + const driver = getDriver() + const rxSession = driver.rxSession() + rxSession + .beginTransaction() + .pipe( + flatMap(transaction => + concat( + transaction + .run( + ` + MATCH (location:Location) + RETURN location {.id} + `, + ) + .records() + .pipe( + map(record => { + const { id: locationIds } = record.get('location') + return { locationIds } + }), + mergeMap(({ locationIds }) => { + return transaction + .run( + ` + MATCH(location:Location {id: $locationIds}), (location2:Location {id: $locationIds}) + WHERE location.id = location2.id AND id(location) < id(location2) + CALL apoc.refactor.mergeNodes([location, location2], { properties: 'combine', mergeRels: true }) YIELD node as updatedLocation + RETURN location {.*},updatedLocation {.*} + `, + { locationIds }, + ) + .records() + .pipe( + map(record => ({ + location: record.get('location'), + updatedLocation: record.get('updatedLocation'), + })), + ) + }), + ), + transaction.commit(), + ).pipe(catchError(error => transaction.rollback().pipe(throwError(error)))), + ), + ) + .subscribe({ + next: ({ updatedLocation, location }) => + // eslint-disable-next-line no-console + console.log(` + Merged: + ============================= + locationId: ${location.id} + updatedLocation: ${location.id} => ${updatedLocation.id} + ============================= + `), + complete: () => { + // eslint-disable-next-line no-console + console.log('Merging of duplicate locations completed') + next() + }, + error: error => { + next(new Error(error), null) + }, + }) +} + +export function down(next) { + next(new Error('Irreversible migration')) +} From c6bfca312c4ac9eb10f230625743215295b44be6 Mon Sep 17 00:00:00 2001 From: mattwr18 Date: Wed, 22 Jan 2020 13:31:48 +0100 Subject: [PATCH 09/12] Favor createdAt over timestamp - it's more human readable and consistent with all other nodes in production --- backend/src/models/Migration.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/models/Migration.js b/backend/src/models/Migration.js index e36d10ac3..1b697c8ad 100644 --- a/backend/src/models/Migration.js +++ b/backend/src/models/Migration.js @@ -1,5 +1,5 @@ export default { title: { type: 'string', primary: true, token: true }, description: { type: 'string' }, - timestamp: { type: 'number', unique: true }, + createdAt: { type: 'string', isoDate: true, default: () => new Date().toISOString() }, } From f3784c5f5009006b73a8e6ab50088d670412b05b Mon Sep 17 00:00:00 2001 From: mattwr18 Date: Thu, 23 Jan 2020 15:04:38 +0100 Subject: [PATCH 10/12] Create .archived directory for older migrations --- neo4j/.archived/add_image_aspect_ratio.sh | 22 ++++++++ ...ge_disabled_relationship_to_report_node.sh | 51 +++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 neo4j/.archived/add_image_aspect_ratio.sh create mode 100644 neo4j/.archived/change_disabled_relationship_to_report_node.sh diff --git a/neo4j/.archived/add_image_aspect_ratio.sh b/neo4j/.archived/add_image_aspect_ratio.sh new file mode 100644 index 000000000..8e2a16a01 --- /dev/null +++ b/neo4j/.archived/add_image_aspect_ratio.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +if [ -z "$NEO4J_USERNAME" ] || [ -z "$NEO4J_PASSWORD" ]; then + echo "Please set NEO4J_USERNAME and NEO4J_PASSWORD environment variables." + echo "Database manipulation is not possible without connecting to the database." + echo "E.g. you could \`cp .env.template .env\` unless you run the script in a docker container" +fi + +until echo 'RETURN "Connection successful" as info;' | cypher-shell +do + echo "Connecting to neo4j failed, trying again..." + sleep 1 +done + +echo " + CALL apoc.periodic.iterate(' + CALL apoc.load.csv("out.csv") yield map as row return row + ',' + MATCH (post:Post) where post.image = row.image + set post.imageAspectRatio = row.aspectRatio + ', {batchSize:10000, iterateList:true, parallel:true}); +" | cypher-shell diff --git a/neo4j/.archived/change_disabled_relationship_to_report_node.sh b/neo4j/.archived/change_disabled_relationship_to_report_node.sh new file mode 100644 index 000000000..3227ec63a --- /dev/null +++ b/neo4j/.archived/change_disabled_relationship_to_report_node.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +ENV_FILE=$(dirname "$0")/.env +[[ -f "$ENV_FILE" ]] && source "$ENV_FILE" + +if [ -z "$NEO4J_USERNAME" ] || [ -z "$NEO4J_PASSWORD" ]; then + echo "Please set NEO4J_USERNAME and NEO4J_PASSWORD environment variables." + echo "Database manipulation is not possible without connecting to the database." + echo "E.g. you could \`cp .env.template .env\` unless you run the script in a docker container" +fi + +until echo 'RETURN "Connection successful" as info;' | cypher-shell +do + echo "Connecting to neo4j failed, trying again..." + sleep 1 +done + +echo " +// convert old DISABLED to new REVIEWED-Report-BELONGS_TO structure +MATCH (moderator:User)-[disabled:DISABLED]->(disabledResource) +WHERE disabledResource:User OR disabledResource:Comment OR disabledResource:Post +DELETE disabled +CREATE (moderator)-[review:REVIEWED]->(report:Report)-[:BELONGS_TO]->(disabledResource) +SET review.createdAt = toString(datetime()), review.updatedAt = review.createdAt, review.disable = true +SET report.id = randomUUID(), report.createdAt = toString(datetime()), report.updatedAt = report.createdAt, report.rule = 'latestReviewUpdatedAtRules', report.closed = false +// if disabledResource has no filed report, then create a moderators default filed report +WITH moderator, disabledResource, report +OPTIONAL MATCH (disabledResourceReporter:User)-[existingFiledReport:FILED]->(disabledResource) +FOREACH(disabledResource IN CASE WHEN existingFiledReport IS NULL THEN [1] ELSE [] END | + CREATE (moderator)-[addModeratorReport:FILED]->(report) + SET addModeratorReport.createdAt = toString(datetime()), addModeratorReport.reasonCategory = 'other', addModeratorReport.reasonDescription = 'Old DISABLED relations didn't enforce mandatory reporting !!! Created automatically to ensure database consistency! Creation date is when the database manipulation happened.' +) +FOREACH(disabledResource IN CASE WHEN existingFiledReport IS NOT NULL THEN [1] ELSE [] END | + CREATE (disabledResourceReporter)-[moveModeratorReport:FILED]->(report) + SET moveModeratorReport = existingFiledReport + DELETE existingFiledReport +) +RETURN disabledResource {.id}; +" | cypher-shell + +echo " +// for FILED resources without DISABLED relation which are handled above, create new FILED-Report-BELONGS_TO structure +MATCH (reporter:User)-[oldReport:REPORTED]->(notDisabledResource) +WHERE notDisabledResource:User OR notDisabledResource:Comment OR notDisabledResource:Post +MERGE (report:Report)-[:BELONGS_TO]->(notDisabledResource) +ON CREATE SET report.id = randomUUID(), report.createdAt = toString(datetime()), report.updatedAt = report.createdAt, report.rule = 'latestReviewUpdatedAtRules', report.closed = false +CREATE (reporter)-[filed:FILED]->(report) +SET report = oldReport +DELETE oldReport +RETURN notDisabledResource {.id}; +" | cypher-shell From 28dae1f854781cc9792453a55e34105359578731 Mon Sep 17 00:00:00 2001 From: mattwr18 Date: Thu, 23 Jan 2020 15:40:08 +0100 Subject: [PATCH 11/12] Update docs, follow @roschaefer suggestions - create command should be run with --date-format to be more human readable, and --template-file to use our template instead of migrate's default - rename migrations - rename createdAt to migratedAt to remove ambiguity - do not merge relationships for Location nodes as we don't want to create duplicate relationships - use singular locationId as it's iterating one at a time --- backend/README.md | 4 ++-- backend/src/db/migrate/store.js | 4 ++-- ... 20200123150105-merge_duplicate_user_accounts.js} | 0 ...20200123150110-merge_duplicate_location_nodes.js} | 12 ++++++------ backend/src/models/Migration.js | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) rename backend/src/db/migrations/{1579387929122-merge_duplicate_user_accounts.js => 20200123150105-merge_duplicate_user_accounts.js} (100%) rename backend/src/db/migrations/{1579387929111-merge_duplicate_location_nodes.js => 20200123150110-merge_duplicate_location_nodes.js} (85%) diff --git a/backend/README.md b/backend/README.md index 7fd49faf8..2ad53ac08 100644 --- a/backend/README.md +++ b/backend/README.md @@ -120,7 +120,7 @@ you have to migrate your data e.g. because your data modeling has changed. {% tab title="Docker" %} Generate a data migration file: ```bash -$ docker-compose exec backend yarn run db:migrate:create your_data_migration +$ docker-compose exec backend yarn run db:migrate:create your_data_migration --date-format 'yyyymmddHHmmss' --template-file src/db/migrate/template.js # Edit the file in ./src/db/migrations/ ``` @@ -132,7 +132,7 @@ $ docker-compose exec backend yarn run db:migrate up {% tab title="Without Docker" %} Generate a data migration file: ```bash -$ yarn run db:migrate:create your_data_migration +$ yarn run db:migrate:create your_data_migration --date-format 'yyyymmddHHmmss' --template-file src/db/migrate/template.js # Edit the file in ./src/db/migrations/ ``` diff --git a/backend/src/db/migrate/store.js b/backend/src/db/migrate/store.js index 81df47708..f66ae91d8 100644 --- a/backend/src/db/migrate/store.js +++ b/backend/src/db/migrate/store.js @@ -35,7 +35,7 @@ class Store { const session = driver.session() const readTxResultPromise = session.readTransaction(async txc => { const result = await txc.run( - 'MATCH (migration:Migration) RETURN migration {.*} ORDER BY migration.createdAt DESC', + 'MATCH (migration:Migration) RETURN migration {.*} ORDER BY migration.migratedAt DESC', ) return result.records.map(r => r.get('migration')) }) @@ -71,7 +71,7 @@ class Store { ` MERGE (migration:Migration { title: $properties.title }) ON CREATE SET migration += $properties, - migration.createdAt = toString(datetime()) + migration.migratedAt = toString(datetime()) RETURN migration `, { properties }, diff --git a/backend/src/db/migrations/1579387929122-merge_duplicate_user_accounts.js b/backend/src/db/migrations/20200123150105-merge_duplicate_user_accounts.js similarity index 100% rename from backend/src/db/migrations/1579387929122-merge_duplicate_user_accounts.js rename to backend/src/db/migrations/20200123150105-merge_duplicate_user_accounts.js diff --git a/backend/src/db/migrations/1579387929111-merge_duplicate_location_nodes.js b/backend/src/db/migrations/20200123150110-merge_duplicate_location_nodes.js similarity index 85% rename from backend/src/db/migrations/1579387929111-merge_duplicate_location_nodes.js rename to backend/src/db/migrations/20200123150110-merge_duplicate_location_nodes.js index 390bfd935..9bd3d4d3b 100644 --- a/backend/src/db/migrations/1579387929111-merge_duplicate_location_nodes.js +++ b/backend/src/db/migrations/20200123150110-merge_duplicate_location_nodes.js @@ -24,19 +24,19 @@ export function up(next) { .records() .pipe( map(record => { - const { id: locationIds } = record.get('location') - return { locationIds } + const { id: locationId } = record.get('location') + return { locationId } }), - mergeMap(({ locationIds }) => { + mergeMap(({ locationId }) => { return transaction .run( ` - MATCH(location:Location {id: $locationIds}), (location2:Location {id: $locationIds}) + MATCH(location:Location {id: $locationId}), (location2:Location {id: $locationId}) WHERE location.id = location2.id AND id(location) < id(location2) - CALL apoc.refactor.mergeNodes([location, location2], { properties: 'combine', mergeRels: true }) YIELD node as updatedLocation + CALL apoc.refactor.mergeNodes([location, location2], { properties: 'combine' }) YIELD node as updatedLocation RETURN location {.*},updatedLocation {.*} `, - { locationIds }, + { locationId }, ) .records() .pipe( diff --git a/backend/src/models/Migration.js b/backend/src/models/Migration.js index 1b697c8ad..8f16b800a 100644 --- a/backend/src/models/Migration.js +++ b/backend/src/models/Migration.js @@ -1,5 +1,5 @@ export default { title: { type: 'string', primary: true, token: true }, description: { type: 'string' }, - createdAt: { type: 'string', isoDate: true, default: () => new Date().toISOString() }, + migratedAt: { type: 'string', isoDate: true, default: () => new Date().toISOString() }, } From e3437010574294b2569529e3740c3e2c3fca147f Mon Sep 17 00:00:00 2001 From: roschaefer Date: Thu, 23 Jan 2020 17:38:31 +0100 Subject: [PATCH 12/12] Reverted a couple changes by @mattwr18 We have to figure out if `mergeRels: true` is actually avoiding duplicate relationships :thinking:. Before: (l1)-[:IS_IN]->(l2) (l1)-[:IS_IN]->(l3) After: (l1)-[:IS_IN]->(new) (l1)-[:IS_IN]->(new) --- backend/README.md | 4 ++-- backend/package.json | 2 +- backend/src/db/migrate/store.js | 12 +++++++----- .../20200123150110-merge_duplicate_location_nodes.js | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/backend/README.md b/backend/README.md index 2ad53ac08..7fd49faf8 100644 --- a/backend/README.md +++ b/backend/README.md @@ -120,7 +120,7 @@ you have to migrate your data e.g. because your data modeling has changed. {% tab title="Docker" %} Generate a data migration file: ```bash -$ docker-compose exec backend yarn run db:migrate:create your_data_migration --date-format 'yyyymmddHHmmss' --template-file src/db/migrate/template.js +$ docker-compose exec backend yarn run db:migrate:create your_data_migration # Edit the file in ./src/db/migrations/ ``` @@ -132,7 +132,7 @@ $ docker-compose exec backend yarn run db:migrate up {% tab title="Without Docker" %} Generate a data migration file: ```bash -$ yarn run db:migrate:create your_data_migration --date-format 'yyyymmddHHmmss' --template-file src/db/migrate/template.js +$ yarn run db:migrate:create your_data_migration # Edit the file in ./src/db/migrations/ ``` diff --git a/backend/package.json b/backend/package.json index 042f96339..e44f325ed 100644 --- a/backend/package.json +++ b/backend/package.json @@ -16,7 +16,7 @@ "db:reset": "yarn run db:clean", "db:seed": "babel-node src/db/seed.js", "db:migrate": "yarn run __migrate --store ./src/db/migrate/store.js", - "db:migrate:create": "yarn run __migrate --template-file ./src/db/migrate/template.js create" + "db:migrate:create": "yarn run __migrate --template-file ./src/db/migrate/template.js --date-format 'yyyymmddHHmmss' create" }, "author": "Human Connection gGmbH", "license": "MIT", diff --git a/backend/src/db/migrate/store.js b/backend/src/db/migrate/store.js index f66ae91d8..8bc73b511 100644 --- a/backend/src/db/migrate/store.js +++ b/backend/src/db/migrate/store.js @@ -35,7 +35,7 @@ class Store { const session = driver.session() const readTxResultPromise = session.readTransaction(async txc => { const result = await txc.run( - 'MATCH (migration:Migration) RETURN migration {.*} ORDER BY migration.migratedAt DESC', + 'MATCH (migration:Migration) RETURN migration {.*} ORDER BY migration.timestamp DESC', ) return result.records.map(r => r.get('migration')) }) @@ -65,14 +65,16 @@ class Store { const writeTxResultPromise = session.writeTransaction(txc => { return Promise.all( migrations.map(async migration => { - const { title, description } = migration - const properties = { title, description } + const { title, description, timestamp } = migration + const properties = { title, description, timestamp } const migrationResult = await txc.run( ` MERGE (migration:Migration { title: $properties.title }) - ON CREATE SET migration += $properties, + ON MATCH SET + migration += $properties + ON CREATE SET + migration += $properties, migration.migratedAt = toString(datetime()) - RETURN migration `, { properties }, ) diff --git a/backend/src/db/migrations/20200123150110-merge_duplicate_location_nodes.js b/backend/src/db/migrations/20200123150110-merge_duplicate_location_nodes.js index 9bd3d4d3b..b2d6b260f 100644 --- a/backend/src/db/migrations/20200123150110-merge_duplicate_location_nodes.js +++ b/backend/src/db/migrations/20200123150110-merge_duplicate_location_nodes.js @@ -33,7 +33,7 @@ export function up(next) { ` MATCH(location:Location {id: $locationId}), (location2:Location {id: $locationId}) WHERE location.id = location2.id AND id(location) < id(location2) - CALL apoc.refactor.mergeNodes([location, location2], { properties: 'combine' }) YIELD node as updatedLocation + CALL apoc.refactor.mergeNodes([location, location2], { properties: 'combine', mergeRels: true }) YIELD node as updatedLocation RETURN location {.*},updatedLocation {.*} `, { locationId },