diff --git a/.dockerignore b/.dockerignore index 84b5adc92..f5a08be39 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,6 +6,7 @@ .env Dockerfile +docker-compose*.yml ./*.png ./*.log @@ -14,3 +15,6 @@ kubernetes/ node_modules/ scripts/ dist/ + +db-migration-worker/ +neo4j/ diff --git a/db-migration-worker/.gitignore b/db-migration-worker/.gitignore new file mode 100644 index 000000000..87cb01310 --- /dev/null +++ b/db-migration-worker/.gitignore @@ -0,0 +1 @@ +.ssh/id_rsa diff --git a/db-migration-worker/.ssh/.dockerignore b/db-migration-worker/.ssh/.dockerignore new file mode 100644 index 000000000..87cb01310 --- /dev/null +++ b/db-migration-worker/.ssh/.dockerignore @@ -0,0 +1 @@ +.ssh/id_rsa diff --git a/db-migration-worker/.ssh/known_hosts b/db-migration-worker/.ssh/known_hosts new file mode 100644 index 000000000..947840cb2 --- /dev/null +++ b/db-migration-worker/.ssh/known_hosts @@ -0,0 +1,3 @@ +|1|GuOYlVEhTowidPs18zj9p5F2j3o=|sDHJYLz9Ftv11oXeGEjs7SpVyg0= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM5N29bI5CeKu1/RBPyM2fwyf7fuajOO+tyhKe1+CC2sZ1XNB5Ff6t6MtCLNRv2mUuvzTbW/HkisDiA5tuXUHOk= +|1|2KP9NV+Q5g2MrtjAeFSVcs8YeOI=|nf3h4wWVwC4xbBS1kzgzE2tBldk= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNhRK6BeIEUxXlS0z/pOfkUkSPfn33g4J1U3L+MyUQYHm+7agT08799ANJhnvELKE1tt4Vx80I9UR81oxzZcy3E= +|1|HonYIRNhKyroUHPKU1HSZw0+Qzs=|5T1btfwFBz2vNSldhqAIfTbfIgQ= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNhRK6BeIEUxXlS0z/pOfkUkSPfn33g4J1U3L+MyUQYHm+7agT08799ANJhnvELKE1tt4Vx80I9UR81oxzZcy3E= diff --git a/db-migration-worker/Dockerfile b/db-migration-worker/Dockerfile new file mode 100644 index 000000000..92dff618f --- /dev/null +++ b/db-migration-worker/Dockerfile @@ -0,0 +1,9 @@ +FROM mongo:4 + +RUN apt-get update \ + && apt-get -y install --no-install-recommends openssh-client \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +COPY .ssh /root/.ssh/ +COPY import.sh . + diff --git a/db-migration-worker/import.sh b/db-migration-worker/import.sh new file mode 100755 index 000000000..ba07217c0 --- /dev/null +++ b/db-migration-worker/import.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +for var in "SSH_USERNAME" "SSH_HOST" "MONGODB_USERNAME" "MONGODB_PASSWORD" "MONGODB_DATABASE" "MONGODB_AUTH_DB" +do + if [[ -z "${!var}" ]]; then + echo "${var} is undefined" + exit 1 + fi +done + +echo "SSH_USERNAME ${SSH_USERNAME}" +echo "SSH_HOST ${SSH_HOST}" +echo "MONGODB_USERNAME ${MONGODB_USERNAME}" +echo "MONGODB_PASSWORD ${MONGODB_PASSWORD}" +echo "MONGODB_DATABASE ${MONGODB_DATABASE}" +echo "MONGODB_AUTH_DB ${MONGODB_AUTH_DB}" +echo "-------------------------------------------------" + +mongo ${MONGODB_DATABASE} --eval "db.dropDatabase();" +rm -f /mongo-export/* + +ssh -4 -M -S my-ctrl-socket -fnNT -L 27018:localhost:27017 -l ${SSH_USERNAME} ${SSH_HOST} +mongodump --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --gzip --archive | mongorestore --gzip --archive +ssh -S my-ctrl-socket -O check -l ${SSH_USERNAME} ${SSH_HOST} +ssh -S my-ctrl-socket -O exit -l ${SSH_USERNAME} ${SSH_HOST} + +for collection in "categories" "badges" "users" "contributions" "comments" "follows" "shouts" +do + mongoexport --db ${MONGODB_DATABASE} --collection $collection --out "/mongo-export/$collection.json" +done diff --git a/docker-compose.override.yml b/docker-compose.override.yml index ef7d52c7e..c5e7d5cf9 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -11,6 +11,30 @@ services: - /nitro-backend/node_modules command: yarn run dev neo4j: + volumes: + - mongo-export:/mongo-export + - ./neo4j/import:/var/lib/neo4j/import ports: - 7687:7687 - 7474:7474 + environment: + - NEO4J_apoc_import_file_enabled=true + db-migration-worker: + build: + context: db-migration-worker + volumes: + - mongo-export:/mongo-export + - ./db-migration-worker/.ssh/:/root/.ssh/ + networks: + - hc-network + environment: + - "SSH_USERNAME=${SSH_USERNAME}" + - "SSH_HOST=${SSH_HOST}" + - "MONGODB_USERNAME=${MONGODB_USERNAME}" + - "MONGODB_PASSWORD=${MONGODB_PASSWORD}" + - "MONGODB_AUTH_DB=${MONGODB_AUTH_DB}" + - "MONGODB_DATABASE=${MONGODB_DATABASE}" + command: "--smallfiles --logpath=/dev/null" + +volumes: + mongo-export: diff --git a/docker-compose.yml b/docker-compose.yml index 5a7650aa1..6905bb893 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,15 +17,14 @@ services: - GRAPHQL_PORT=4000 - GRAPHQL_URI=http://localhost:4000 - CLIENT_URI=http://localhost:3000 - - JWT_SECRET="b/&&7b78BF&fv/Vd" + - JWT_SECRET=b/&&7b78BF&fv/Vd - MOCK=false - MAPBOX_TOKEN=pk.eyJ1IjoiaHVtYW4tY29ubmVjdGlvbiIsImEiOiJjajl0cnBubGoweTVlM3VwZ2lzNTNud3ZtIn0.KZ8KK9l70omjXbEkkbHGsQ neo4j: image: humanconnection/neo4j:latest build: - context: . - dockerfile: Dockerfile.neo4j + context: neo4j networks: - hc-network volumes: diff --git a/Dockerfile.neo4j b/neo4j/Dockerfile similarity index 86% rename from Dockerfile.neo4j rename to neo4j/Dockerfile index cb7fd228f..07344b47b 100644 --- a/Dockerfile.neo4j +++ b/neo4j/Dockerfile @@ -1,2 +1,3 @@ FROM neo4j:3.5.0 RUN wget https://github.com/neo4j-contrib/neo4j-apoc-procedures/releases/download/3.5.0.1/apoc-3.5.0.1-all.jar -P plugins/ +COPY import ./import diff --git a/neo4j/import/comments.cql b/neo4j/import/comments.cql new file mode 100644 index 000000000..16537b730 --- /dev/null +++ b/neo4j/import/comments.cql @@ -0,0 +1,12 @@ +CALL apoc.load.json('file:/mongo-export/comments.json') YIELD value as comment +MERGE (c:Comment {id: comment._id["$oid"]}) +ON CREATE SET +c.content = comment.content, +c.contentExcerpt = comment.contentExcerpt, +c.deleted = comment.deleted, +c.disabled = false +WITH comment +MATCH (p:Post {id: comment.contributionId}), (u:User {id: comment.userId}) +MERGE (c)-[:COMMENTS]->(p) +MERGE (u)-[:WROTE]->(c) +; diff --git a/neo4j/import/contributions.cql b/neo4j/import/contributions.cql new file mode 100644 index 000000000..86226b98f --- /dev/null +++ b/neo4j/import/contributions.cql @@ -0,0 +1,23 @@ +CALL apoc.load.json('file:/mongo-export/contributions.json') YIELD value as post +MERGE (p:Post {id: post._id["$oid"]}) +ON CREATE SET +p.title = post.title, +p.slug = post.slug, +p.image = post.teaserImg, +p.content = post.content, +p.contentExcerpt = post.contentExcerpt, +p.visibility = toLower(post.visibility), +p.createdAt = post.createdAt.`$date`, +p.updatedAt = post.updatedAt.`$date`, +p.deleted = post.deleted, +p.disabled = NOT post.isEnabled +WITH p, post, post.tags AS tags, post.categoryIds as categoryIds +UNWIND tags AS tag +UNWIND categoryIds AS categoryId +MATCH (c:Category {id: categoryId}), + (u:User {id: post.userId}) +MERGE (t:Tag {id: apoc.create.uuid(), name: tag}) +MERGE (p)-[:TAGGED]->(t) +MERGE (u)-[:WROTE]->(p) +MERGE (p)-[:CATEGORIZED]->(c) +; diff --git a/neo4j/import/import.sh b/neo4j/import/import.sh new file mode 100755 index 000000000..319f1a591 --- /dev/null +++ b/neo4j/import/import.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +echo "MATCH (n) OPTIONAL MATCH (n)-[r]-() DELETE n,r;" | cypher-shell +for collection in "users" "contributions" "comments" +do + echo "Import ${collection}..." && cypher-shell < $SCRIPT_DIRECTORY/$collection.cql +done diff --git a/neo4j/import/todo b/neo4j/import/todo new file mode 100644 index 000000000..0b86c8bc5 --- /dev/null +++ b/neo4j/import/todo @@ -0,0 +1,29 @@ +CALL apoc.load.json('file:/mongo-export/categories.json') YIELD value as category +MERGE(c:Category {id: category._id["$oid"]}) +ON CREATE SET c.name = category.title, + c.slug = category.slug, + c.icon = category.icon + + +CALL apoc.load.json('file:/mongo-export/badges.json') YIELD value as badge +MERGE(b:Badge {id: badge._id["$oid"]}) +ON CREATE SET b.key = badge.key, + b.type = badge.type, + b.icon = badge.image.path, + b.status = badge.status + + + + + + +CALL apoc.load.json('file:/mongo-export/follows.json') YIELD value as follow +MATCH (u1:User {id: follow.userId}), + (u2:User {id: follow.foreignId}) +MERGE (u1)-[:FOLLOWS]->(u2) + + +CALL apoc.load.json('file:/mongo-export/shouts.json') YIELD value as shout +MATCH (u:User {id: shout.userId}), + (p:Post {id: shout.foreignId}) +MERGE (u)-[:SHOUTED]->(p) diff --git a/neo4j/import/users.cql b/neo4j/import/users.cql new file mode 100644 index 000000000..5f87bb273 --- /dev/null +++ b/neo4j/import/users.cql @@ -0,0 +1,20 @@ +CALL apoc.load.json('file:/mongo-export/users.json') YIELD value as user +MERGE(u:User {id: user._id["$oid"]}) +ON CREATE SET +u.name = user.name, +u.slug = user.slug, +u.email = user.email, +u.password = user.password, +u.avatar = user.avatar, +u.coverImg = user.coverImg, +u.wasInvited = user.wasInvited, +u.role = toLower(user.role), +u.createdAt = user.createdAt.`$date`, +u.updatedAt = user.updatedAt.`$date`, +u.deleted = user.deletedAt IS NOT NULL, +u.disabled = false +WITH u, user, user.badgeIds AS badgeIds +UNWIND badgeIds AS badgeId +MATCH (b:Badge {id: badgeId}) +MERGE (b)-[:REWARDED]->(u) +; diff --git a/src/middleware/fixImageUrlsMiddleware.js b/src/middleware/fixImageUrlsMiddleware.js index 919a46ab6..c3b828dae 100644 --- a/src/middleware/fixImageUrlsMiddleware.js +++ b/src/middleware/fixImageUrlsMiddleware.js @@ -1,14 +1,25 @@ -const urlSearchAlpha = 'https://api-alpha.human-connection.org' -const urlSearchLocal = 'http://localhost:3000' +const legacyUrls = [ + 'https://api-alpha.human-connection.org', + 'https://staging-api.human-connection.org', + 'http://localhost:3000' +] export const fixUrl = (url) => { - url = url.replace(urlSearchAlpha, '') - url = url.replace(urlSearchLocal, '') + legacyUrls.forEach((legacyUrl) => { + url = url.replace(legacyUrl, '') + }) return url } -const fixImageURLs = (result, recursive) => { - if (result && typeof result === 'string' && (result.indexOf(urlSearchAlpha) === 0 || result.indexOf(urlSearchLocal) === 0)) { + +const checkUrl = (thing) => { + return thing && typeof thing === 'string' && legacyUrls.find((legacyUrl) => { + return thing.indexOf(legacyUrl) === 0 + }) +} + +export const fixImageURLs = (result, recursive) => { + if (checkUrl(result)) { result = fixUrl(result) } else if (result && Array.isArray(result)) { result.forEach((res, index) => { diff --git a/src/middleware/fixImageUrlsMiddleware.spec.js b/src/middleware/fixImageUrlsMiddleware.spec.js new file mode 100644 index 000000000..081154c5c --- /dev/null +++ b/src/middleware/fixImageUrlsMiddleware.spec.js @@ -0,0 +1,30 @@ +import { fixImageURLs } from './fixImageUrlsMiddleware' + +describe('fixImageURLs', () => { + describe('image url of legacy alpha', () => { + it('removes domain', () => { + const url = 'https://api-alpha.human-connection.org/uploads/4bfaf9172c4ba03d7645108bbbd16f0a696a37d01eacd025fb131e5da61b15d9.png' + expect(fixImageURLs(url)).toEqual('/uploads/4bfaf9172c4ba03d7645108bbbd16f0a696a37d01eacd025fb131e5da61b15d9.png') + }) + }) + + describe('image url of legacy staging', () => { + it('removes domain', () => { + const url = 'https://staging-api.human-connection.org/uploads/1b3c39a24f27e2fb62b69074b2f71363b63b263f0c4574047d279967124c026e.jpeg' + expect(fixImageURLs(url)).toEqual('/uploads/1b3c39a24f27e2fb62b69074b2f71363b63b263f0c4574047d279967124c026e.jpeg') + }) + }) + + describe('object', () => { + it('returns untouched', () => { + const object = { some: 'thing' } + expect(fixImageURLs(object)).toEqual(object) + }) + }) + + describe('some string', () => { + it('returns untouched', () => {}) + const string = 'Yeah I\'m a String' + expect(fixImageURLs(string)).toEqual(string) + }) +}) diff --git a/src/seed/reset-db.js b/src/seed/reset-db.js index 616ff71e8..7d7c4f3f9 100644 --- a/src/seed/reset-db.js +++ b/src/seed/reset-db.js @@ -11,7 +11,12 @@ if (process.env.NODE_ENV === 'production') { const driver = neo4j().getDriver() const session = driver.session() -query('MATCH (n) DETACH DELETE n', session).then(() => { +const deleteAll = ` +MATCH (n) +OPTIONAL MATCH (n)-[r]-() +DELETE n,r +` +query(deleteAll, session).then(() => { /* eslint-disable-next-line no-console */ console.log('Successfully deleted all nodes and relations!') }).catch((err) => {