diff --git a/docker-compose.maintenance.yml b/docker-compose.maintenance.yml index b18f97513..33a9d28f6 100644 --- a/docker-compose.maintenance.yml +++ b/docker-compose.maintenance.yml @@ -6,7 +6,6 @@ services: build: context: maintenance volumes: - - mongo-export:/mongo-export - uploads:/uploads - neo4j-data:/data - ./maintenance/migration/:/migration @@ -36,7 +35,6 @@ services: - 7474:7474 volumes: - mongo-export: uploads: neo4j-data: diff --git a/maintenance/Dockerfile b/maintenance/Dockerfile index 83154d836..1fafce5e8 100644 --- a/maintenance/Dockerfile +++ b/maintenance/Dockerfile @@ -1,26 +1,11 @@ -FROM humanconnection/neo4j:latest as base +FROM humanconnection/neo4j:latest ENV NODE_ENV=maintenance EXPOSE 7687 7474 -VOLUME /mongo-export /uploads /data RUN apk upgrade --update RUN apk add --no-cache mongodb-tools openssh nodejs yarn rsync -FROM humanconnection/nitro-backend:latest as backend - -FROM base - -COPY --from=backend /nitro-backend /nitro-backend -COPY seeding/package.json . -# Install graphql-request manually, it's required for seeding and not included -# in backend's preinstalled node_modules/ -RUN yarn install && \ - cp -r ./node_modules/* /nitro-backend/node_modules/ && \ - rm -rf ./node_modules/ yarn.lock && \ - mv package.json /nitro-backend/package.json -# We have to do this odd copying to prevent cleaning `node_modules` folder which -# would happen if we `yarn install|add` in the target directory - +COPY known_hosts /root/.ssh/known_hosts COPY migration ./migration COPY ./binaries/* /usr/local/bin/ diff --git a/maintenance/binaries/import_legacy_uploads b/maintenance/binaries/import_legacy_uploads index 73240715b..24ae0fca5 100755 --- a/maintenance/binaries/import_legacy_uploads +++ b/maintenance/binaries/import_legacy_uploads @@ -9,5 +9,5 @@ do fi done -create_private_ssh_key_from_env +[ -z "$SSH_PRIVATE_KEY" ] || create_private_ssh_key_from_env rsync --archive --update --verbose ${SSH_USERNAME}@${SSH_HOST}:${UPLOADS_DIRECTORY}/* /uploads/ diff --git a/maintenance/binaries/reset_db b/maintenance/binaries/reset_db deleted file mode 100755 index ce768089a..000000000 --- a/maintenance/binaries/reset_db +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -set -e - -cd /nitro-backend && yarn run db:reset diff --git a/maintenance/binaries/seed_db b/maintenance/binaries/seed_db deleted file mode 100755 index ce768089a..000000000 --- a/maintenance/binaries/seed_db +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -set -e - -cd /nitro-backend && yarn run db:reset diff --git a/maintenance/known_hosts b/maintenance/known_hosts new file mode 100644 index 000000000..947840cb2 --- /dev/null +++ b/maintenance/known_hosts @@ -0,0 +1,3 @@ +|1|GuOYlVEhTowidPs18zj9p5F2j3o=|sDHJYLz9Ftv11oXeGEjs7SpVyg0= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM5N29bI5CeKu1/RBPyM2fwyf7fuajOO+tyhKe1+CC2sZ1XNB5Ff6t6MtCLNRv2mUuvzTbW/HkisDiA5tuXUHOk= +|1|2KP9NV+Q5g2MrtjAeFSVcs8YeOI=|nf3h4wWVwC4xbBS1kzgzE2tBldk= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNhRK6BeIEUxXlS0z/pOfkUkSPfn33g4J1U3L+MyUQYHm+7agT08799ANJhnvELKE1tt4Vx80I9UR81oxzZcy3E= +|1|HonYIRNhKyroUHPKU1HSZw0+Qzs=|5T1btfwFBz2vNSldhqAIfTbfIgQ= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNhRK6BeIEUxXlS0z/pOfkUkSPfn33g4J1U3L+MyUQYHm+7agT08799ANJhnvELKE1tt4Vx80I9UR81oxzZcy3E= diff --git a/maintenance/migration/mongo/import.sh b/maintenance/migration/mongo/import.sh index 89c255a76..328560bfc 100755 --- a/maintenance/migration/mongo/import.sh +++ b/maintenance/migration/mongo/import.sh @@ -9,15 +9,16 @@ echo "MONGODB_DATABASE ${MONGODB_DATABASE}" echo "MONGODB_AUTH_DB ${MONGODB_AUTH_DB}" echo "-------------------------------------------------" -create_private_ssh_key_from_env +[ -z "$SSH_PRIVATE_KEY" ] || create_private_ssh_key_from_env -rm -rf /mongo-export/* +rm -rf /tmp/mongo-export/* +mkdir -p /tmp/mongo-export ssh -4 -M -S my-ctrl-socket -fnNT -L 27018:localhost:27017 -l ${SSH_USERNAME} ${SSH_HOST} for collection in "categories" "badges" "users" "contributions" "comments" "follows" "shouts" do - mongoexport --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --db ${MONGODB_DATABASE} --collection $collection --out "/mongo-export/$collection.json" + mongoexport --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --db ${MONGODB_DATABASE} --collection $collection --out "/tmp/mongo-export/$collection.json" done ssh -S my-ctrl-socket -O check -l ${SSH_USERNAME} ${SSH_HOST} diff --git a/maintenance/migration/neo4j/badges.cql b/maintenance/migration/neo4j/badges.cql index 90e4755b4..f4bf67dda 100644 --- a/maintenance/migration/neo4j/badges.cql +++ b/maintenance/migration/neo4j/badges.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/mongo-export/badges.json') YIELD value as badge +CALL apoc.load.json('file:/tmp/mongo-export/badges.json') YIELD value as badge MERGE(b:Badge {id: badge._id["$oid"]}) ON CREATE SET b.key = badge.key, diff --git a/maintenance/migration/neo4j/categories.cql b/maintenance/migration/neo4j/categories.cql index a2bf6a352..c22354cbe 100644 --- a/maintenance/migration/neo4j/categories.cql +++ b/maintenance/migration/neo4j/categories.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/mongo-export/categories.json') YIELD value as category +CALL apoc.load.json('file:/tmp/mongo-export/categories.json') YIELD value as category MERGE(c:Category {id: category._id["$oid"]}) ON CREATE SET c.name = category.title, diff --git a/maintenance/migration/neo4j/comments.cql b/maintenance/migration/neo4j/comments.cql index 6709acbc8..eb645108a 100644 --- a/maintenance/migration/neo4j/comments.cql +++ b/maintenance/migration/neo4j/comments.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/mongo-export/comments.json') YIELD value as json +CALL apoc.load.json('file:/tmp/mongo-export/comments.json') YIELD value as json MERGE (comment:Comment {id: json._id["$oid"]}) ON CREATE SET comment.content = json.content, diff --git a/maintenance/migration/neo4j/contributions.cql b/maintenance/migration/neo4j/contributions.cql index 0c7b18959..134c276cf 100644 --- a/maintenance/migration/neo4j/contributions.cql +++ b/maintenance/migration/neo4j/contributions.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/mongo-export/contributions.json') YIELD value as post +CALL apoc.load.json('file:/tmp/mongo-export/contributions.json') YIELD value as post MERGE (p:Post {id: post._id["$oid"]}) ON CREATE SET p.title = post.title, diff --git a/maintenance/migration/neo4j/follows.cql b/maintenance/migration/neo4j/follows.cql index 0dad6a435..6f5416723 100644 --- a/maintenance/migration/neo4j/follows.cql +++ b/maintenance/migration/neo4j/follows.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/mongo-export/follows.json') YIELD value as follow +CALL apoc.load.json('file:/tmp/mongo-export/follows.json') YIELD value as follow MATCH (u1:User {id: follow.userId}), (u2:User {id: follow.foreignId}) MERGE (u1)-[:FOLLOWS]->(u2) ; diff --git a/maintenance/migration/neo4j/shouts.cql b/maintenance/migration/neo4j/shouts.cql index 60aca50c9..cd72ab66b 100644 --- a/maintenance/migration/neo4j/shouts.cql +++ b/maintenance/migration/neo4j/shouts.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/mongo-export/shouts.json') YIELD value as shout +CALL apoc.load.json('file:/tmp/mongo-export/shouts.json') YIELD value as shout MATCH (u:User {id: shout.userId}), (p:Post {id: shout.foreignId}) MERGE (u)-[:SHOUTED]->(p) ; diff --git a/maintenance/migration/neo4j/users.cql b/maintenance/migration/neo4j/users.cql index 5f87bb273..22eb46882 100644 --- a/maintenance/migration/neo4j/users.cql +++ b/maintenance/migration/neo4j/users.cql @@ -1,4 +1,4 @@ -CALL apoc.load.json('file:/mongo-export/users.json') YIELD value as user +CALL apoc.load.json('file:/tmp/mongo-export/users.json') YIELD value as user MERGE(u:User {id: user._id["$oid"]}) ON CREATE SET u.name = user.name, diff --git a/maintenance/seeding/.env.template b/maintenance/seeding/.env.template deleted file mode 100644 index abc62b2dc..000000000 --- a/maintenance/seeding/.env.template +++ /dev/null @@ -1,12 +0,0 @@ -NEO4J_URI=bolt://localhost:7687 -NEO4J_USER=neo4j -NEO4J_PASSWORD=letmein -GRAPHQL_PORT=4000 -GRAPHQL_URI=http://localhost:4000 -CLIENT_URI=http://localhost:3000 -MOCK=false - -JWT_SECRET="b/&&7b78BF&fv/Vd" -MAPBOX_TOKEN="pk.eyJ1IjoiaHVtYW4tY29ubmVjdGlvbiIsImEiOiJjajl0cnBubGoweTVlM3VwZ2lzNTNud3ZtIn0.KZ8KK9l70omjXbEkkbHGsQ" - -PRIVATE_KEY_PASSPHRASE="a7dsf78sadg87ad87sfagsadg78" diff --git a/maintenance/seeding/package.json b/maintenance/seeding/package.json deleted file mode 100644 index f3e5c115f..000000000 --- a/maintenance/seeding/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "license": "MIT", - "scripts": { - "start": "node dist/", - "db:script:seed": "wait-on tcp:4001 && node dist/seed/seed-db.js", - "db:reset": "node dist/seed/reset-db.js", - "db:seed": "cross-env GRAPHQL_URI=http://localhost:4001 GRAPHQL_PORT=4001 DISABLED_MIDDLEWARES=permissions run-p --race start db:script:seed" - }, - "dependencies": { - "graphql-request": "*" - } -}