IT4Change/Ocelot-Social
3
0
Fork 0
mirror of https://github.com/IT4Change/Ocelot-Social.git synced 2025-12-13 07:45:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #529 from Human-Connection/split_data_import

Browse Source 
Performant import of production data
This commit is contained in:
mattwr18 2019-05-07 18:08:53 -03:00 committed by GitHub
commit 8baf9ca25e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 54 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
backend/docker-compose.prod.yml
View File

@ -1,9 +0,0 @@
version: "3.7"
services:
neo4j:
environment:
- NEO4J_PASSWORD=letmein
backend:
environment:
- NEO4J_PASSWORD=letmein

6
deployment/legacy-migration/maintenance-worker/binaries/create_private_ssh_key_from_env
View File

@ -1,6 +0,0 @@
#!/usr/bin/env bash
set -e
mkdir -p ~/.ssh
echo $SSH_PRIVATE_KEY | base64 -d > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa

2
deployment/legacy-migration/maintenance-worker/binaries/idle Executable file
View File

@ -0,0 +1,2 @@
#!/usr/bin/env bash
tail -f /dev/null

2
deployment/legacy-migration/maintenance-worker/binaries/import_legacy_db
View File

@ -1,6 +1,6 @@
#!/usr/bin/env bash
set -e
for var in "SSH_USERNAME" "SSH_HOST" "MONGODB_USERNAME" "MONGODB_PASSWORD" "MONGODB_DATABASE" "MONGODB_AUTH_DB" "NEO4J_URI"
for var in "SSH_USERNAME" "SSH_HOST" "MONGODB_USERNAME" "MONGODB_PASSWORD" "MONGODB_DATABASE" "MONGODB_AUTH_DB"
do
if [[ -z "${!var}" ]]; then
echo "${var} is undefined"

1
deployment/legacy-migration/maintenance-worker/binaries/import_legacy_uploads
View File

@ -9,5 +9,4 @@ do
fi
done
[ -z "$SSH_PRIVATE_KEY" ] || create_private_ssh_key_from_env
rsync --archive --update --verbose ${SSH_USERNAME}@${SSH_HOST}:${UPLOADS_DIRECTORY}/* /uploads/

7
deployment/legacy-migration/maintenance-worker/migration/mongo/import.sh
View File

@ -9,16 +9,17 @@ echo "MONGODB_DATABASE ${MONGODB_DATABASE}"
echo "MONGODB_AUTH_DB ${MONGODB_AUTH_DB}"
echo "-------------------------------------------------"
[ -z "$SSH_PRIVATE_KEY" ] || create_private_ssh_key_from_env
rm -rf /tmp/mongo-export/*
mkdir -p /tmp/mongo-export
mkdir -p /tmp/mongo-export/
ssh -4 -M -S my-ctrl-socket -fnNT -L 27018:localhost:27017 -l ${SSH_USERNAME} ${SSH_HOST}
for collection in "categories" "badges" "users" "contributions" "comments" "follows" "shouts"
do
mongoexport --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --db ${MONGODB_DATABASE} --collection $collection --out "/tmp/mongo-export/$collection.json"
mongoexport --db ${MONGODB_DATABASE} --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --collection $collection --collection $collection --out "/tmp/mongo-export/$collection.json"
mkdir -p /tmp/mongo-export/splits/$collection/
split -l 1000 -a 3 /tmp/mongo-export/$collection.json /tmp/mongo-export/splits/$collection/
done
ssh -S my-ctrl-socket -O check -l ${SSH_USERNAME} ${SSH_HOST}

2
deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql
View File

@ -1,4 +1,4 @@
CALL apoc.load.json('file:/tmp/mongo-export/badges.json') YIELD value as badge
CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as badge
MERGE(b:Badge {id: badge._id["$oid"]})
ON CREATE SET
b.key = badge.key,

2
deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql
View File

@ -1,4 +1,4 @@
CALL apoc.load.json('file:/tmp/mongo-export/categories.json') YIELD value as category
CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as category
MERGE(c:Category {id: category._id["$oid"]})
ON CREATE SET
c.name = category.title,

3
deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql
View File

@ -1,4 +1,5 @@
CALL apoc.load.json('file:/tmp/mongo-export/comments.json') YIELD value as json
CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as json
MERGE (comment:Comment {id: json._id["$oid"]})
ON CREATE SET
comment.content = json.content,

2
deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql
View File

@ -1,4 +1,4 @@
CALL apoc.load.json('file:/tmp/mongo-export/contributions.json') YIELD value as post
CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as post
MERGE (p:Post {id: post._id["$oid"]})
ON CREATE SET
p.title = post.title,

2
deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql
View File

@ -1,4 +1,4 @@
CALL apoc.load.json('file:/tmp/mongo-export/follows.json') YIELD value as follow
CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as follow
MATCH (u1:User {id: follow.userId}), (u2:User {id: follow.foreignId})
MERGE (u1)-[:FOLLOWS]->(u2)
;

12
deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh
View File

@ -1,9 +1,17 @@
#!/usr/bin/env bash
set -e
SECONDS=0
SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
echo "MATCH (n) OPTIONAL MATCH (n)-[r]-() DELETE n,r;" | cypher-shell -a $NEO4J_URI
echo "MATCH (n) DETACH DELETE n;" | cypher-shell
for collection in "badges" "categories" "users" "follows" "contributions" "shouts" "comments"
do
echo "Import ${collection}..." && cypher-shell -a $NEO4J_URI < $SCRIPT_DIRECTORY/$collection.cql
for chunk in /tmp/mongo-export/splits/$collection/*
do
mv $chunk /tmp/mongo-export/splits/current-chunk.json
echo "Import ${chunk}" && cypher-shell < $SCRIPT_DIRECTORY/$collection.cql
done
done
echo "Time elapsed: $SECONDS seconds"

2
deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql
View File

@ -1,4 +1,4 @@
CALL apoc.load.json('file:/tmp/mongo-export/shouts.json') YIELD value as shout
CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as shout
MATCH (u:User {id: shout.userId}), (p:Post {id: shout.foreignId})
MERGE (u)-[:SHOUTED]->(p)
;

2
deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql
View File

@ -1,4 +1,4 @@
CALL apoc.load.json('file:/tmp/mongo-export/users.json') YIELD value as user
CALL apoc.load.json('file:/tmp/mongo-export/splits/current-chunk.json') YIELD value as user
MERGE(u:User {id: user._id["$oid"]})
ON CREATE SET
u.name = user.name,

20
deployment/legacy-migration/maintenance-worker/docker-compose.yml → docker-compose.maintenance.yml
View File

@ -4,14 +4,17 @@ services:
maintenance:
image: humanconnection/maintenance-worker:latest
build:
context: .
context: deployment/legacy-migration/maintenance-worker
volumes:
- uploads:/uploads
- neo4j-data:/data
- ./migration/:/migration
- ./deployment/legacy-migration/maintenance-worker/migration/:/migration
- ./deployment/legacy-migration/maintenance-worker/ssh/:/root/.ssh
networks:
- hc-network
environment:
- NEO4J_dbms_security_auth__enabled=false
- NEO4J_dbms_memory_heap_max__size=2G
- GRAPHQL_PORT=4000
- GRAPHQL_URI=http://localhost:4000
- CLIENT_URI=http://localhost:3000
@ -19,12 +22,9 @@ services:
- MOCK=false
- MAPBOX_TOKEN=pk.eyJ1IjoiaHVtYW4tY29ubmVjdGlvbiIsImEiOiJjajl0cnBubGoweTVlM3VwZ2lzNTNud3ZtIn0.KZ8KK9l70omjXbEkkbHGsQ
- PRIVATE_KEY_PASSPHRASE=a7dsf78sadg87ad87sfagsadg78
- NEO4J_URI=bolt://localhost:7687
- NEO4J_apoc_import_file_enabled=true
- NEO4J_AUTH=none
- "SSH_USERNAME=${SSH_USERNAME}"
- "SSH_HOST=${SSH_HOST}"
- "SSH_PRIVATE_KEY=${SSH_PRIVATE_KEY}"
- "MONGODB_USERNAME=${MONGODB_USERNAME}"
- "MONGODB_PASSWORD=${MONGODB_PASSWORD}"
- "MONGODB_AUTH_DB=${MONGODB_AUTH_DB}"
@ -34,9 +34,11 @@ services:
- 7687:7687
- 7474:7474
volumes:
uploads:
neo4j-data:
networks:
hc-network:
volumes:
webapp_node_modules:
backend_node_modules:
neo4j-data:
uploads:

2
docker-compose.override.yml
View File

@ -18,6 +18,7 @@ services:
volumes:
- ./backend:/nitro-backend
- backend_node_modules:/nitro-backend/node_modules
- uploads:/nitro-backend/public/uploads
command: yarn run dev
neo4j:
environment:
@ -32,3 +33,4 @@ volumes:
webapp_node_modules:
backend_node_modules:
neo4j-data:
uploads:

17
neo4j/migrate.sh
View File

@ -4,15 +4,30 @@
# the initial default user. Before we can create constraints, we have to change
# the default password. This is a security feature of neo4j.
if echo ":exit" | cypher-shell --password neo4j 2> /dev/null ; then
echo "CALL dbms.security.changePassword('${NEO4J_PASSWORD}');" | cypher-shell --password neo4j
if [[ -z "${NEO4J_PASSWORD}" ]]; then
echo "NEO4J_PASSWORD environment variable is undefined. I cannot set the initial password."
else
echo "CALL dbms.security.changePassword('${NEO4J_PASSWORD}');" | cypher-shell --password neo4j
fi
fi
set -e
echo '
CALL db.index.fulltext.createNodeIndex("full_text_search",["Post"],["title", "content"]);
CREATE CONSTRAINT ON (p:Post) ASSERT p.id IS UNIQUE;
CREATE CONSTRAINT ON (c:Comment) ASSERT c.id IS UNIQUE;
CREATE CONSTRAINT ON (c:Category) ASSERT c.id IS UNIQUE;
CREATE CONSTRAINT ON (u:User) ASSERT u.id IS UNIQUE;
CREATE CONSTRAINT ON (o:Organization) ASSERT o.id IS UNIQUE;
CREATE CONSTRAINT ON (t:Tag) ASSERT t.id IS UNIQUE;
CREATE CONSTRAINT ON (p:Post) ASSERT p.slug IS UNIQUE;
CREATE CONSTRAINT ON (c:Category) ASSERT c.slug IS UNIQUE;
CREATE CONSTRAINT ON (u:User) ASSERT u.slug IS UNIQUE;
CREATE CONSTRAINT ON (o:Organization) ASSERT o.slug IS UNIQUE;
' | cypher-shell
echo "Successfully created all indices and unique constraints:"
echo 'CALL db.indexes();' | cypher-shell

16
webapp/docker-compose.override.yml
View File

@ -1,16 +0,0 @@
version: '3.7'
services:
webapp:
build:
context: .
target: build-and-test
volumes:
- .:/nitro-web
- node_modules:/nitro-web/node_modules
- nuxt:/nitro-web/.nuxt
command: yarn run dev
volumes:
node_modules:
nuxt:

10
webapp/docker-compose.travis.yml
View File

@ -1,10 +0,0 @@
version: "3.7"
services:
webapp:
build:
context: .
target: build-and-test
environment:
- GRAPHQL_URI=http://backend:4123
- NODE_ENV=test

23
webapp/docker-compose.yml
View File

@ -1,23 +0,0 @@
version: '3.7'
services:
webapp:
image: humanconnection/nitro-web:latest
build:
context: .
target: production
ports:
- 3000:3000
networks:
- hc-network
environment:
- HOST=0.0.0.0
- GRAPHQL_URI=http://backend:4000
- MAPBOX_TOKEN="pk.eyJ1IjoiaHVtYW4tY29ubmVjdGlvbiIsImEiOiJjajl0cnBubGoweTVlM3VwZ2lzNTNud3ZtIn0.bZ8KK9l70omjXbEkkbHGsQ"
networks:
hc-network:
name: hc-network
volumes:
node_modules: