improved accuracy of import to reflect different types of contributions & follows. Cleaned neo4j import folder structure

deployment/legacy-migration/maintenance-worker/migration/mongo/export.sh

@@ -8,11 +8,18 @@ set +o allexport
 
 # Export collection function defintion
 function export_collection () {
-  "${EXPORT_MONGOEXPORT_BIN}" --db ${MONGODB_DATABASE} --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --collection $1  --collection $1 --out "${EXPORT_PATH}$1.json"
+  "${EXPORT_MONGOEXPORT_BIN}" --db ${MONGODB_DATABASE} --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --collection $1 --out "${EXPORT_PATH}$1.json"
   mkdir -p ${EXPORT_PATH}splits/$1/
   split -l ${MONGO_EXPORT_SPLIT_SIZE} -a 3 ${EXPORT_PATH}$1.json ${EXPORT_PATH}splits/$1/
 }
 
+# Export collection with query function defintion
+function export_collection_query () {
+  "${EXPORT_MONGOEXPORT_BIN}" --db ${MONGODB_DATABASE} --host localhost -d ${MONGODB_DATABASE} --port 27018 --username ${MONGODB_USERNAME} --password ${MONGODB_PASSWORD} --authenticationDatabase ${MONGODB_AUTH_DB} --collection $1 --out "${EXPORT_PATH}$1_$3.json" --query "$2"
+  mkdir -p ${EXPORT_PATH}splits/$1_$3/
+  split -l ${MONGO_EXPORT_SPLIT_SIZE} -a 3 ${EXPORT_PATH}$1_$3.json ${EXPORT_PATH}splits/$1_$3/
+}
+
 # Delete old export & ensure directory
 rm -rf ${EXPORT_PATH}*
 mkdir -p ${EXPORT_PATH}
@@ -24,9 +31,12 @@ ssh -4 -M -S my-ctrl-socket -fnNT -L 27018:localhost:27017 -l ${SSH_USERNAME} ${
 export_collection "badges"
 export_collection "categories"
 export_collection "comments"
-export_collection "contributions"
+export_collection_query "contributions" "{'type': 'DELETED'}" "DELETED"
+export_collection_query "contributions" "{'type': 'post'}" "post"
+export_collection_query "contributions" "{'type': 'cando'}" "cando"
 export_collection "emotions"
-export_collection "follows"
+export_collection_query "follows" "{'foreignService': 'organizations'}" "organizations"
+export_collection_query "follows" "{'foreignService': 'users'}" "users"
 export_collection "invites"
 export_collection "notifications"
 export_collection "organizations"

deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh

@@ -10,7 +10,7 @@ set +o allexport
 function delete_collection () {
   # Delete from Database
   echo "Delete $1"
-  "${IMPORT_CYPHERSHELL_BIN}" < $(dirname "$0")/$1_delete.cql > /dev/null
+  "${IMPORT_CYPHERSHELL_BIN}" < $(dirname "$0")/$1/delete.cql > /dev/null
   # Delete index file
   rm -f "${IMPORT_PATH}splits/$1.index"
 }
@@ -34,7 +34,7 @@ function import_collection () {
       # calculate the path of the chunk
       export IMPORT_CHUNK_PATH_CQL_FILE="${IMPORT_CHUNK_PATH_CQL}$1/${CHUNK_FILE_NAME}"
       # load the neo4j command and replace file variable with actual path
-      NEO4J_COMMAND="$(envsubst '${IMPORT_CHUNK_PATH_CQL_FILE}' < $(dirname "$0")/$1.cql)"
+      NEO4J_COMMAND="$(envsubst '${IMPORT_CHUNK_PATH_CQL_FILE}' < $(dirname "$0")/$2)"
       # run the import of the chunk
       echo "Import $1 ${CHUNK_FILE_NAME} (${chunk})"
       echo "${NEO4J_COMMAND}" | "${IMPORT_CYPHERSHELL_BIN}" > /dev/null
@@ -75,13 +75,16 @@ echo "DONE"
 
 # Import Data
 echo "Start Importing Data"
-import_collection "badges"
-import_collection "categories"
-import_collection "users"
-import_collection "follows"
-import_collection "contributions"
-import_collection "shouts"
-import_collection "comments"
+import_collection "badges" "badges/badges.cql"
+import_collection "categories" "categories/categories.cql"
+import_collection "users" "users/users.cql"
+import_collection "follows_users" "follows/follows.cql"
+#import_collection "follows_organizations" "follows/follows.cql"
+import_collection "contributions_post" "contributions/contributions.cql"
+import_collection "contributions_cando" "contributions/contributions.cql"
+#import_collection "contributions_DELETED" "contributions/contributions.cql"
+import_collection "shouts" "shouts/shouts.cql"
+import_collection "comments" "comments/comments.cql"
 
 # import_collection "emotions"
 # import_collection "invites"