From 271af7dde2fc58db321fd0a65b12a966b6999818 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Wed, 29 May 2019 20:02:30 +0200 Subject: [PATCH] keep index file for already imported files, do not reimport already imported file, invites, notifications, emotions and organisations scripts and descriptions --- .../migration/neo4j/emotions.cql | 35 +++++ .../migration/neo4j/emotions_delete.cql | 0 .../migration/neo4j/import.sh | 51 +++++-- .../migration/neo4j/invites.cql | 39 +++++ .../migration/neo4j/invites_delete.cql | 0 .../migration/neo4j/notifications.cql | 48 ++++++ .../migration/neo4j/notifications_delete.cql | 0 .../migration/neo4j/organizations.cql | 137 ++++++++++++++++++ .../migration/neo4j/organizations_delete.cql | 0 9 files changed, 298 insertions(+), 12 deletions(-) create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions_delete.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/invites.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/invites_delete.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications_delete.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations_delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions.cql new file mode 100644 index 000000000..8aad9e923 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions.cql @@ -0,0 +1,35 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] userId: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] contributionId: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] rated: { +[ ] type: String, +[ ] required: true, +[ ] enum: ['funny', 'happy', 'surprised', 'cry', 'angry'] + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as emotion; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions_delete.cql new file mode 100644 index 000000000..e69de29bb diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh index ecf4b8de4..af2a6887f 100755 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh @@ -8,18 +8,43 @@ set +o allexport # Delete collection function defintion function delete_collection () { + # Delete from Database echo "Delete $1" "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} < $(dirname "$0")/$1_delete.cql > /dev/null + # Delete index file + rm -f "${IMPORT_PATH}splits/$1.index" } # Import collection function defintion function import_collection () { + # index file of those chunks we have already imported + INDEX_FILE="${IMPORT_PATH}splits/$1.index" + # load index file + if [ -f "$INDEX_FILE" ]; then + readarray -t IMPORT_INDEX <$INDEX_FILE + else + declare -a IMPORT_INDEX + fi + # for each chunk import data for chunk in ${IMPORT_PATH}splits/$1/* do - export IMPORT_CHUNK_PATH_CQL_FILE="${IMPORT_CHUNK_PATH_CQL}$1/$(basename "${chunk}")" - NEO4J_COMMAND="$(envsubst '${IMPORT_CHUNK_PATH_CQL_FILE}' < $(dirname "$0")/$1.cql)" - echo "Import ${chunk}" - echo "${NEO4J_COMMAND}" | "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} > /dev/null + CHUNK_FILE_NAME=$(basename "${chunk}") + # does the index not contain the chunk file name? + if [[ ! " ${IMPORT_INDEX[@]} " =~ " ${CHUNK_FILE_NAME} " ]]; then + # calculate the path of the chunk + export IMPORT_CHUNK_PATH_CQL_FILE="${IMPORT_CHUNK_PATH_CQL}$1/${CHUNK_FILE_NAME}" + # load the neo4j command and replace file variable with actual path + NEO4J_COMMAND="$(envsubst '${IMPORT_CHUNK_PATH_CQL_FILE}' < $(dirname "$0")/$1.cql)" + # run the import of the chunk + echo "Import $1 ${CHUNK_FILE_NAME} (${chunk})" + echo "${NEO4J_COMMAND}" | "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} > /dev/null + # add file to array and file + IMPORT_INDEX+=("${CHUNK_FILE_NAME}") + echo "${CHUNK_FILE_NAME}" >> ${INDEX_FILE} + echo "${IMPORT_INDEX[*]}" + else + echo "Skipping $1 ${CHUNK_FILE_NAME} (${chunk})" + fi done } @@ -36,10 +61,11 @@ delete_collection "contributions" delete_collection "shouts" delete_collection "comments" -#delete_collection "emotions" -#delete_collection "invites" -#delete_collection "notifications" -#delete_collection "organizations" +delete_collection "emotions" +delete_collection "invites" +delete_collection "notifications" +delete_collection "organizations" + #delete_collection "pages" #delete_collection "projects" #delete_collection "settings" @@ -59,10 +85,11 @@ import_collection "contributions" import_collection "shouts" import_collection "comments" -#import_collection "emotions" -#import_collection "invites" -#import_collection "notifications" -#import_collection "organizations" +import_collection "emotions" +import_collection "invites" +import_collection "notifications" +import_collection "organizations" + #import_collection "pages" #import_collection "projects" #import_collection "settings" diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/invites.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/invites.cql new file mode 100644 index 000000000..f4a5bf006 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/invites.cql @@ -0,0 +1,39 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] email: { +[ ] type: String, +[ ] required: true, +[-] index: true, +[ ] unique: true + }, +[ ] code: { +[ ] type: String, +[-] index: true, +[ ] required: true + }, +[ ] role: { +[ ] type: String, +[ ] enum: ['admin', 'moderator', 'manager', 'editor', 'user'], +[ ] default: 'user' + }, +[ ] invitedByUserId: { type: String }, +[ ] language: { type: String }, +[ ] badgeIds: [], +[ ] wasUsed: { +[ ] type: Boolean, +[-] index: true + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as invite; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/invites_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/invites_delete.cql new file mode 100644 index 000000000..e69de29bb diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications.cql new file mode 100644 index 000000000..aa6ac8eb9 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications.cql @@ -0,0 +1,48 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] userId: { // User this notification is sent to +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] type: { +[ ] type: String, +[ ] required: true, +[ ] enum: ['comment','comment-mention','contribution-mention','following-contribution'] + }, +[ ] relatedUserId: { +[ ] type: String, +[-] index: true + }, +[ ] relatedContributionId: { +[ ] type: String, +[-] index: true + }, +[ ] relatedOrganizationId: { +[ ] type: String, +[-] index: true + }, +[ ] relatedCommentId: {type: String }, +[ ] unseen: { +[ ] type: Boolean, +[ ] default: true, +[-] index: true + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as notification; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications_delete.cql new file mode 100644 index 000000000..e69de29bb diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations.cql new file mode 100644 index 000000000..50e8f6765 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations.cql @@ -0,0 +1,137 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { + name: { + type: String, + required: true, + index: true + }, + slug: { + type: String, + required: true, + unique: true, + index: true + }, + followersCounts: { + users: { + type: Number, + default: 0 + }, + organizations: { + type: Number, + default: 0 + }, + projects: { + type: Number, + default: 0 + } + }, + followingCounts: { + users: { + type: Number, + default: 0 + }, + organizations: { + type: Number, + default: 0 + }, + projects: { + type: Number, + default: 0 + } + }, + categoryIds: { + type: Array, + required: true, + index: true + }, + logo: { type: String }, + coverImg: { type: String }, + userId: { + type: String, + required: true, + index: true + }, + description: { + type: String, + required: true + }, + descriptionExcerpt: { type: String }, // will be generated automatically + publicEmail: { type: String }, + url: { type: String }, + type: { + type: String, + index: true, + enum: ['ngo', 'npo', 'goodpurpose', 'ev', 'eva'] + }, + language: { + type: String, + required: true, + default: 'de', + index: true + }, + addresses: { + type: [{ + street: { + type: String, + required: true + }, + zipCode: { + type: String, + required: true + }, + city: { + type: String, + required: true + }, + country: { + type: String, + required: true + }, + lat: { + type: Number, + required: true + }, + lng: { + type: Number, + required: true + } + }], + default: [] + }, + createdAt: { + type: Date, + default: Date.now + }, + updatedAt: { + type: Date, + default: Date.now + }, + isEnabled: { + type: Boolean, + default: false, + index: true + }, + reviewedBy: { + type: String, + default: null, + index: true + }, + tags: { + type: Array, + index: true + }, + deleted: { + type: Boolean, + default: false, + index: true + }, + wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as organisation; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations_delete.cql new file mode 100644 index 000000000..e69de29bb