keep index file for already imported files, do not reimport already imported file, invites, notifications, emotions and organisations scripts and descriptions

deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions.cql

@@ -0,0 +1,35 @@
+/*
+// Alpha Model
+// [ ] Not modeled in Nitro
+// [X] Modeled in Nitro
+// [-] Omitted in Nitro
+// [?] Unclear / has work to be done for Nitro
+    {
+[ ]   userId: {
+[ ]     type: String,
+[ ]     required: true,
+[-]     index: true
+      },
+[ ]   contributionId: {
+[ ]     type: String,
+[ ]     required: true,
+[-]     index: true
+      },
+[ ]   rated: {
+[ ]     type: String,
+[ ]     required: true,
+[ ]     enum: ['funny', 'happy', 'surprised', 'cry', 'angry']
+      },
+[ ]   createdAt: {
+[ ]     type: Date,
+[ ]     default: Date.now
+      },
+[ ]   updatedAt: {
+[ ]     type: Date,
+[ ]     default: Date.now
+      },
+[ ]   wasSeeded: { type: Boolean }
+    }
+*/
+
+CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as emotion;

deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh

@@ -8,18 +8,43 @@ set +o allexport
 
 # Delete collection function defintion
 function delete_collection () {
+  # Delete from Database
   echo "Delete $1"
   "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} < $(dirname "$0")/$1_delete.cql > /dev/null
+  # Delete index file
+  rm -f "${IMPORT_PATH}splits/$1.index"
 }
 
 # Import collection function defintion
 function import_collection () {
+  # index file of those chunks we have already imported
+  INDEX_FILE="${IMPORT_PATH}splits/$1.index"
+  # load index file
+  if [ -f "$INDEX_FILE" ]; then
+    readarray -t IMPORT_INDEX <$INDEX_FILE
+  else 
+     declare -a IMPORT_INDEX
+  fi
+  # for each chunk import data
   for chunk in ${IMPORT_PATH}splits/$1/*
   do
-    export IMPORT_CHUNK_PATH_CQL_FILE="${IMPORT_CHUNK_PATH_CQL}$1/$(basename "${chunk}")"
-    NEO4J_COMMAND="$(envsubst '${IMPORT_CHUNK_PATH_CQL_FILE}' < $(dirname "$0")/$1.cql)"
-    echo "Import ${chunk}"
-    echo "${NEO4J_COMMAND}" | "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} > /dev/null
+    CHUNK_FILE_NAME=$(basename "${chunk}")
+    # does the index not contain the chunk file name?
+    if [[ ! " ${IMPORT_INDEX[@]} " =~ " ${CHUNK_FILE_NAME} " ]]; then
+      # calculate the path of the chunk
+      export IMPORT_CHUNK_PATH_CQL_FILE="${IMPORT_CHUNK_PATH_CQL}$1/${CHUNK_FILE_NAME}"
+      # load the neo4j command and replace file variable with actual path
+      NEO4J_COMMAND="$(envsubst '${IMPORT_CHUNK_PATH_CQL_FILE}' < $(dirname "$0")/$1.cql)"
+      # run the import of the chunk
+      echo "Import $1 ${CHUNK_FILE_NAME} (${chunk})"
+      echo "${NEO4J_COMMAND}" | "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} > /dev/null
+      # add file to array and file
+      IMPORT_INDEX+=("${CHUNK_FILE_NAME}")
+      echo "${CHUNK_FILE_NAME}" >> ${INDEX_FILE}
+      echo "${IMPORT_INDEX[*]}"
+    else
+      echo "Skipping $1 ${CHUNK_FILE_NAME} (${chunk})"
+    fi
   done
 }
 
@@ -36,10 +61,11 @@ delete_collection "contributions"
 delete_collection "shouts"
 delete_collection "comments"
 
-#delete_collection "emotions"
-#delete_collection "invites"
-#delete_collection "notifications"
-#delete_collection "organizations"
+delete_collection "emotions"
+delete_collection "invites"
+delete_collection "notifications"
+delete_collection "organizations"
+
 #delete_collection "pages"
 #delete_collection "projects"
 #delete_collection "settings"
@@ -59,10 +85,11 @@ import_collection "contributions"
 import_collection "shouts"
 import_collection "comments"
 
-#import_collection "emotions"
-#import_collection "invites"
-#import_collection "notifications"
-#import_collection "organizations"
+import_collection "emotions"
+import_collection "invites"
+import_collection "notifications"
+import_collection "organizations"
+
 #import_collection "pages"
 #import_collection "projects"
 #import_collection "settings"

deployment/legacy-migration/maintenance-worker/migration/neo4j/invites.cql

@@ -0,0 +1,39 @@
+/*
+// Alpha Model
+// [ ] Not modeled in Nitro
+// [X] Modeled in Nitro
+// [-] Omitted in Nitro
+// [?] Unclear / has work to be done for Nitro
+    {
+[ ]   email: {
+[ ]     type: String,
+[ ]     required: true,
+[-]     index: true,
+[ ]     unique: true
+      },
+[ ]   code: {
+[ ]     type: String,
+[-]     index: true,
+[ ]     required: true
+      },
+[ ]   role: {
+[ ]     type: String,
+[ ]     enum: ['admin', 'moderator', 'manager', 'editor', 'user'],
+[ ]     default: 'user'
+      },
+[ ]   invitedByUserId: { type: String },
+[ ]   language: { type: String },
+[ ]   badgeIds: [],
+[ ]   wasUsed: {
+[ ]     type: Boolean,
+[-]     index: true
+      },
+[ ]   createdAt: {
+[ ]     type: Date,
+[ ]     default: Date.now
+      },
+[ ]   wasSeeded: { type: Boolean }
+  }
+*/
+
+CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as invite;

deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications.cql

@@ -0,0 +1,48 @@
+/*
+// Alpha Model
+// [ ] Not modeled in Nitro
+// [X] Modeled in Nitro
+// [-] Omitted in Nitro
+// [?] Unclear / has work to be done for Nitro
+    {
+[ ]   userId: { // User this notification is sent to
+[ ]     type: String,
+[ ]     required: true,
+[-]     index: true
+      },
+[ ]   type: {
+[ ]     type: String,
+[ ]     required: true,
+[ ]     enum: ['comment','comment-mention','contribution-mention','following-contribution']
+      },
+[ ]   relatedUserId: {
+[ ]     type: String,
+[-]     index: true
+      },
+[ ]   relatedContributionId: {
+[ ]     type: String,
+[-]     index: true
+      },
+[ ]   relatedOrganizationId: {
+[ ]     type: String,
+[-]     index: true
+      },
+[ ]   relatedCommentId: {type: String },
+[ ]   unseen: {
+[ ]     type: Boolean,
+[ ]     default: true,
+[-]     index: true
+      },
+[ ]   createdAt: {
+[ ]     type: Date,
+[ ]     default: Date.now
+      },
+[ ]   updatedAt: {
+[ ]     type: Date,
+[ ]     default: Date.now
+      },
+[ ]   wasSeeded: { type: Boolean }
+  }
+*/
+
+CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as notification;

deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations.cql

@@ -0,0 +1,137 @@
+/*
+// Alpha Model
+// [ ] Not modeled in Nitro
+// [X] Modeled in Nitro
+// [-] Omitted in Nitro
+// [?] Unclear / has work to be done for Nitro
+    {
+      name: {
+        type: String,
+        required: true,
+        index: true
+      },
+      slug: {
+        type: String,
+        required: true,
+        unique: true,
+        index: true
+      },
+      followersCounts: {
+        users: {
+          type: Number,
+          default: 0
+        },
+        organizations: {
+          type: Number,
+          default: 0
+        },
+        projects: {
+          type: Number,
+          default: 0
+        }
+      },
+      followingCounts: {
+        users: {
+          type: Number,
+          default: 0
+        },
+        organizations: {
+          type: Number,
+          default: 0
+        },
+        projects: {
+          type: Number,
+          default: 0
+        }
+      },
+      categoryIds: {
+        type: Array,
+        required: true,
+        index: true
+      },
+      logo: { type: String },
+      coverImg: { type: String },
+      userId: {
+        type: String,
+        required: true,
+        index: true
+      },
+      description: {
+        type: String,
+        required: true
+      },
+      descriptionExcerpt: { type: String }, // will be generated automatically
+      publicEmail: { type: String },
+      url: { type: String },
+      type: {
+        type: String,
+        index: true,
+        enum: ['ngo', 'npo', 'goodpurpose', 'ev', 'eva']
+      },
+      language: {
+        type: String,
+        required: true,
+        default: 'de',
+        index: true
+      },
+      addresses: {
+        type: [{
+          street: {
+            type: String,
+            required: true
+          },
+          zipCode: {
+            type: String,
+            required: true
+          },
+          city: {
+            type: String,
+            required: true
+          },
+          country: {
+            type: String,
+            required: true
+          },
+          lat: {
+            type: Number,
+            required: true
+          },
+          lng: {
+            type: Number,
+            required: true
+          }
+        }],
+        default: []
+      },
+      createdAt: {
+        type: Date,
+        default: Date.now
+      },
+      updatedAt: {
+        type: Date,
+        default: Date.now
+      },
+      isEnabled: {
+        type: Boolean,
+        default: false,
+        index: true
+      },
+      reviewedBy: {
+        type: String,
+        default: null,
+        index: true
+      },
+      tags: {
+        type: Array,
+        index: true
+      },
+      deleted: {
+        type: Boolean,
+        default: false,
+        index: true
+      },
+      wasSeeded: { type: Boolean }
+    }
+*/
+
+CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as organisation;