From 6f0447515a5a813bbb3e5880f4c8164d0601221e Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Wed, 29 May 2019 15:37:28 +0200 Subject: [PATCH 01/13] - fixed several errors handling import - split graphql schema into parts - which data is imported which is not - a list --- backend/package.json | 7 +- backend/src/graphql-schema.js | 11 +- backend/src/resolvers/badges.spec.js | 4 +- backend/src/seed/factories/badges.js | 4 +- backend/src/seed/factories/posts.js | 2 +- backend/src/seed/factories/users.js | 2 +- backend/src/types/enum/BadgeStatus.gql | 4 + backend/src/types/enum/BadgeType.gql | 4 + backend/src/types/enum/UserGroup.gql | 5 + backend/src/types/enum/Visibility.gql | 5 + backend/src/types/index.js | 30 ++++ backend/src/types/scalar/Date.gql_ | 1 + backend/src/types/scalar/DateTime.gql_ | 1 + backend/src/types/scalar/Time.gql_ | 1 + backend/src/types/scalar/Upload.gql | 1 + backend/src/types/schema.gql | 134 ++++++++++++++++++ .../schema_full.gql_} | 0 backend/src/types/type/Badge.gql | 13 ++ backend/src/types/type/Category.gql | 13 ++ backend/src/types/type/Comment.gql | 14 ++ backend/src/types/type/Post.gql | 43 ++++++ backend/src/types/type/Tag.gql | 10 ++ backend/src/types/type/User.gql | 80 +++++++++++ backend/yarn.lock | 14 ++ .../migration/neo4j/_delete_all.cql | 1 + .../migration/neo4j/badges.cql | 44 +++++- .../migration/neo4j/categories.cql | 34 ++++- .../migration/neo4j/comments.cql | 70 +++++++-- .../migration/neo4j/contributions.cql | 130 ++++++++++++++++- .../migration/neo4j/follows.cql | 34 ++++- .../migration/neo4j/import.sh | 10 +- .../migration/neo4j/shouts.cql | 34 ++++- .../migration/neo4j/users.cql | 100 ++++++++++++- 33 files changed, 825 insertions(+), 35 deletions(-) create mode 100644 backend/src/types/enum/BadgeStatus.gql create mode 100644 backend/src/types/enum/BadgeType.gql create mode 100644 backend/src/types/enum/UserGroup.gql create mode 100644 backend/src/types/enum/Visibility.gql create mode 100644 backend/src/types/index.js create mode 100644 backend/src/types/scalar/Date.gql_ create mode 100644 backend/src/types/scalar/DateTime.gql_ create mode 100644 backend/src/types/scalar/Time.gql_ create mode 100644 backend/src/types/scalar/Upload.gql create mode 100644 backend/src/types/schema.gql rename backend/src/{schema.graphql => types/schema_full.gql_} (100%) create mode 100644 backend/src/types/type/Badge.gql create mode 100644 backend/src/types/type/Category.gql create mode 100644 backend/src/types/type/Comment.gql create mode 100644 backend/src/types/type/Post.gql create mode 100644 backend/src/types/type/Tag.gql create mode 100644 backend/src/types/type/User.gql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/_delete_all.cql diff --git a/backend/package.json b/backend/package.json index 9f2bbe7ab..4006a5bdb 100644 --- a/backend/package.json +++ b/backend/package.json @@ -6,8 +6,8 @@ "scripts": { "build": "babel src/ -d dist/ --copy-files", "start": "node dist/", - "dev": "nodemon --exec babel-node src/ -e js,graphql", - "dev:debug": "nodemon --exec babel-node --inspect=0.0.0.0:9229 src/index.js -e js,graphql", + "dev": "nodemon --exec babel-node src/ -e js,gql", + "dev:debug": "nodemon --exec babel-node --inspect=0.0.0.0:9229 src/index.js -e js,gql", "lint": "eslint src --config .eslintrc.js", "test": "run-s test:jest test:cucumber", "test:before:server": "cross-env GRAPHQL_URI=http://localhost:4123 GRAPHQL_PORT=4123 yarn run dev 2> /dev/null", @@ -68,6 +68,7 @@ "jsonwebtoken": "~8.5.1", "linkifyjs": "~2.1.8", "lodash": "~4.17.11", + "merge-graphql-schemas": "^1.5.8", "ms": "~2.1.1", "neo4j-driver": "~1.7.4", "neo4j-graphql-js": "git+https://github.com/Human-Connection/neo4j-graphql-js.git#temporary_fixes", @@ -94,8 +95,8 @@ "chai": "~4.2.0", "cucumber": "~5.1.0", "eslint": "~5.16.0", - "eslint-config-standard": "~12.0.0", "eslint-config-prettier": "~4.3.0", + "eslint-config-standard": "~12.0.0", "eslint-plugin-import": "~2.17.3", "eslint-plugin-jest": "~22.6.4", "eslint-plugin-node": "~9.1.0", diff --git a/backend/src/graphql-schema.js b/backend/src/graphql-schema.js index a846541e7..c0b06af9c 100644 --- a/backend/src/graphql-schema.js +++ b/backend/src/graphql-schema.js @@ -1,6 +1,4 @@ -import fs from 'fs' -import path from 'path' - +// resolvers import userManagement from './resolvers/user_management.js' import statistics from './resolvers/statistics.js' import reports from './resolvers/reports.js' @@ -14,9 +12,10 @@ import notifications from './resolvers/notifications' import comments from './resolvers/comments' import users from './resolvers/users' -export const typeDefs = fs - .readFileSync(process.env.GRAPHQL_SCHEMA || path.join(__dirname, 'schema.graphql')) - .toString('utf-8') +// types +import types from './types' + +export const typeDefs = types export const resolvers = { Query: { diff --git a/backend/src/resolvers/badges.spec.js b/backend/src/resolvers/badges.spec.js index f5e15f4a2..8383d9838 100644 --- a/backend/src/resolvers/badges.spec.js +++ b/backend/src/resolvers/badges.spec.js @@ -41,8 +41,8 @@ describe('badges', () => { mutation( $id: ID $key: String! - $type: BadgeTypeEnum! - $status: BadgeStatusEnum! + $type: BadgeType! + $status: BadgeStatus! $icon: String! ) { CreateBadge(id: $id, key: $key, type: $type, status: $status, icon: $icon) { diff --git a/backend/src/seed/factories/badges.js b/backend/src/seed/factories/badges.js index 52fa9da33..6414e9f36 100644 --- a/backend/src/seed/factories/badges.js +++ b/backend/src/seed/factories/badges.js @@ -14,8 +14,8 @@ export default function(params) { mutation( $id: ID $key: String! - $type: BadgeTypeEnum! - $status: BadgeStatusEnum! + $type: BadgeType! + $status: BadgeStatus! $icon: String! ) { CreateBadge(id: $id, key: $key, type: $type, status: $status, icon: $icon) { diff --git a/backend/src/seed/factories/posts.js b/backend/src/seed/factories/posts.js index 1468b05d7..ea92f7d9f 100644 --- a/backend/src/seed/factories/posts.js +++ b/backend/src/seed/factories/posts.js @@ -26,7 +26,7 @@ export default function(params) { $title: String! $content: String! $image: String - $visibility: VisibilityEnum + $visibility: Visibility $deleted: Boolean ) { CreatePost( diff --git a/backend/src/seed/factories/users.js b/backend/src/seed/factories/users.js index 1ab362ce2..ca17d1721 100644 --- a/backend/src/seed/factories/users.js +++ b/backend/src/seed/factories/users.js @@ -23,7 +23,7 @@ export default function create(params) { $email: String! $avatar: String $about: String - $role: UserGroupEnum + $role: UserGroup ) { CreateUser( id: $id diff --git a/backend/src/types/enum/BadgeStatus.gql b/backend/src/types/enum/BadgeStatus.gql new file mode 100644 index 000000000..b109663b3 --- /dev/null +++ b/backend/src/types/enum/BadgeStatus.gql @@ -0,0 +1,4 @@ +enum BadgeStatus { + permanent + temporary +} \ No newline at end of file diff --git a/backend/src/types/enum/BadgeType.gql b/backend/src/types/enum/BadgeType.gql new file mode 100644 index 000000000..eccf2e661 --- /dev/null +++ b/backend/src/types/enum/BadgeType.gql @@ -0,0 +1,4 @@ +enum BadgeType { + role + crowdfunding +} \ No newline at end of file diff --git a/backend/src/types/enum/UserGroup.gql b/backend/src/types/enum/UserGroup.gql new file mode 100644 index 000000000..af25bcc69 --- /dev/null +++ b/backend/src/types/enum/UserGroup.gql @@ -0,0 +1,5 @@ +enum UserGroup { + admin + moderator + user +} \ No newline at end of file diff --git a/backend/src/types/enum/Visibility.gql b/backend/src/types/enum/Visibility.gql new file mode 100644 index 000000000..4f9d5591a --- /dev/null +++ b/backend/src/types/enum/Visibility.gql @@ -0,0 +1,5 @@ +enum Visibility { + public + friends + private +} \ No newline at end of file diff --git a/backend/src/types/index.js b/backend/src/types/index.js new file mode 100644 index 000000000..ca67aadbd --- /dev/null +++ b/backend/src/types/index.js @@ -0,0 +1,30 @@ +import fs from 'fs' +import path from 'path' +import { mergeTypes } from 'merge-graphql-schemas' + +const findGqlFiles = dir => { + var results = [] + var list = fs.readdirSync(dir) + list.forEach(file => { + file = path.join(dir, file).toString('utf-8') + var stat = fs.statSync(file) + if (stat && stat.isDirectory()) { + // Recurse into a subdirectory + results = results.concat(findGqlFiles(file)) + } else { + if (file.split('.').pop() === 'gql') { + // Is a gql file + results.push(file) + } + } + }) + return results +} + +let typeDefs = [] + +findGqlFiles(__dirname).forEach(file => { + typeDefs.push(fs.readFileSync(file).toString('utf-8')) +}) + +export default mergeTypes(typeDefs, { all: true }) diff --git a/backend/src/types/scalar/Date.gql_ b/backend/src/types/scalar/Date.gql_ new file mode 100644 index 000000000..7b0004ea3 --- /dev/null +++ b/backend/src/types/scalar/Date.gql_ @@ -0,0 +1 @@ +scalar Date \ No newline at end of file diff --git a/backend/src/types/scalar/DateTime.gql_ b/backend/src/types/scalar/DateTime.gql_ new file mode 100644 index 000000000..af973932f --- /dev/null +++ b/backend/src/types/scalar/DateTime.gql_ @@ -0,0 +1 @@ +scalar DateTime \ No newline at end of file diff --git a/backend/src/types/scalar/Time.gql_ b/backend/src/types/scalar/Time.gql_ new file mode 100644 index 000000000..53becdd66 --- /dev/null +++ b/backend/src/types/scalar/Time.gql_ @@ -0,0 +1 @@ +scalar Time \ No newline at end of file diff --git a/backend/src/types/scalar/Upload.gql b/backend/src/types/scalar/Upload.gql new file mode 100644 index 000000000..fca9ea1fc --- /dev/null +++ b/backend/src/types/scalar/Upload.gql @@ -0,0 +1 @@ +scalar Upload \ No newline at end of file diff --git a/backend/src/types/schema.gql b/backend/src/types/schema.gql new file mode 100644 index 000000000..ab8b25399 --- /dev/null +++ b/backend/src/types/schema.gql @@ -0,0 +1,134 @@ +type Query { + isLoggedIn: Boolean! + # Get the currently logged in User based on the given JWT Token + currentUser: User + # Get the latest Network Statistics + statistics: Statistics! + findPosts(filter: String!, limit: Int = 10): [Post]! @cypher( + statement: """ + CALL db.index.fulltext.queryNodes('full_text_search', $filter) + YIELD node as post, score + MATCH (post)<-[:WROTE]-(user:User) + WHERE score >= 0.2 + AND NOT user.deleted = true AND NOT user.disabled = true + AND NOT post.deleted = true AND NOT post.disabled = true + RETURN post + LIMIT $limit + """ + ) + CommentByPost(postId: ID!): [Comment]! +} + +type Mutation { + # Get a JWT Token for the given Email and password + login(email: String!, password: String!): String! + signup(email: String!, password: String!): Boolean! + changePassword(oldPassword:String!, newPassword: String!): String! + report(id: ID!, description: String): Report + disable(id: ID!): ID + enable(id: ID!): ID + reward(fromBadgeId: ID!, toUserId: ID!): ID + unreward(fromBadgeId: ID!, toUserId: ID!): ID + # Shout the given Type and ID + shout(id: ID!, type: ShoutTypeEnum): Boolean! + # Unshout the given Type and ID + unshout(id: ID!, type: ShoutTypeEnum): Boolean! + # Follow the given Type and ID + follow(id: ID!, type: FollowTypeEnum): Boolean! + # Unfollow the given Type and ID + unfollow(id: ID!, type: FollowTypeEnum): Boolean! +} + +type Statistics { + countUsers: Int! + countPosts: Int! + countComments: Int! + countNotifications: Int! + countOrganizations: Int! + countProjects: Int! + countInvites: Int! + countFollows: Int! + countShouts: Int! +} + +type Notification { + id: ID! + read: Boolean, + user: User @relation(name: "NOTIFIED", direction: "OUT") + post: Post @relation(name: "NOTIFIED", direction: "IN") + createdAt: String +} + +type Location { + id: ID! + name: String! + nameEN: String + nameDE: String + nameFR: String + nameNL: String + nameIT: String + nameES: String + namePT: String + namePL: String + type: String! + lat: Float + lng: Float + parent: Location @cypher(statement: "MATCH (this)-[:IS_IN]->(l:Location) RETURN l") +} + +type Report { + id: ID! + submitter: User @relation(name: "REPORTED", direction: "IN") + description: String + type: String! @cypher(statement: "MATCH (resource)<-[:REPORTED]-(this) RETURN labels(resource)[0]") + createdAt: String + comment: Comment @relation(name: "REPORTED", direction: "OUT") + post: Post @relation(name: "REPORTED", direction: "OUT") + user: User @relation(name: "REPORTED", direction: "OUT") +} + +enum ShoutTypeEnum { + Post + Organization + Project +} +enum FollowTypeEnum { + User + Organization + Project +} + +type Reward { + id: ID! + user: User @relation(name: "REWARDED", direction: "IN") + rewarderId: ID + createdAt: String + badge: Badge @relation(name: "REWARDED", direction: "OUT") +} + +type Organization { + id: ID! + createdBy: User @relation(name: "CREATED_ORGA", direction: "IN") + ownedBy: [User] @relation(name: "OWNING_ORGA", direction: "IN") + name: String! + slug: String + description: String! + descriptionExcerpt: String + deleted: Boolean + disabled: Boolean + + tags: [Tag]! @relation(name: "TAGGED", direction: "OUT") + categories: [Category]! @relation(name: "CATEGORIZED", direction: "OUT") +} + +type SharedInboxEndpoint { + id: ID! + uri: String +} + +type SocialMedia { + id: ID! + url: String + ownedBy: [User]! @relation(name: "OWNED", direction: "IN") +} + diff --git a/backend/src/schema.graphql b/backend/src/types/schema_full.gql_ similarity index 100% rename from backend/src/schema.graphql rename to backend/src/types/schema_full.gql_ diff --git a/backend/src/types/type/Badge.gql b/backend/src/types/type/Badge.gql new file mode 100644 index 000000000..68c5d5707 --- /dev/null +++ b/backend/src/types/type/Badge.gql @@ -0,0 +1,13 @@ +type Badge { + id: ID! + key: String! + type: BadgeType! + status: BadgeStatus! + icon: String! + #createdAt: DateTime + #updatedAt: DateTime + createdAt: String + updatedAt: String + + rewarded: [User]! @relation(name: "REWARDED", direction: "OUT") +} \ No newline at end of file diff --git a/backend/src/types/type/Category.gql b/backend/src/types/type/Category.gql new file mode 100644 index 000000000..d5e089a6f --- /dev/null +++ b/backend/src/types/type/Category.gql @@ -0,0 +1,13 @@ +type Category { + id: ID! + name: String! + slug: String! + icon: String! + #createdAt: DateTime + #updatedAt: DateTime + createdAt: String + updatedAt: String + + posts: [Post]! @relation(name: "CATEGORIZED", direction: "IN") + postCount: Int! @cypher(statement: "MATCH (this)<-[:CATEGORIZED]-(r:Post) RETURN COUNT(r)") +} \ No newline at end of file diff --git a/backend/src/types/type/Comment.gql b/backend/src/types/type/Comment.gql new file mode 100644 index 000000000..077366e8a --- /dev/null +++ b/backend/src/types/type/Comment.gql @@ -0,0 +1,14 @@ +type Comment { + id: ID! + activityId: String + postId: ID + author: User @relation(name: "WROTE", direction: "IN") + content: String! + contentExcerpt: String + post: Post @relation(name: "COMMENTS", direction: "OUT") + createdAt: String + updatedAt: String + deleted: Boolean + disabled: Boolean + disabledBy: User @relation(name: "DISABLED", direction: "IN") +} \ No newline at end of file diff --git a/backend/src/types/type/Post.gql b/backend/src/types/type/Post.gql new file mode 100644 index 000000000..6e23862ed --- /dev/null +++ b/backend/src/types/type/Post.gql @@ -0,0 +1,43 @@ +type Post { + id: ID! + activityId: String + objectId: String + author: User @relation(name: "WROTE", direction: "IN") + title: String! + slug: String + content: String! + contentExcerpt: String + image: String + imageUpload: Upload + visibility: Visibility + deleted: Boolean + disabled: Boolean + disabledBy: User @relation(name: "DISABLED", direction: "IN") + createdAt: String + updatedAt: String + + relatedContributions: [Post]! @cypher( + statement: """ + MATCH (this)-[:TAGGED|CATEGORIZED]->(categoryOrTag)<-[:TAGGED|CATEGORIZED]-(post:Post) + RETURN DISTINCT post + LIMIT 10 + """ + ) + + tags: [Tag]! @relation(name: "TAGGED", direction: "OUT") + categories: [Category]! @relation(name: "CATEGORIZED", direction: "OUT") + + comments: [Comment]! @relation(name: "COMMENTS", direction: "IN") + commentsCount: Int! @cypher(statement: "MATCH (this)<-[:COMMENTS]-(r:Comment) WHERE NOT r.deleted = true AND NOT r.disabled = true RETURN COUNT(r)") + + shoutedBy: [User]! @relation(name: "SHOUTED", direction: "IN") + shoutedCount: Int! @cypher(statement: "MATCH (this)<-[:SHOUTED]-(r:User) WHERE NOT r.deleted = true AND NOT r.disabled = true RETURN COUNT(DISTINCT r)") + + # Has the currently logged in user shouted that post? + shoutedByCurrentUser: Boolean! @cypher( + statement: """ + MATCH (this)<-[:SHOUTED]-(u:User {id: $cypherParams.currentUserId}) + RETURN COUNT(u) >= 1 + """ + ) +} \ No newline at end of file diff --git a/backend/src/types/type/Tag.gql b/backend/src/types/type/Tag.gql new file mode 100644 index 000000000..ecbd0b46a --- /dev/null +++ b/backend/src/types/type/Tag.gql @@ -0,0 +1,10 @@ +type Tag { + id: ID! + name: String! + taggedPosts: [Post]! @relation(name: "TAGGED", direction: "IN") + taggedOrganizations: [Organization]! @relation(name: "TAGGED", direction: "IN") + taggedCount: Int! @cypher(statement: "MATCH (this)<-[:TAGGED]-(p) RETURN COUNT(DISTINCT p)") + taggedCountUnique: Int! @cypher(statement: "MATCH (this)<-[:TAGGED]-(p)<-[:WROTE]-(u:User) RETURN COUNT(DISTINCT u)") + deleted: Boolean + disabled: Boolean +} \ No newline at end of file diff --git a/backend/src/types/type/User.gql b/backend/src/types/type/User.gql new file mode 100644 index 000000000..1287aa45f --- /dev/null +++ b/backend/src/types/type/User.gql @@ -0,0 +1,80 @@ +type User { + id: ID! + actorId: String + name: String + email: String! + slug: String + password: String! + avatar: String + coverImg: String + avatarUpload: Upload + deleted: Boolean + disabled: Boolean + disabledBy: User @relation(name: "DISABLED", direction: "IN") + role: UserGroup + publicKey: String + privateKey: String + + wasInvited: Boolean + wasSeeded: Boolean + + location: Location @cypher(statement: "MATCH (this)-[:IS_IN]->(l:Location) RETURN l") + locationName: String + about: String + socialMedia: [SocialMedia]! @relation(name: "OWNED", direction: "OUT") + + #createdAt: DateTime + #updatedAt: DateTime + createdAt: String + updatedAt: String + + notifications(read: Boolean): [Notification]! @relation(name: "NOTIFIED", direction: "IN") + + friends: [User]! @relation(name: "FRIENDS", direction: "BOTH") + friendsCount: Int! @cypher(statement: "MATCH (this)<-[:FRIENDS]->(r:User) RETURN COUNT(DISTINCT r)") + + following: [User]! @relation(name: "FOLLOWS", direction: "OUT") + followingCount: Int! @cypher(statement: "MATCH (this)-[:FOLLOWS]->(r:User) RETURN COUNT(DISTINCT r)") + + followedBy: [User]! @relation(name: "FOLLOWS", direction: "IN") + followedByCount: Int! @cypher(statement: "MATCH (this)<-[:FOLLOWS]-(r:User) RETURN COUNT(DISTINCT r)") + + # Is the currently logged in user following that user? + followedByCurrentUser: Boolean! @cypher( + statement: """ + MATCH (this)<-[:FOLLOWS]-(u:User {id: $cypherParams.currentUserId}) + RETURN COUNT(u) >= 1 + """ + ) + + #contributions: [WrittenPost]! + #contributions2(first: Int = 10, offset: Int = 0): [WrittenPost2]! + # @cypher( + # statement: "MATCH (this)-[w:WROTE]->(p:Post) RETURN p as Post, w.timestamp as timestamp" + # ) + contributions: [Post]! @relation(name: "WROTE", direction: "OUT") + contributionsCount: Int! @cypher( + statement: """ + MATCH (this)-[:WROTE]->(r:Post) + WHERE (NOT exists(r.deleted) OR r.deleted = false) + AND (NOT exists(r.disabled) OR r.disabled = false) + RETURN COUNT(r) + """ + ) + + comments: [Comment]! @relation(name: "WROTE", direction: "OUT") + commentsCount: Int! @cypher(statement: "MATCH (this)-[:WROTE]->(r:Comment) WHERE NOT r.deleted = true AND NOT r.disabled = true RETURN COUNT(r)") + + shouted: [Post]! @relation(name: "SHOUTED", direction: "OUT") + shoutedCount: Int! @cypher(statement: "MATCH (this)-[:SHOUTED]->(r:Post) WHERE NOT r.deleted = true AND NOT r.disabled = true RETURN COUNT(DISTINCT r)") + + organizationsCreated: [Organization] @relation(name: "CREATED_ORGA", direction: "OUT") + organizationsOwned: [Organization] @relation(name: "OWNING_ORGA", direction: "OUT") + + blacklisted: [User]! @relation(name: "BLACKLISTED", direction: "OUT") + + categories: [Category]! @relation(name: "CATEGORIZED", direction: "OUT") + + badges: [Badge]! @relation(name: "REWARDED", direction: "IN") + badgesCount: Int! @cypher(statement: "MATCH (this)<-[:REWARDED]-(r:Badge) RETURN COUNT(r)") +} \ No newline at end of file diff --git a/backend/yarn.lock b/backend/yarn.lock index 176af3893..534d6eef7 100644 --- a/backend/yarn.lock +++ b/backend/yarn.lock @@ -2656,6 +2656,11 @@ deep-is@~0.1.3: resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" integrity sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ= +deepmerge@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-2.2.1.tgz#5d3ff22a01c00f645405a2fbc17d0778a1801170" + integrity sha512-R9hc1Xa/NOBi9WRVUWg19rl1UB7Tt4kuPd+thNJgFZoxXsTz7ncaPaeIm+40oSGuP33DfMb4sZt1QIGiJzC4EA== + define-properties@^1.1.2: version "1.1.3" resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.3.tgz#cf88da6cbee26fe6db7094f61d870cbd84cee9f1" @@ -5388,6 +5393,15 @@ merge-descriptors@1.0.1: resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" integrity sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E= +merge-graphql-schemas@^1.5.8: + version "1.5.8" + resolved "https://registry.yarnpkg.com/merge-graphql-schemas/-/merge-graphql-schemas-1.5.8.tgz#89457b60312aabead44d5b2b7625643f8ab9e369" + integrity sha512-0TGOKebltvmWR9h9dPYS2vAqMPThXwJ6gVz7O5MtpBp2sunAg/M25iMSNI7YhU6PDJVtGtldTfqV9a+55YhB+A== + dependencies: + deepmerge "^2.2.1" + glob "^7.1.3" + is-glob "^4.0.0" + merge-stream@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-1.0.1.tgz#4041202d508a342ba00174008df0c251b8c135e1" diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/_delete_all.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/_delete_all.cql new file mode 100644 index 000000000..d01871300 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/_delete_all.cql @@ -0,0 +1 @@ +MATCH (n) DETACH DELETE n; \ No newline at end of file diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql index 62eddf124..62cd4a2cc 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql @@ -1,4 +1,46 @@ -CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as badge +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[?] image: { +[?] path: { // Path is incorrect in Nitro - is icon the correct name for this field? +[X] type: String, +[X] required: true + }, +[ ] alt: { // If we use an image - should we not have an alt? +[ ] type: String, +[ ] required: true + } + }, +[?] status: { +[X] type: String, +[X] enum: ['permanent', 'temporary'], +[ ] default: 'permanent', // Default value is missing in Nitro +[X] required: true + }, +[?] type: { +[?] type: String, // in nitro this is a defined enum - seems good for now +[X] required: true + }, +[X] key: { +[X] type: String, +[X] required: true + }, +[?] createdAt: { +[?] type: Date, // Type is modeled as string in Nitro which is incorrect +[ ] default: Date.now // Default value is missing in Nitro + }, +[?] updatedAt: { +[?] type: Date, // Type is modeled as string in Nitro which is incorrect +[ ] default: Date.now // Default value is missing in Nitro + } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as badge MERGE(b:Badge {id: badge._id["$oid"]}) ON CREATE SET b.key = badge.key, diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql index 81e73c1b9..2866c8aae 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql @@ -1,4 +1,35 @@ -CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as category +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[X] title: { +[X] type: String, +[X] required: true + }, +[?] slug: { +[X] type: String, +[X] required: true, +[ ] unique: true // Unique value is not enforced in Nitro? + }, +[?] icon: { // Nitro adds required: true +[X] type: String, +[ ] unique: true // Unique value is not enforced in Nitro? + }, +[?] createdAt: { +[?] type: Date, // Type is modeled as string in Nitro which is incorrect +[ ] default: Date.now // Default value is missing in Nitro + }, +[?] updatedAt: { +[?] type: Date, // Type is modeled as string in Nitro which is incorrect +[ ] default: Date.now // Default value is missing in Nitro + } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as category MERGE(c:Category {id: category._id["$oid"]}) ON CREATE SET c.name = category.title, @@ -8,6 +39,7 @@ c.createdAt = category.createdAt.`$date`, c.updatedAt = category.updatedAt.`$date` ; +// Transform icon names MATCH (c:Category) WHERE (c.icon = "categories-justforfun") SET c.icon = 'smile' diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql index 1c8eb9397..1cdc1bfc2 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql @@ -1,15 +1,65 @@ -CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as json +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[?] userId: { +[X] type: String, +[ ] required: true, // Not required in Nitro +[-] index: true + }, +[?] contributionId: { +[X] type: String, +[ ] required: true, // Not required in Nitro +[-] index: true + }, +[X] content: { +[X] type: String, +[X] required: true + }, +[?] contentExcerpt: { // Generated from content +[X] type: String, +[ ] required: true // Not required in Nitro + }, +[ ] hasMore: { type: Boolean }, +[ ] upvotes: { +[ ] type: Array, +[ ] default: [] + }, +[ ] upvoteCount: { +[ ] type: Number, +[ ] default: 0 + }, +[?] deleted: { +[X] type: Boolean, +[ ] default: false, // Default value is missing in Nitro +[-] index: true + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } +*/ -MERGE (comment:Comment {id: json._id["$oid"]}) +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as comment +MERGE (c:Comment {id: comment._id["$oid"]}) ON CREATE SET -comment.content = json.content, -comment.contentExcerpt = json.contentExcerpt, -comment.deleted = json.deleted, -comment.disabled = false -WITH comment, json, json.contributionId as postId +c.content = comment.content, +c.contentExcerpt = comment.contentExcerpt, +c.deleted = comment.deleted, +c.disabled = false +WITH c, comment, comment.contributionId as postId MATCH (post:Post {id: postId}) -WITH comment, post, json.userId as userId +WITH c, post, comment.userId as userId MATCH (author:User {id: userId}) -MERGE (comment)-[:COMMENTS]->(post) -MERGE (author)-[:WROTE]->(comment) +MERGE (c)-[:COMMENTS]->(post) +MERGE (author)-[:WROTE]->(c) ; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql index e4e148af3..98d8f24e9 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql @@ -1,4 +1,132 @@ -CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as post +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro +[?] { //Modeled incorrect as Post +[?] userId: { +[X] type: String, +[ ] required: true, // Not required in Nitro +[-] index: true + }, +[ ] organizationId: { +[ ] type: String, +[-] index: true + }, +[X] categoryIds: { +[X] type: Array, +[-] index: true + }, +[X] title: { +[X] type: String, +[X] required: true + }, +[?] slug: { // Generated from title +[X] type: String, +[ ] required: true, // Not required in Nitro +[?] unique: true, // Unique value is not enforced in Nitro? +[-] index: true + }, +[ ] type: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] cando: { +[ ] difficulty: { +[ ] type: String, +[ ] enum: ['easy', 'medium', 'hard'] + }, +[ ] reasonTitle: { type: String }, +[ ] reason: { type: String } + }, +[X] content: { +[X] type: String, +[X] required: true + }, +[?] contentExcerpt: { // Generated from content +[X] type: String, +[?] required: true // Not required in Nitro + }, +[ ] hasMore: { type: Boolean }, +[?] teaserImg: { type: String }, // Path is incorrect in Nitro +[ ] language: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] shoutCount: { +[ ] type: Number, +[ ] default: 0, +[-] index: true + }, +[ ] meta: { +[ ] hasVideo: { +[ ] type: Boolean, +[ ] default: false + }, +[ ] embedds: { +[ ] type: Object, +[ ] default: {} + } + }, +[?] visibility: { +[X] type: String, +[X] enum: ['public', 'friends', 'private'], +[ ] default: 'public', // Default value is missing in Nitro +[-] index: true + }, +[?] isEnabled: { +[X] type: Boolean, +[ ] default: true, // Default value is missing in Nitro +[-] index: true + }, +[?] tags: { type: Array }, // ensure this is working properly +[ ] emotions: { +[ ] type: Object, +[-] index: true, +[ ] default: { +[ ] angry: { +[ ] count: 0, +[ ] percent: 0 +[ ] }, +[ ] cry: { +[ ] count: 0, +[ ] percent: 0 +[ ] }, +[ ] surprised: { +[ ] count: 0, +[ ] percent: 0 + }, +[ ] happy: { +[ ] count: 0, +[ ] percent: 0 + }, +[ ] funny: { +[ ] count: 0, +[ ] percent: 0 + } + } + }, +[?] deleted: { +[X] type: Boolean, +[ ] default: false, // Default value is missing in Nitro +[-] index: true + }, +[?] createdAt: { +[?] type: Date, // Type is modeled as string in Nitro which is incorrect +[ ] default: Date.now // Default value is missing in Nitro + }, +[?] updatedAt: { +[?] type: Date, // Type is modeled as string in Nitro which is incorrect +[ ] default: Date.now // Default value is missing in Nitro + }, +[ ] wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as post MERGE (p:Post {id: post._id["$oid"]}) ON CREATE SET p.title = post.title, diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql index bf3837b04..fac858a9a 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql @@ -1,4 +1,36 @@ -CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as follow +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[?] userId: { +[-] type: String, +[ ] required: true, +[-] index: true + }, +[?] foreignId: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[?] foreignService: { // db.getCollection('follows').distinct('foreignService') returns 'organizations' and 'users' +[ ] type: String, +[ ] required: true, +[ ] index: true + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } + index: +[?] { userId: 1, foreignId: 1, foreignService: 1 },{ unique: true } // is the unique constrain modeled? +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as follow MATCH (u1:User {id: follow.userId}), (u2:User {id: follow.foreignId}) MERGE (u1)-[:FOLLOWS]->(u2) ; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh index 12cc7ce67..470158642 100755 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh @@ -10,10 +10,10 @@ set +o allexport function import_collection () { for chunk in ${IMPORT_PATH}splits/$1/* do - mv $chunk ${IMPORT_CHUNK_PATH} - NEO4J_COMMAND="$(envsubst '${IMPORT_CHUNK_PATH_CQL}' < $(dirname "$0")/$1.cql)" + export IMPORT_CHUNK_PATH_CQL_FILE="${IMPORT_CHUNK_PATH_CQL}$1/$(basename "${chunk}")" + NEO4J_COMMAND="$(envsubst '${IMPORT_CHUNK_PATH_CQL_FILE}' < $(dirname "$0")/$1.cql)" echo "Import ${chunk}" - echo "${NEO4J_COMMAND}" | "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} + echo "${NEO4J_COMMAND}" | "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} > /dev/null done } @@ -22,7 +22,8 @@ SECONDS=0 # Delete all Neo4J Database content echo "Deleting Database Contents" -echo "MATCH (n) DETACH DELETE n;" | "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} +"${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} < $(dirname "$0")/_delete_all.cql > /dev/null +echo "DONE" # Import Data echo "Start Importing Data" @@ -44,5 +45,6 @@ import_collection "comments" #import_collection "systemnotifications" #import_collection "userscandos" #import_collection "usersettings" +echo "DONE" echo "Time elapsed: $SECONDS seconds" diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql index a82a7a33d..d370b4b4a 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql @@ -1,4 +1,36 @@ -CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as shout +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[?] userId: { +[X] type: String, +[ ] required: true, // Not required in Nitro +[-] index: true + }, +[?] foreignId: { +[X] type: String, +[ ] required: true, // Not required in Nitro +[-] index: true + }, +[?] foreignService: { // db.getCollection('shots').distinct('foreignService') returns 'contributions' +[X] type: String, +[ ] required: true, // Not required in Nitro +[-] index: true + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } + index: +[?] { userId: 1, foreignId: 1 },{ unique: true } // is the unique constrain modeled? +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as shout MATCH (u:User {id: shout.userId}), (p:Post {id: shout.foreignId}) MERGE (u)-[:SHOUTED]->(p) ; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql index 693fd75b6..aec5499fc 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql @@ -1,4 +1,101 @@ -CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as user +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[?] email: { +[X] type: String, +[-] index: true, +[X] required: true, +[?] unique: true //unique constrain missing in Nitro + }, +[?] password: { // Not required in Alpha -> verify if always present +[X] type: String + }, +[X] name: { type: String }, +[X] slug: { +[X] type: String, +[-] index: true + }, +[ ] gender: { type: String }, +[ ] followersCounts: { +[ ] users: { +[ ] type: Number, +[ ] default: 0 + }, +[ ] organizations: { +[ ] type: Number, +[ ] default: 0 + }, +[ ] projects: { +[ ] type: Number, +[ ] default: 0 + } + }, +[ ] followingCounts: { +[ ] users: { +[ ] type: Number, +[ ] default: 0 + }, +[ ] organizations: { +[ ] type: Number, +[ ] default: 0 + }, +[ ] projects: { +[ ] type: Number, +[ ] default: 0 + } + }, +[ ] timezone: { type: String }, +[?] avatar: { type: String }, // Path is incorrect in Nitro +[?] coverImg: { type: String }, // Path is incorrect in Nitro, was not modeled in latest Nitro - do we want this? +[ ] doiToken: { type: String }, +[ ] confirmedAt: { type: Date }, +[?] badgeIds: [], // Verify this is working properly +[?] deletedAt: { type: Date }, // The Date of deletion is not saved in Nitro +[?] createdAt: { +[?] type: Date, // Modeled as String in Nitro +[ ] default: Date.now // Default value is missing in Nitro + }, +[?] updatedAt: { +[?] type: Date, // Modeled as String in Nitro +[ ] default: Date.now // Default value is missing in Nitro + }, +[ ] lastActiveAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] isVerified: { type: Boolean }, +[?] role: { +[X] type: String, +[-] index: true, +[?] enum: ['admin', 'moderator', 'manager', 'editor', 'user'], // missing roles manager & editor in Nitro +[ ] default: 'user' // Default value is missing in Nitro + }, +[ ] verifyToken: { type: String }, +[ ] verifyShortToken: { type: String }, +[ ] verifyExpires: { type: Date }, +[ ] verifyChanges: { type: Object }, +[ ] resetToken: { type: String }, +[ ] resetShortToken: { type: String }, +[ ] resetExpires: { type: Date }, +[X] wasSeeded: { type: Boolean }, +[X] wasInvited: { type: Boolean }, +[ ] language: { +[ ] type: String, +[ ] default: 'en' + }, +[ ] termsAndConditionsAccepted: { type: Date }, // we display the terms and conditions on registration +[ ] systemNotificationsSeen: { +[ ] type: Array, +[ ] default: [] + } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as user MERGE(u:User {id: user._id["$oid"]}) ON CREATE SET u.name = user.name, @@ -8,6 +105,7 @@ u.password = user.password, u.avatar = user.avatar, u.coverImg = user.coverImg, u.wasInvited = user.wasInvited, +u.wasSeeded = user.wasSeeded, u.role = toLower(user.role), u.createdAt = user.createdAt.`$date`, u.updatedAt = user.updatedAt.`$date`, From 7e44667dd142500cd0ac20c5259b5894f93fcacd Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Wed, 29 May 2019 15:45:02 +0200 Subject: [PATCH 02/13] modified .env file for neo4j import since its had problems with file locks on windows (cypher-shell) --- .../legacy-migration/maintenance-worker/migration/neo4j/.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/.env b/deployment/legacy-migration/maintenance-worker/migration/neo4j/.env index 7691306ac..c61bb0c15 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/.env +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/.env @@ -5,9 +5,9 @@ # Import Settings # On Windows this resolves to C:\Users\dornhoeschen\AppData\Local\Temp\mongo-export (MinGW) IMPORT_PATH='/tmp/mongo-export/' -IMPORT_CHUNK_PATH='/tmp/mongo-export/splits/current-chunk.json' +IMPORT_CHUNK_PATH='/tmp/mongo-export/splits/' -IMPORT_CHUNK_PATH_CQL='/tmp/mongo-export/splits/current-chunk.json' +IMPORT_CHUNK_PATH_CQL='/tmp/mongo-export/splits/' # On Windows this path needs to be windows style since the cypher-shell runs native - note the forward slash # IMPORT_CHUNK_PATH_CQL='C:/Users/dornhoeschen/AppData/Local/Temp/mongo-export/splits/current-chunk.json' From 67595f6400eaff082c6428b1219ef8337228955e Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Wed, 29 May 2019 15:45:32 +0200 Subject: [PATCH 03/13] missing change for windows comment --- .../legacy-migration/maintenance-worker/migration/neo4j/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/.env b/deployment/legacy-migration/maintenance-worker/migration/neo4j/.env index c61bb0c15..16220f3e6 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/.env +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/.env @@ -9,7 +9,7 @@ IMPORT_CHUNK_PATH='/tmp/mongo-export/splits/' IMPORT_CHUNK_PATH_CQL='/tmp/mongo-export/splits/' # On Windows this path needs to be windows style since the cypher-shell runs native - note the forward slash -# IMPORT_CHUNK_PATH_CQL='C:/Users/dornhoeschen/AppData/Local/Temp/mongo-export/splits/current-chunk.json' +# IMPORT_CHUNK_PATH_CQL='C:/Users/dornhoeschen/AppData/Local/Temp/mongo-export/splits/' IMPORT_CYPHERSHELL_BIN='cypher-shell' # On Windows use something like this From 816fbdf7cdd993c02c64c0d9eb7471c3a126c756 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Wed, 29 May 2019 16:41:27 +0200 Subject: [PATCH 04/13] fixed slug required - it was not in Nitro --- backend/src/types/type/Category.gql | 2 +- .../maintenance-worker/migration/neo4j/categories.cql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/types/type/Category.gql b/backend/src/types/type/Category.gql index d5e089a6f..5920ebbdb 100644 --- a/backend/src/types/type/Category.gql +++ b/backend/src/types/type/Category.gql @@ -1,7 +1,7 @@ type Category { id: ID! name: String! - slug: String! + slug: String icon: String! #createdAt: DateTime #updatedAt: DateTime diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql index 2866c8aae..0862fe0d9 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql @@ -11,7 +11,7 @@ }, [?] slug: { [X] type: String, -[X] required: true, +[ ] required: true, // Not required in Nitro [ ] unique: true // Unique value is not enforced in Nitro? }, [?] icon: { // Nitro adds required: true From 968578f10a3ad7e47b78e039aba6888b6f62d554 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Wed, 29 May 2019 17:30:46 +0200 Subject: [PATCH 05/13] delete cql files --- .../migration/neo4j/badges_delete.cql | 1 + .../migration/neo4j/categories_delete.cql | 1 + .../migration/neo4j/comments_delete.cql | 1 + .../migration/neo4j/contributions_delete.cql | 2 ++ .../migration/neo4j/follows_delete.cql | 1 + .../migration/neo4j/import.sh | 27 ++++++++++++++++++- .../migration/neo4j/shouts_delete.cql | 1 + .../migration/neo4j/users_delete.cql | 1 + 8 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/badges_delete.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/categories_delete.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/comments_delete.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions_delete.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/follows_delete.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts_delete.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/users_delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges_delete.cql new file mode 100644 index 000000000..2a6f8c244 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges_delete.cql @@ -0,0 +1 @@ +MATCH (n:Badge) DETACH DELETE n; \ No newline at end of file diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories_delete.cql new file mode 100644 index 000000000..c06b5ef2b --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories_delete.cql @@ -0,0 +1 @@ +MATCH (n:Category) DETACH DELETE n; \ No newline at end of file diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments_delete.cql new file mode 100644 index 000000000..c4a7961c5 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments_delete.cql @@ -0,0 +1 @@ +MATCH (n:Comment) DETACH DELETE n; \ No newline at end of file diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions_delete.cql new file mode 100644 index 000000000..70adad664 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions_delete.cql @@ -0,0 +1,2 @@ +MATCH (n:Post) DETACH DELETE n; +MATCH (n:Tag) DETACH DELETE n; \ No newline at end of file diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows_delete.cql new file mode 100644 index 000000000..3624448c3 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows_delete.cql @@ -0,0 +1 @@ +// this is just a relation between users(?) - no need to delete \ No newline at end of file diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh index 470158642..ecf4b8de4 100755 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh @@ -6,6 +6,12 @@ set -o allexport source $(dirname "$0")/.env set +o allexport +# Delete collection function defintion +function delete_collection () { + echo "Delete $1" + "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} < $(dirname "$0")/$1_delete.cql > /dev/null +} + # Import collection function defintion function import_collection () { for chunk in ${IMPORT_PATH}splits/$1/* @@ -22,7 +28,25 @@ SECONDS=0 # Delete all Neo4J Database content echo "Deleting Database Contents" -"${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} < $(dirname "$0")/_delete_all.cql > /dev/null +delete_collection "badges" +delete_collection "categories" +delete_collection "users" +delete_collection "follows" +delete_collection "contributions" +delete_collection "shouts" +delete_collection "comments" + +#delete_collection "emotions" +#delete_collection "invites" +#delete_collection "notifications" +#delete_collection "organizations" +#delete_collection "pages" +#delete_collection "projects" +#delete_collection "settings" +#delete_collection "status" +#delete_collection "systemnotifications" +#delete_collection "userscandos" +#delete_collection "usersettings" echo "DONE" # Import Data @@ -34,6 +58,7 @@ import_collection "follows" import_collection "contributions" import_collection "shouts" import_collection "comments" + #import_collection "emotions" #import_collection "invites" #import_collection "notifications" diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts_delete.cql new file mode 100644 index 000000000..21c2e1f90 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts_delete.cql @@ -0,0 +1 @@ +// this is just a relation between users and contributions - no need to delete \ No newline at end of file diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/users_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/users_delete.cql new file mode 100644 index 000000000..23935b3e0 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/users_delete.cql @@ -0,0 +1 @@ +MATCH (n:User) DETACH DELETE n; \ No newline at end of file From 271af7dde2fc58db321fd0a65b12a966b6999818 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Wed, 29 May 2019 20:02:30 +0200 Subject: [PATCH 06/13] keep index file for already imported files, do not reimport already imported file, invites, notifications, emotions and organisations scripts and descriptions --- .../migration/neo4j/emotions.cql | 35 +++++ .../migration/neo4j/emotions_delete.cql | 0 .../migration/neo4j/import.sh | 51 +++++-- .../migration/neo4j/invites.cql | 39 +++++ .../migration/neo4j/invites_delete.cql | 0 .../migration/neo4j/notifications.cql | 48 ++++++ .../migration/neo4j/notifications_delete.cql | 0 .../migration/neo4j/organizations.cql | 137 ++++++++++++++++++ .../migration/neo4j/organizations_delete.cql | 0 9 files changed, 298 insertions(+), 12 deletions(-) create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions_delete.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/invites.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/invites_delete.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications_delete.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations_delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions.cql new file mode 100644 index 000000000..8aad9e923 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions.cql @@ -0,0 +1,35 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] userId: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] contributionId: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] rated: { +[ ] type: String, +[ ] required: true, +[ ] enum: ['funny', 'happy', 'surprised', 'cry', 'angry'] + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as emotion; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/emotions_delete.cql new file mode 100644 index 000000000..e69de29bb diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh index ecf4b8de4..af2a6887f 100755 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh @@ -8,18 +8,43 @@ set +o allexport # Delete collection function defintion function delete_collection () { + # Delete from Database echo "Delete $1" "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} < $(dirname "$0")/$1_delete.cql > /dev/null + # Delete index file + rm -f "${IMPORT_PATH}splits/$1.index" } # Import collection function defintion function import_collection () { + # index file of those chunks we have already imported + INDEX_FILE="${IMPORT_PATH}splits/$1.index" + # load index file + if [ -f "$INDEX_FILE" ]; then + readarray -t IMPORT_INDEX <$INDEX_FILE + else + declare -a IMPORT_INDEX + fi + # for each chunk import data for chunk in ${IMPORT_PATH}splits/$1/* do - export IMPORT_CHUNK_PATH_CQL_FILE="${IMPORT_CHUNK_PATH_CQL}$1/$(basename "${chunk}")" - NEO4J_COMMAND="$(envsubst '${IMPORT_CHUNK_PATH_CQL_FILE}' < $(dirname "$0")/$1.cql)" - echo "Import ${chunk}" - echo "${NEO4J_COMMAND}" | "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} > /dev/null + CHUNK_FILE_NAME=$(basename "${chunk}") + # does the index not contain the chunk file name? + if [[ ! " ${IMPORT_INDEX[@]} " =~ " ${CHUNK_FILE_NAME} " ]]; then + # calculate the path of the chunk + export IMPORT_CHUNK_PATH_CQL_FILE="${IMPORT_CHUNK_PATH_CQL}$1/${CHUNK_FILE_NAME}" + # load the neo4j command and replace file variable with actual path + NEO4J_COMMAND="$(envsubst '${IMPORT_CHUNK_PATH_CQL_FILE}' < $(dirname "$0")/$1.cql)" + # run the import of the chunk + echo "Import $1 ${CHUNK_FILE_NAME} (${chunk})" + echo "${NEO4J_COMMAND}" | "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} > /dev/null + # add file to array and file + IMPORT_INDEX+=("${CHUNK_FILE_NAME}") + echo "${CHUNK_FILE_NAME}" >> ${INDEX_FILE} + echo "${IMPORT_INDEX[*]}" + else + echo "Skipping $1 ${CHUNK_FILE_NAME} (${chunk})" + fi done } @@ -36,10 +61,11 @@ delete_collection "contributions" delete_collection "shouts" delete_collection "comments" -#delete_collection "emotions" -#delete_collection "invites" -#delete_collection "notifications" -#delete_collection "organizations" +delete_collection "emotions" +delete_collection "invites" +delete_collection "notifications" +delete_collection "organizations" + #delete_collection "pages" #delete_collection "projects" #delete_collection "settings" @@ -59,10 +85,11 @@ import_collection "contributions" import_collection "shouts" import_collection "comments" -#import_collection "emotions" -#import_collection "invites" -#import_collection "notifications" -#import_collection "organizations" +import_collection "emotions" +import_collection "invites" +import_collection "notifications" +import_collection "organizations" + #import_collection "pages" #import_collection "projects" #import_collection "settings" diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/invites.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/invites.cql new file mode 100644 index 000000000..f4a5bf006 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/invites.cql @@ -0,0 +1,39 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] email: { +[ ] type: String, +[ ] required: true, +[-] index: true, +[ ] unique: true + }, +[ ] code: { +[ ] type: String, +[-] index: true, +[ ] required: true + }, +[ ] role: { +[ ] type: String, +[ ] enum: ['admin', 'moderator', 'manager', 'editor', 'user'], +[ ] default: 'user' + }, +[ ] invitedByUserId: { type: String }, +[ ] language: { type: String }, +[ ] badgeIds: [], +[ ] wasUsed: { +[ ] type: Boolean, +[-] index: true + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as invite; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/invites_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/invites_delete.cql new file mode 100644 index 000000000..e69de29bb diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications.cql new file mode 100644 index 000000000..aa6ac8eb9 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications.cql @@ -0,0 +1,48 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] userId: { // User this notification is sent to +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] type: { +[ ] type: String, +[ ] required: true, +[ ] enum: ['comment','comment-mention','contribution-mention','following-contribution'] + }, +[ ] relatedUserId: { +[ ] type: String, +[-] index: true + }, +[ ] relatedContributionId: { +[ ] type: String, +[-] index: true + }, +[ ] relatedOrganizationId: { +[ ] type: String, +[-] index: true + }, +[ ] relatedCommentId: {type: String }, +[ ] unseen: { +[ ] type: Boolean, +[ ] default: true, +[-] index: true + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as notification; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/notifications_delete.cql new file mode 100644 index 000000000..e69de29bb diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations.cql new file mode 100644 index 000000000..50e8f6765 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations.cql @@ -0,0 +1,137 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { + name: { + type: String, + required: true, + index: true + }, + slug: { + type: String, + required: true, + unique: true, + index: true + }, + followersCounts: { + users: { + type: Number, + default: 0 + }, + organizations: { + type: Number, + default: 0 + }, + projects: { + type: Number, + default: 0 + } + }, + followingCounts: { + users: { + type: Number, + default: 0 + }, + organizations: { + type: Number, + default: 0 + }, + projects: { + type: Number, + default: 0 + } + }, + categoryIds: { + type: Array, + required: true, + index: true + }, + logo: { type: String }, + coverImg: { type: String }, + userId: { + type: String, + required: true, + index: true + }, + description: { + type: String, + required: true + }, + descriptionExcerpt: { type: String }, // will be generated automatically + publicEmail: { type: String }, + url: { type: String }, + type: { + type: String, + index: true, + enum: ['ngo', 'npo', 'goodpurpose', 'ev', 'eva'] + }, + language: { + type: String, + required: true, + default: 'de', + index: true + }, + addresses: { + type: [{ + street: { + type: String, + required: true + }, + zipCode: { + type: String, + required: true + }, + city: { + type: String, + required: true + }, + country: { + type: String, + required: true + }, + lat: { + type: Number, + required: true + }, + lng: { + type: Number, + required: true + } + }], + default: [] + }, + createdAt: { + type: Date, + default: Date.now + }, + updatedAt: { + type: Date, + default: Date.now + }, + isEnabled: { + type: Boolean, + default: false, + index: true + }, + reviewedBy: { + type: String, + default: null, + index: true + }, + tags: { + type: Array, + index: true + }, + deleted: { + type: Boolean, + default: false, + index: true + }, + wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as organisation; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations_delete.cql new file mode 100644 index 000000000..e69de29bb From 355205028f7ac591bd92bc47294b98c2cea0b203 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Wed, 29 May 2019 20:04:03 +0200 Subject: [PATCH 07/13] removed debug output --- .../maintenance-worker/migration/neo4j/import.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh index af2a6887f..38db82e4a 100755 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh @@ -41,7 +41,6 @@ function import_collection () { # add file to array and file IMPORT_INDEX+=("${CHUNK_FILE_NAME}") echo "${CHUNK_FILE_NAME}" >> ${INDEX_FILE} - echo "${IMPORT_INDEX[*]}" else echo "Skipping $1 ${CHUNK_FILE_NAME} (${chunk})" fi From 37be5481c041299f2a678b6d9f446501dca70fca Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Wed, 29 May 2019 20:43:58 +0200 Subject: [PATCH 08/13] remaining legacy table descriptions and dummy import scripts --- .../migration/neo4j/import.sh | 23 +-- .../migration/neo4j/organizations.cql | 194 +++++++++--------- .../migration/neo4j/pages.cql | 55 +++++ .../migration/neo4j/pages_delete.cql | 0 .../migration/neo4j/projects.cql | 44 ++++ .../migration/neo4j/projects_delete.cql | 0 .../migration/neo4j/settings.cql | 36 ++++ .../migration/neo4j/settings_delete.cql | 0 .../migration/neo4j/status.cql | 19 ++ .../migration/neo4j/status_delete.cql | 0 .../migration/neo4j/systemnotifications.cql | 61 ++++++ .../neo4j/systemnotifications_delete.cql | 0 .../migration/neo4j/userscandos.cql | 35 ++++ .../migration/neo4j/userscandos_delete.cql | 0 .../migration/neo4j/usersettings.cql | 43 ++++ .../migration/neo4j/usersettings_delete.cql | 0 16 files changed, 401 insertions(+), 109 deletions(-) create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/pages.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/pages_delete.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/projects.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/projects_delete.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/settings.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/settings_delete.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/status.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/status_delete.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications_delete.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/userscandos.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/userscandos_delete.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/usersettings.cql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/usersettings_delete.cql diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh index 38db82e4a..5720baaf0 100755 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh @@ -60,11 +60,10 @@ delete_collection "contributions" delete_collection "shouts" delete_collection "comments" -delete_collection "emotions" -delete_collection "invites" -delete_collection "notifications" -delete_collection "organizations" - +#delete_collection "emotions" +#delete_collection "invites" +#delete_collection "notifications" +#delete_collection "organizations" #delete_collection "pages" #delete_collection "projects" #delete_collection "settings" @@ -88,14 +87,14 @@ import_collection "emotions" import_collection "invites" import_collection "notifications" import_collection "organizations" +import_collection "pages" +import_collection "projects" +import_collection "settings" +import_collection "status" +import_collection "systemnotifications" +import_collection "userscandos" +import_collection "usersettings" -#import_collection "pages" -#import_collection "projects" -#import_collection "settings" -#import_collection "status" -#import_collection "systemnotifications" -#import_collection "userscandos" -#import_collection "usersettings" echo "DONE" echo "Time elapsed: $SECONDS seconds" diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations.cql index 50e8f6765..e473e697c 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/organizations.cql @@ -5,132 +5,132 @@ // [-] Omitted in Nitro // [?] Unclear / has work to be done for Nitro { - name: { - type: String, - required: true, - index: true +[ ] name: { +[ ] type: String, +[ ] required: true, +[-] index: true }, - slug: { - type: String, - required: true, - unique: true, - index: true +[ ] slug: { +[ ] type: String, +[ ] required: true, +[ ] unique: true, +[-] index: true }, - followersCounts: { - users: { - type: Number, - default: 0 +[ ] followersCounts: { +[ ] users: { +[ ] type: Number, +[ ] default: 0 }, - organizations: { - type: Number, - default: 0 +[ ] organizations: { +[ ] type: Number, +[ ] default: 0 }, - projects: { - type: Number, - default: 0 +[ ] projects: { +[ ] type: Number, +[ ] default: 0 } }, - followingCounts: { - users: { - type: Number, - default: 0 +[ ] followingCounts: { +[ ] users: { +[ ] type: Number, +[ ] default: 0 }, - organizations: { - type: Number, - default: 0 +[ ] organizations: { +[ ] type: Number, +[ ] default: 0 }, - projects: { - type: Number, - default: 0 +[ ] projects: { +[ ] type: Number, +[ ] default: 0 } }, - categoryIds: { - type: Array, - required: true, - index: true +[ ] categoryIds: { +[ ] type: Array, +[ ] required: true, +[-] index: true }, - logo: { type: String }, - coverImg: { type: String }, - userId: { - type: String, - required: true, - index: true +[ ] logo: { type: String }, +[ ] coverImg: { type: String }, +[ ] userId: { +[ ] type: String, +[ ] required: true, +[-] index: true }, - description: { - type: String, - required: true +[ ] description: { +[ ] type: String, +[ ] required: true }, - descriptionExcerpt: { type: String }, // will be generated automatically - publicEmail: { type: String }, - url: { type: String }, - type: { - type: String, - index: true, - enum: ['ngo', 'npo', 'goodpurpose', 'ev', 'eva'] +[ ] descriptionExcerpt: { type: String }, // will be generated automatically +[ ] publicEmail: { type: String }, +[ ] url: { type: String }, +[ ] type: { +[ ] type: String, +[-] index: true, +[ ] enum: ['ngo', 'npo', 'goodpurpose', 'ev', 'eva'] }, - language: { - type: String, - required: true, - default: 'de', - index: true +[ ] language: { +[ ] type: String, +[ ] required: true, +[ ] default: 'de', +[-] index: true }, - addresses: { - type: [{ - street: { - type: String, - required: true +[ ] addresses: { +[ ] type: [{ +[ ] street: { +[ ] type: String, +[ ] required: true }, - zipCode: { - type: String, - required: true +[ ] zipCode: { +[ ] type: String, +[ ] required: true }, - city: { - type: String, - required: true +[ ] city: { +[ ] type: String, +[ ] required: true }, - country: { - type: String, - required: true +[ ] country: { +[ ] type: String, +[ ] required: true }, - lat: { - type: Number, - required: true +[ ] lat: { +[ ] type: Number, +[ ] required: true }, - lng: { - type: Number, - required: true +[ ] lng: { +[ ] type: Number, +[ ] required: true } }], - default: [] +[ ] default: [] }, - createdAt: { - type: Date, - default: Date.now +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now }, - updatedAt: { - type: Date, - default: Date.now +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now }, - isEnabled: { - type: Boolean, - default: false, - index: true +[ ] isEnabled: { +[ ] type: Boolean, +[ ] default: false, +[-] index: true }, - reviewedBy: { - type: String, - default: null, - index: true +[ ] reviewedBy: { +[ ] type: String, +[ ] default: null, +[-] index: true }, - tags: { - type: Array, - index: true +[ ] tags: { +[ ] type: Array, +[-] index: true }, - deleted: { - type: Boolean, - default: false, - index: true +[ ] deleted: { +[ ] type: Boolean, +[ ] default: false, +[-] index: true }, - wasSeeded: { type: Boolean } +[ ] wasSeeded: { type: Boolean } } */ diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/pages.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/pages.cql new file mode 100644 index 000000000..18223136b --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/pages.cql @@ -0,0 +1,55 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] title: { +[ ] type: String, +[ ] required: true + }, +[ ] slug: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] type: { +[ ] type: String, +[ ] required: true, +[ ] default: 'page' + }, +[ ] key: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] content: { +[ ] type: String, +[ ] required: true + }, +[ ] language: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] active: { +[ ] type: Boolean, +[ ] default: true, +[-] index: true + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } + index: +[ ] { slug: 1, language: 1 },{ unique: true } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as page; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/pages_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/pages_delete.cql new file mode 100644 index 000000000..e69de29bb diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/projects.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/projects.cql new file mode 100644 index 000000000..ed859c157 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/projects.cql @@ -0,0 +1,44 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] name: { +[ ] type: String, +[ ] required: true + }, +[ ] slug: { type: String }, +[ ] followerIds: [], +[ ] categoryIds: { type: Array }, +[ ] logo: { type: String }, +[ ] userId: { +[ ] type: String, +[ ] required: true + }, +[ ] description: { +[ ] type: String, +[ ] required: true + }, +[ ] content: { +[ ] type: String, +[ ] required: true + }, +[ ] addresses: { +[ ] type: Array, +[ ] default: [] + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as project; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/projects_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/projects_delete.cql new file mode 100644 index 000000000..e69de29bb diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/settings.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/settings.cql new file mode 100644 index 000000000..1d557d30c --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/settings.cql @@ -0,0 +1,36 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] key: { +[ ] type: String, +[ ] default: 'system', +[-] index: true, +[ ] unique: true + }, +[ ] invites: { +[ ] userCanInvite: { +[ ] type: Boolean, +[ ] required: true, +[ ] default: false + }, +[ ] maxInvitesByUser: { +[ ] type: Number, +[ ] required: true, +[ ] default: 1 + }, +[ ] onlyUserWithBadgesCanInvite: { +[ ] type: Array, +[ ] default: [] + } + }, +[ ] maintenance: false + }, { +[ ] timestamps: true + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as setting; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/settings_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/settings_delete.cql new file mode 100644 index 000000000..e69de29bb diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/status.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/status.cql new file mode 100644 index 000000000..010c2ca09 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/status.cql @@ -0,0 +1,19 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] maintenance: { +[ ] type: Boolean, +[ ] default: false + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as status; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/status_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/status_delete.cql new file mode 100644 index 000000000..e69de29bb diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications.cql new file mode 100644 index 000000000..4bd33eb7c --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications.cql @@ -0,0 +1,61 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] type: { +[ ] type: String, +[ ] default: 'info', +[ ] required: true, +[-] index: true + }, +[ ] title: { +[ ] type: String, +[ ] required: true + }, +[ ] content: { +[ ] type: String, +[ ] required: true + }, +[ ] slot: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] language: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] permanent: { +[ ] type: Boolean, +[ ] default: false + }, +[ ] requireConfirmation: { +[ ] type: Boolean, +[ ] default: false + }, +[ ] active: { +[ ] type: Boolean, +[ ] default: true, +[-] index: true + }, +[ ] totalCount: { +[ ] type: Number, +[ ] default: 0 + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as systemnotification; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/systemnotifications_delete.cql new file mode 100644 index 000000000..e69de29bb diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/userscandos.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/userscandos.cql new file mode 100644 index 000000000..55f58f171 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/userscandos.cql @@ -0,0 +1,35 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] userId: { +[ ] type: String, +[ ] required: true + }, +[ ] contributionId: { +[ ] type: String, +[ ] required: true + }, +[ ] done: { +[ ] type: Boolean, +[ ] default: false + }, +[ ] doneAt: { type: Date }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } + index: +[ ] { userId: 1, contributionId: 1 },{ unique: true } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as usercando; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/userscandos_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/userscandos_delete.cql new file mode 100644 index 000000000..e69de29bb diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/usersettings.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/usersettings.cql new file mode 100644 index 000000000..722625944 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/usersettings.cql @@ -0,0 +1,43 @@ +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[ ] userId: { +[ ] type: String, +[ ] required: true, +[ ] unique: true + }, +[ ] blacklist: { +[ ] type: Array, +[ ] default: [] + }, +[ ] uiLanguage: { +[ ] type: String, +[ ] required: true + }, +[ ] contentLanguages: { +[ ] type: Array, +[ ] default: [] + }, +[ ] filter: { +[ ] categoryIds: { +[ ] type: Array, +[ ] index: true + }, +[ ] emotions: { +[ ] type: Array, +[ ] index: true + } + }, +[ ] hideUsersWithoutTermsOfUseSigniture: {type: Boolean}, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as usersetting; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/usersettings_delete.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/usersettings_delete.cql new file mode 100644 index 000000000..e69de29bb From 4c758a90833aa2efe3c403b7b0825bcc70fdacc2 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Fri, 31 May 2019 15:46:37 +0200 Subject: [PATCH 09/13] fixed filename extension check --- backend/src/types/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/types/index.js b/backend/src/types/index.js index ca67aadbd..bcdceed44 100644 --- a/backend/src/types/index.js +++ b/backend/src/types/index.js @@ -12,7 +12,7 @@ const findGqlFiles = dir => { // Recurse into a subdirectory results = results.concat(findGqlFiles(file)) } else { - if (file.split('.').pop() === 'gql') { + if (path.extname(file) === '.gql') { // Is a gql file results.push(file) } From 754decd308ac3558de6ca1a31ea57eebd8afb7bc Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Fri, 31 May 2019 16:04:17 +0200 Subject: [PATCH 10/13] also wildcard include resolvers --- backend/src/graphql-schema.js | 42 ++-------------------------------- backend/src/resolvers/index.js | 6 +++++ 2 files changed, 8 insertions(+), 40 deletions(-) create mode 100644 backend/src/resolvers/index.js diff --git a/backend/src/graphql-schema.js b/backend/src/graphql-schema.js index c0b06af9c..01b40dfc5 100644 --- a/backend/src/graphql-schema.js +++ b/backend/src/graphql-schema.js @@ -1,40 +1,2 @@ -// resolvers -import userManagement from './resolvers/user_management.js' -import statistics from './resolvers/statistics.js' -import reports from './resolvers/reports.js' -import posts from './resolvers/posts.js' -import moderation from './resolvers/moderation.js' -import follow from './resolvers/follow.js' -import shout from './resolvers/shout.js' -import rewards from './resolvers/rewards.js' -import socialMedia from './resolvers/socialMedia.js' -import notifications from './resolvers/notifications' -import comments from './resolvers/comments' -import users from './resolvers/users' - -// types -import types from './types' - -export const typeDefs = types - -export const resolvers = { - Query: { - ...statistics.Query, - ...userManagement.Query, - ...notifications.Query, - ...comments.Query, - }, - Mutation: { - ...userManagement.Mutation, - ...reports.Mutation, - ...posts.Mutation, - ...moderation.Mutation, - ...follow.Mutation, - ...shout.Mutation, - ...rewards.Mutation, - ...socialMedia.Mutation, - ...notifications.Mutation, - ...comments.Mutation, - ...users.Mutation, - }, -} +export { default as typeDefs } from './types' +export { default as resolvers } from './resolvers' diff --git a/backend/src/resolvers/index.js b/backend/src/resolvers/index.js new file mode 100644 index 000000000..e10a047b3 --- /dev/null +++ b/backend/src/resolvers/index.js @@ -0,0 +1,6 @@ +import path from 'path' +import { fileLoader, mergeResolvers } from 'merge-graphql-schemas' + +const resolversArray = fileLoader(path.join(__dirname, './'), { extensions: ['!(*.spec).js'] }) + +export default mergeResolvers(resolversArray) From 10864e7d1809677b6b624a518cae6f6baee46f3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Fri, 31 May 2019 16:01:01 +0200 Subject: [PATCH 11/13] Remove command line arguments `-u` and `-p` @ulfgebhardt: The docs at `man cypher-shell` say that you can pass `NEO4J_USERNAME` and `NEO4J_PASSWORD` per environment variable. So the command line arguments are obsolete here. --- .../maintenance-worker/migration/neo4j/import.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh index 5720baaf0..08c05c8cc 100755 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh @@ -10,7 +10,7 @@ set +o allexport function delete_collection () { # Delete from Database echo "Delete $1" - "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} < $(dirname "$0")/$1_delete.cql > /dev/null + "${IMPORT_CYPHERSHELL_BIN}" < $(dirname "$0")/$1_delete.cql > /dev/null # Delete index file rm -f "${IMPORT_PATH}splits/$1.index" } @@ -22,7 +22,7 @@ function import_collection () { # load index file if [ -f "$INDEX_FILE" ]; then readarray -t IMPORT_INDEX <$INDEX_FILE - else + else declare -a IMPORT_INDEX fi # for each chunk import data @@ -37,7 +37,7 @@ function import_collection () { NEO4J_COMMAND="$(envsubst '${IMPORT_CHUNK_PATH_CQL_FILE}' < $(dirname "$0")/$1.cql)" # run the import of the chunk echo "Import $1 ${CHUNK_FILE_NAME} (${chunk})" - echo "${NEO4J_COMMAND}" | "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} > /dev/null + echo "${NEO4J_COMMAND}" | "${IMPORT_CYPHERSHELL_BIN}" > /dev/null # add file to array and file IMPORT_INDEX+=("${CHUNK_FILE_NAME}") echo "${CHUNK_FILE_NAME}" >> ${INDEX_FILE} From 58add8fc5fbf6f42aec15c69668f2ca485d70c61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Fri, 31 May 2019 16:02:27 +0200 Subject: [PATCH 12/13] Install `envsubst` in Dockerfile @ulfgebhardt please setup docker on your machine or a remote machine. Installing `envsubst` on alpine fails with circular dependencies (awkward). So this repo here has a solution: https://github.com/cirocosta/alpine-envsubst/blob/master/Dockerfile#L6 --- .../legacy-migration/maintenance-worker/Dockerfile | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/deployment/legacy-migration/maintenance-worker/Dockerfile b/deployment/legacy-migration/maintenance-worker/Dockerfile index 1fafce5e8..c4bc5a0a5 100644 --- a/deployment/legacy-migration/maintenance-worker/Dockerfile +++ b/deployment/legacy-migration/maintenance-worker/Dockerfile @@ -3,6 +3,16 @@ FROM humanconnection/neo4j:latest ENV NODE_ENV=maintenance EXPOSE 7687 7474 +ENV BUILD_DEPS="gettext" \ + RUNTIME_DEPS="libintl" + +RUN set -x && \ + apk add --update $RUNTIME_DEPS && \ + apk add --virtual build_deps $BUILD_DEPS && \ + cp /usr/bin/envsubst /usr/local/bin/envsubst && \ + apk del build_deps + + RUN apk upgrade --update RUN apk add --no-cache mongodb-tools openssh nodejs yarn rsync From b51b8423e75de1213e1ebceedac26f4c516dd265 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Fri, 31 May 2019 16:40:35 +0200 Subject: [PATCH 13/13] fixed fileloader pattern to actually load smth --- backend/src/resolvers/index.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/src/resolvers/index.js b/backend/src/resolvers/index.js index e10a047b3..3d3a91d68 100644 --- a/backend/src/resolvers/index.js +++ b/backend/src/resolvers/index.js @@ -1,6 +1,5 @@ import path from 'path' import { fileLoader, mergeResolvers } from 'merge-graphql-schemas' -const resolversArray = fileLoader(path.join(__dirname, './'), { extensions: ['!(*.spec).js'] }) - +const resolversArray = fileLoader(path.join(__dirname, './!(*.spec).js')) export default mergeResolvers(resolversArray)