From 6f0447515a5a813bbb3e5880f4c8164d0601221e Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Wed, 29 May 2019 15:37:28 +0200 Subject: [PATCH] - fixed several errors handling import - split graphql schema into parts - which data is imported which is not - a list --- backend/package.json | 7 +- backend/src/graphql-schema.js | 11 +- backend/src/resolvers/badges.spec.js | 4 +- backend/src/seed/factories/badges.js | 4 +- backend/src/seed/factories/posts.js | 2 +- backend/src/seed/factories/users.js | 2 +- backend/src/types/enum/BadgeStatus.gql | 4 + backend/src/types/enum/BadgeType.gql | 4 + backend/src/types/enum/UserGroup.gql | 5 + backend/src/types/enum/Visibility.gql | 5 + backend/src/types/index.js | 30 ++++ backend/src/types/scalar/Date.gql_ | 1 + backend/src/types/scalar/DateTime.gql_ | 1 + backend/src/types/scalar/Time.gql_ | 1 + backend/src/types/scalar/Upload.gql | 1 + backend/src/types/schema.gql | 134 ++++++++++++++++++ .../schema_full.gql_} | 0 backend/src/types/type/Badge.gql | 13 ++ backend/src/types/type/Category.gql | 13 ++ backend/src/types/type/Comment.gql | 14 ++ backend/src/types/type/Post.gql | 43 ++++++ backend/src/types/type/Tag.gql | 10 ++ backend/src/types/type/User.gql | 80 +++++++++++ backend/yarn.lock | 14 ++ .../migration/neo4j/_delete_all.cql | 1 + .../migration/neo4j/badges.cql | 44 +++++- .../migration/neo4j/categories.cql | 34 ++++- .../migration/neo4j/comments.cql | 70 +++++++-- .../migration/neo4j/contributions.cql | 130 ++++++++++++++++- .../migration/neo4j/follows.cql | 34 ++++- .../migration/neo4j/import.sh | 10 +- .../migration/neo4j/shouts.cql | 34 ++++- .../migration/neo4j/users.cql | 100 ++++++++++++- 33 files changed, 825 insertions(+), 35 deletions(-) create mode 100644 backend/src/types/enum/BadgeStatus.gql create mode 100644 backend/src/types/enum/BadgeType.gql create mode 100644 backend/src/types/enum/UserGroup.gql create mode 100644 backend/src/types/enum/Visibility.gql create mode 100644 backend/src/types/index.js create mode 100644 backend/src/types/scalar/Date.gql_ create mode 100644 backend/src/types/scalar/DateTime.gql_ create mode 100644 backend/src/types/scalar/Time.gql_ create mode 100644 backend/src/types/scalar/Upload.gql create mode 100644 backend/src/types/schema.gql rename backend/src/{schema.graphql => types/schema_full.gql_} (100%) create mode 100644 backend/src/types/type/Badge.gql create mode 100644 backend/src/types/type/Category.gql create mode 100644 backend/src/types/type/Comment.gql create mode 100644 backend/src/types/type/Post.gql create mode 100644 backend/src/types/type/Tag.gql create mode 100644 backend/src/types/type/User.gql create mode 100644 deployment/legacy-migration/maintenance-worker/migration/neo4j/_delete_all.cql diff --git a/backend/package.json b/backend/package.json index 9f2bbe7ab..4006a5bdb 100644 --- a/backend/package.json +++ b/backend/package.json @@ -6,8 +6,8 @@ "scripts": { "build": "babel src/ -d dist/ --copy-files", "start": "node dist/", - "dev": "nodemon --exec babel-node src/ -e js,graphql", - "dev:debug": "nodemon --exec babel-node --inspect=0.0.0.0:9229 src/index.js -e js,graphql", + "dev": "nodemon --exec babel-node src/ -e js,gql", + "dev:debug": "nodemon --exec babel-node --inspect=0.0.0.0:9229 src/index.js -e js,gql", "lint": "eslint src --config .eslintrc.js", "test": "run-s test:jest test:cucumber", "test:before:server": "cross-env GRAPHQL_URI=http://localhost:4123 GRAPHQL_PORT=4123 yarn run dev 2> /dev/null", @@ -68,6 +68,7 @@ "jsonwebtoken": "~8.5.1", "linkifyjs": "~2.1.8", "lodash": "~4.17.11", + "merge-graphql-schemas": "^1.5.8", "ms": "~2.1.1", "neo4j-driver": "~1.7.4", "neo4j-graphql-js": "git+https://github.com/Human-Connection/neo4j-graphql-js.git#temporary_fixes", @@ -94,8 +95,8 @@ "chai": "~4.2.0", "cucumber": "~5.1.0", "eslint": "~5.16.0", - "eslint-config-standard": "~12.0.0", "eslint-config-prettier": "~4.3.0", + "eslint-config-standard": "~12.0.0", "eslint-plugin-import": "~2.17.3", "eslint-plugin-jest": "~22.6.4", "eslint-plugin-node": "~9.1.0", diff --git a/backend/src/graphql-schema.js b/backend/src/graphql-schema.js index a846541e7..c0b06af9c 100644 --- a/backend/src/graphql-schema.js +++ b/backend/src/graphql-schema.js @@ -1,6 +1,4 @@ -import fs from 'fs' -import path from 'path' - +// resolvers import userManagement from './resolvers/user_management.js' import statistics from './resolvers/statistics.js' import reports from './resolvers/reports.js' @@ -14,9 +12,10 @@ import notifications from './resolvers/notifications' import comments from './resolvers/comments' import users from './resolvers/users' -export const typeDefs = fs - .readFileSync(process.env.GRAPHQL_SCHEMA || path.join(__dirname, 'schema.graphql')) - .toString('utf-8') +// types +import types from './types' + +export const typeDefs = types export const resolvers = { Query: { diff --git a/backend/src/resolvers/badges.spec.js b/backend/src/resolvers/badges.spec.js index f5e15f4a2..8383d9838 100644 --- a/backend/src/resolvers/badges.spec.js +++ b/backend/src/resolvers/badges.spec.js @@ -41,8 +41,8 @@ describe('badges', () => { mutation( $id: ID $key: String! - $type: BadgeTypeEnum! - $status: BadgeStatusEnum! + $type: BadgeType! + $status: BadgeStatus! $icon: String! ) { CreateBadge(id: $id, key: $key, type: $type, status: $status, icon: $icon) { diff --git a/backend/src/seed/factories/badges.js b/backend/src/seed/factories/badges.js index 52fa9da33..6414e9f36 100644 --- a/backend/src/seed/factories/badges.js +++ b/backend/src/seed/factories/badges.js @@ -14,8 +14,8 @@ export default function(params) { mutation( $id: ID $key: String! - $type: BadgeTypeEnum! - $status: BadgeStatusEnum! + $type: BadgeType! + $status: BadgeStatus! $icon: String! ) { CreateBadge(id: $id, key: $key, type: $type, status: $status, icon: $icon) { diff --git a/backend/src/seed/factories/posts.js b/backend/src/seed/factories/posts.js index 1468b05d7..ea92f7d9f 100644 --- a/backend/src/seed/factories/posts.js +++ b/backend/src/seed/factories/posts.js @@ -26,7 +26,7 @@ export default function(params) { $title: String! $content: String! $image: String - $visibility: VisibilityEnum + $visibility: Visibility $deleted: Boolean ) { CreatePost( diff --git a/backend/src/seed/factories/users.js b/backend/src/seed/factories/users.js index 1ab362ce2..ca17d1721 100644 --- a/backend/src/seed/factories/users.js +++ b/backend/src/seed/factories/users.js @@ -23,7 +23,7 @@ export default function create(params) { $email: String! $avatar: String $about: String - $role: UserGroupEnum + $role: UserGroup ) { CreateUser( id: $id diff --git a/backend/src/types/enum/BadgeStatus.gql b/backend/src/types/enum/BadgeStatus.gql new file mode 100644 index 000000000..b109663b3 --- /dev/null +++ b/backend/src/types/enum/BadgeStatus.gql @@ -0,0 +1,4 @@ +enum BadgeStatus { + permanent + temporary +} \ No newline at end of file diff --git a/backend/src/types/enum/BadgeType.gql b/backend/src/types/enum/BadgeType.gql new file mode 100644 index 000000000..eccf2e661 --- /dev/null +++ b/backend/src/types/enum/BadgeType.gql @@ -0,0 +1,4 @@ +enum BadgeType { + role + crowdfunding +} \ No newline at end of file diff --git a/backend/src/types/enum/UserGroup.gql b/backend/src/types/enum/UserGroup.gql new file mode 100644 index 000000000..af25bcc69 --- /dev/null +++ b/backend/src/types/enum/UserGroup.gql @@ -0,0 +1,5 @@ +enum UserGroup { + admin + moderator + user +} \ No newline at end of file diff --git a/backend/src/types/enum/Visibility.gql b/backend/src/types/enum/Visibility.gql new file mode 100644 index 000000000..4f9d5591a --- /dev/null +++ b/backend/src/types/enum/Visibility.gql @@ -0,0 +1,5 @@ +enum Visibility { + public + friends + private +} \ No newline at end of file diff --git a/backend/src/types/index.js b/backend/src/types/index.js new file mode 100644 index 000000000..ca67aadbd --- /dev/null +++ b/backend/src/types/index.js @@ -0,0 +1,30 @@ +import fs from 'fs' +import path from 'path' +import { mergeTypes } from 'merge-graphql-schemas' + +const findGqlFiles = dir => { + var results = [] + var list = fs.readdirSync(dir) + list.forEach(file => { + file = path.join(dir, file).toString('utf-8') + var stat = fs.statSync(file) + if (stat && stat.isDirectory()) { + // Recurse into a subdirectory + results = results.concat(findGqlFiles(file)) + } else { + if (file.split('.').pop() === 'gql') { + // Is a gql file + results.push(file) + } + } + }) + return results +} + +let typeDefs = [] + +findGqlFiles(__dirname).forEach(file => { + typeDefs.push(fs.readFileSync(file).toString('utf-8')) +}) + +export default mergeTypes(typeDefs, { all: true }) diff --git a/backend/src/types/scalar/Date.gql_ b/backend/src/types/scalar/Date.gql_ new file mode 100644 index 000000000..7b0004ea3 --- /dev/null +++ b/backend/src/types/scalar/Date.gql_ @@ -0,0 +1 @@ +scalar Date \ No newline at end of file diff --git a/backend/src/types/scalar/DateTime.gql_ b/backend/src/types/scalar/DateTime.gql_ new file mode 100644 index 000000000..af973932f --- /dev/null +++ b/backend/src/types/scalar/DateTime.gql_ @@ -0,0 +1 @@ +scalar DateTime \ No newline at end of file diff --git a/backend/src/types/scalar/Time.gql_ b/backend/src/types/scalar/Time.gql_ new file mode 100644 index 000000000..53becdd66 --- /dev/null +++ b/backend/src/types/scalar/Time.gql_ @@ -0,0 +1 @@ +scalar Time \ No newline at end of file diff --git a/backend/src/types/scalar/Upload.gql b/backend/src/types/scalar/Upload.gql new file mode 100644 index 000000000..fca9ea1fc --- /dev/null +++ b/backend/src/types/scalar/Upload.gql @@ -0,0 +1 @@ +scalar Upload \ No newline at end of file diff --git a/backend/src/types/schema.gql b/backend/src/types/schema.gql new file mode 100644 index 000000000..ab8b25399 --- /dev/null +++ b/backend/src/types/schema.gql @@ -0,0 +1,134 @@ +type Query { + isLoggedIn: Boolean! + # Get the currently logged in User based on the given JWT Token + currentUser: User + # Get the latest Network Statistics + statistics: Statistics! + findPosts(filter: String!, limit: Int = 10): [Post]! @cypher( + statement: """ + CALL db.index.fulltext.queryNodes('full_text_search', $filter) + YIELD node as post, score + MATCH (post)<-[:WROTE]-(user:User) + WHERE score >= 0.2 + AND NOT user.deleted = true AND NOT user.disabled = true + AND NOT post.deleted = true AND NOT post.disabled = true + RETURN post + LIMIT $limit + """ + ) + CommentByPost(postId: ID!): [Comment]! +} + +type Mutation { + # Get a JWT Token for the given Email and password + login(email: String!, password: String!): String! + signup(email: String!, password: String!): Boolean! + changePassword(oldPassword:String!, newPassword: String!): String! + report(id: ID!, description: String): Report + disable(id: ID!): ID + enable(id: ID!): ID + reward(fromBadgeId: ID!, toUserId: ID!): ID + unreward(fromBadgeId: ID!, toUserId: ID!): ID + # Shout the given Type and ID + shout(id: ID!, type: ShoutTypeEnum): Boolean! + # Unshout the given Type and ID + unshout(id: ID!, type: ShoutTypeEnum): Boolean! + # Follow the given Type and ID + follow(id: ID!, type: FollowTypeEnum): Boolean! + # Unfollow the given Type and ID + unfollow(id: ID!, type: FollowTypeEnum): Boolean! +} + +type Statistics { + countUsers: Int! + countPosts: Int! + countComments: Int! + countNotifications: Int! + countOrganizations: Int! + countProjects: Int! + countInvites: Int! + countFollows: Int! + countShouts: Int! +} + +type Notification { + id: ID! + read: Boolean, + user: User @relation(name: "NOTIFIED", direction: "OUT") + post: Post @relation(name: "NOTIFIED", direction: "IN") + createdAt: String +} + +type Location { + id: ID! + name: String! + nameEN: String + nameDE: String + nameFR: String + nameNL: String + nameIT: String + nameES: String + namePT: String + namePL: String + type: String! + lat: Float + lng: Float + parent: Location @cypher(statement: "MATCH (this)-[:IS_IN]->(l:Location) RETURN l") +} + +type Report { + id: ID! + submitter: User @relation(name: "REPORTED", direction: "IN") + description: String + type: String! @cypher(statement: "MATCH (resource)<-[:REPORTED]-(this) RETURN labels(resource)[0]") + createdAt: String + comment: Comment @relation(name: "REPORTED", direction: "OUT") + post: Post @relation(name: "REPORTED", direction: "OUT") + user: User @relation(name: "REPORTED", direction: "OUT") +} + +enum ShoutTypeEnum { + Post + Organization + Project +} +enum FollowTypeEnum { + User + Organization + Project +} + +type Reward { + id: ID! + user: User @relation(name: "REWARDED", direction: "IN") + rewarderId: ID + createdAt: String + badge: Badge @relation(name: "REWARDED", direction: "OUT") +} + +type Organization { + id: ID! + createdBy: User @relation(name: "CREATED_ORGA", direction: "IN") + ownedBy: [User] @relation(name: "OWNING_ORGA", direction: "IN") + name: String! + slug: String + description: String! + descriptionExcerpt: String + deleted: Boolean + disabled: Boolean + + tags: [Tag]! @relation(name: "TAGGED", direction: "OUT") + categories: [Category]! @relation(name: "CATEGORIZED", direction: "OUT") +} + +type SharedInboxEndpoint { + id: ID! + uri: String +} + +type SocialMedia { + id: ID! + url: String + ownedBy: [User]! @relation(name: "OWNED", direction: "IN") +} + diff --git a/backend/src/schema.graphql b/backend/src/types/schema_full.gql_ similarity index 100% rename from backend/src/schema.graphql rename to backend/src/types/schema_full.gql_ diff --git a/backend/src/types/type/Badge.gql b/backend/src/types/type/Badge.gql new file mode 100644 index 000000000..68c5d5707 --- /dev/null +++ b/backend/src/types/type/Badge.gql @@ -0,0 +1,13 @@ +type Badge { + id: ID! + key: String! + type: BadgeType! + status: BadgeStatus! + icon: String! + #createdAt: DateTime + #updatedAt: DateTime + createdAt: String + updatedAt: String + + rewarded: [User]! @relation(name: "REWARDED", direction: "OUT") +} \ No newline at end of file diff --git a/backend/src/types/type/Category.gql b/backend/src/types/type/Category.gql new file mode 100644 index 000000000..d5e089a6f --- /dev/null +++ b/backend/src/types/type/Category.gql @@ -0,0 +1,13 @@ +type Category { + id: ID! + name: String! + slug: String! + icon: String! + #createdAt: DateTime + #updatedAt: DateTime + createdAt: String + updatedAt: String + + posts: [Post]! @relation(name: "CATEGORIZED", direction: "IN") + postCount: Int! @cypher(statement: "MATCH (this)<-[:CATEGORIZED]-(r:Post) RETURN COUNT(r)") +} \ No newline at end of file diff --git a/backend/src/types/type/Comment.gql b/backend/src/types/type/Comment.gql new file mode 100644 index 000000000..077366e8a --- /dev/null +++ b/backend/src/types/type/Comment.gql @@ -0,0 +1,14 @@ +type Comment { + id: ID! + activityId: String + postId: ID + author: User @relation(name: "WROTE", direction: "IN") + content: String! + contentExcerpt: String + post: Post @relation(name: "COMMENTS", direction: "OUT") + createdAt: String + updatedAt: String + deleted: Boolean + disabled: Boolean + disabledBy: User @relation(name: "DISABLED", direction: "IN") +} \ No newline at end of file diff --git a/backend/src/types/type/Post.gql b/backend/src/types/type/Post.gql new file mode 100644 index 000000000..6e23862ed --- /dev/null +++ b/backend/src/types/type/Post.gql @@ -0,0 +1,43 @@ +type Post { + id: ID! + activityId: String + objectId: String + author: User @relation(name: "WROTE", direction: "IN") + title: String! + slug: String + content: String! + contentExcerpt: String + image: String + imageUpload: Upload + visibility: Visibility + deleted: Boolean + disabled: Boolean + disabledBy: User @relation(name: "DISABLED", direction: "IN") + createdAt: String + updatedAt: String + + relatedContributions: [Post]! @cypher( + statement: """ + MATCH (this)-[:TAGGED|CATEGORIZED]->(categoryOrTag)<-[:TAGGED|CATEGORIZED]-(post:Post) + RETURN DISTINCT post + LIMIT 10 + """ + ) + + tags: [Tag]! @relation(name: "TAGGED", direction: "OUT") + categories: [Category]! @relation(name: "CATEGORIZED", direction: "OUT") + + comments: [Comment]! @relation(name: "COMMENTS", direction: "IN") + commentsCount: Int! @cypher(statement: "MATCH (this)<-[:COMMENTS]-(r:Comment) WHERE NOT r.deleted = true AND NOT r.disabled = true RETURN COUNT(r)") + + shoutedBy: [User]! @relation(name: "SHOUTED", direction: "IN") + shoutedCount: Int! @cypher(statement: "MATCH (this)<-[:SHOUTED]-(r:User) WHERE NOT r.deleted = true AND NOT r.disabled = true RETURN COUNT(DISTINCT r)") + + # Has the currently logged in user shouted that post? + shoutedByCurrentUser: Boolean! @cypher( + statement: """ + MATCH (this)<-[:SHOUTED]-(u:User {id: $cypherParams.currentUserId}) + RETURN COUNT(u) >= 1 + """ + ) +} \ No newline at end of file diff --git a/backend/src/types/type/Tag.gql b/backend/src/types/type/Tag.gql new file mode 100644 index 000000000..ecbd0b46a --- /dev/null +++ b/backend/src/types/type/Tag.gql @@ -0,0 +1,10 @@ +type Tag { + id: ID! + name: String! + taggedPosts: [Post]! @relation(name: "TAGGED", direction: "IN") + taggedOrganizations: [Organization]! @relation(name: "TAGGED", direction: "IN") + taggedCount: Int! @cypher(statement: "MATCH (this)<-[:TAGGED]-(p) RETURN COUNT(DISTINCT p)") + taggedCountUnique: Int! @cypher(statement: "MATCH (this)<-[:TAGGED]-(p)<-[:WROTE]-(u:User) RETURN COUNT(DISTINCT u)") + deleted: Boolean + disabled: Boolean +} \ No newline at end of file diff --git a/backend/src/types/type/User.gql b/backend/src/types/type/User.gql new file mode 100644 index 000000000..1287aa45f --- /dev/null +++ b/backend/src/types/type/User.gql @@ -0,0 +1,80 @@ +type User { + id: ID! + actorId: String + name: String + email: String! + slug: String + password: String! + avatar: String + coverImg: String + avatarUpload: Upload + deleted: Boolean + disabled: Boolean + disabledBy: User @relation(name: "DISABLED", direction: "IN") + role: UserGroup + publicKey: String + privateKey: String + + wasInvited: Boolean + wasSeeded: Boolean + + location: Location @cypher(statement: "MATCH (this)-[:IS_IN]->(l:Location) RETURN l") + locationName: String + about: String + socialMedia: [SocialMedia]! @relation(name: "OWNED", direction: "OUT") + + #createdAt: DateTime + #updatedAt: DateTime + createdAt: String + updatedAt: String + + notifications(read: Boolean): [Notification]! @relation(name: "NOTIFIED", direction: "IN") + + friends: [User]! @relation(name: "FRIENDS", direction: "BOTH") + friendsCount: Int! @cypher(statement: "MATCH (this)<-[:FRIENDS]->(r:User) RETURN COUNT(DISTINCT r)") + + following: [User]! @relation(name: "FOLLOWS", direction: "OUT") + followingCount: Int! @cypher(statement: "MATCH (this)-[:FOLLOWS]->(r:User) RETURN COUNT(DISTINCT r)") + + followedBy: [User]! @relation(name: "FOLLOWS", direction: "IN") + followedByCount: Int! @cypher(statement: "MATCH (this)<-[:FOLLOWS]-(r:User) RETURN COUNT(DISTINCT r)") + + # Is the currently logged in user following that user? + followedByCurrentUser: Boolean! @cypher( + statement: """ + MATCH (this)<-[:FOLLOWS]-(u:User {id: $cypherParams.currentUserId}) + RETURN COUNT(u) >= 1 + """ + ) + + #contributions: [WrittenPost]! + #contributions2(first: Int = 10, offset: Int = 0): [WrittenPost2]! + # @cypher( + # statement: "MATCH (this)-[w:WROTE]->(p:Post) RETURN p as Post, w.timestamp as timestamp" + # ) + contributions: [Post]! @relation(name: "WROTE", direction: "OUT") + contributionsCount: Int! @cypher( + statement: """ + MATCH (this)-[:WROTE]->(r:Post) + WHERE (NOT exists(r.deleted) OR r.deleted = false) + AND (NOT exists(r.disabled) OR r.disabled = false) + RETURN COUNT(r) + """ + ) + + comments: [Comment]! @relation(name: "WROTE", direction: "OUT") + commentsCount: Int! @cypher(statement: "MATCH (this)-[:WROTE]->(r:Comment) WHERE NOT r.deleted = true AND NOT r.disabled = true RETURN COUNT(r)") + + shouted: [Post]! @relation(name: "SHOUTED", direction: "OUT") + shoutedCount: Int! @cypher(statement: "MATCH (this)-[:SHOUTED]->(r:Post) WHERE NOT r.deleted = true AND NOT r.disabled = true RETURN COUNT(DISTINCT r)") + + organizationsCreated: [Organization] @relation(name: "CREATED_ORGA", direction: "OUT") + organizationsOwned: [Organization] @relation(name: "OWNING_ORGA", direction: "OUT") + + blacklisted: [User]! @relation(name: "BLACKLISTED", direction: "OUT") + + categories: [Category]! @relation(name: "CATEGORIZED", direction: "OUT") + + badges: [Badge]! @relation(name: "REWARDED", direction: "IN") + badgesCount: Int! @cypher(statement: "MATCH (this)<-[:REWARDED]-(r:Badge) RETURN COUNT(r)") +} \ No newline at end of file diff --git a/backend/yarn.lock b/backend/yarn.lock index 176af3893..534d6eef7 100644 --- a/backend/yarn.lock +++ b/backend/yarn.lock @@ -2656,6 +2656,11 @@ deep-is@~0.1.3: resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" integrity sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ= +deepmerge@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-2.2.1.tgz#5d3ff22a01c00f645405a2fbc17d0778a1801170" + integrity sha512-R9hc1Xa/NOBi9WRVUWg19rl1UB7Tt4kuPd+thNJgFZoxXsTz7ncaPaeIm+40oSGuP33DfMb4sZt1QIGiJzC4EA== + define-properties@^1.1.2: version "1.1.3" resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.3.tgz#cf88da6cbee26fe6db7094f61d870cbd84cee9f1" @@ -5388,6 +5393,15 @@ merge-descriptors@1.0.1: resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" integrity sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E= +merge-graphql-schemas@^1.5.8: + version "1.5.8" + resolved "https://registry.yarnpkg.com/merge-graphql-schemas/-/merge-graphql-schemas-1.5.8.tgz#89457b60312aabead44d5b2b7625643f8ab9e369" + integrity sha512-0TGOKebltvmWR9h9dPYS2vAqMPThXwJ6gVz7O5MtpBp2sunAg/M25iMSNI7YhU6PDJVtGtldTfqV9a+55YhB+A== + dependencies: + deepmerge "^2.2.1" + glob "^7.1.3" + is-glob "^4.0.0" + merge-stream@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-1.0.1.tgz#4041202d508a342ba00174008df0c251b8c135e1" diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/_delete_all.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/_delete_all.cql new file mode 100644 index 000000000..d01871300 --- /dev/null +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/_delete_all.cql @@ -0,0 +1 @@ +MATCH (n) DETACH DELETE n; \ No newline at end of file diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql index 62eddf124..62cd4a2cc 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/badges.cql @@ -1,4 +1,46 @@ -CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as badge +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[?] image: { +[?] path: { // Path is incorrect in Nitro - is icon the correct name for this field? +[X] type: String, +[X] required: true + }, +[ ] alt: { // If we use an image - should we not have an alt? +[ ] type: String, +[ ] required: true + } + }, +[?] status: { +[X] type: String, +[X] enum: ['permanent', 'temporary'], +[ ] default: 'permanent', // Default value is missing in Nitro +[X] required: true + }, +[?] type: { +[?] type: String, // in nitro this is a defined enum - seems good for now +[X] required: true + }, +[X] key: { +[X] type: String, +[X] required: true + }, +[?] createdAt: { +[?] type: Date, // Type is modeled as string in Nitro which is incorrect +[ ] default: Date.now // Default value is missing in Nitro + }, +[?] updatedAt: { +[?] type: Date, // Type is modeled as string in Nitro which is incorrect +[ ] default: Date.now // Default value is missing in Nitro + } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as badge MERGE(b:Badge {id: badge._id["$oid"]}) ON CREATE SET b.key = badge.key, diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql index 81e73c1b9..2866c8aae 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/categories.cql @@ -1,4 +1,35 @@ -CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as category +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[X] title: { +[X] type: String, +[X] required: true + }, +[?] slug: { +[X] type: String, +[X] required: true, +[ ] unique: true // Unique value is not enforced in Nitro? + }, +[?] icon: { // Nitro adds required: true +[X] type: String, +[ ] unique: true // Unique value is not enforced in Nitro? + }, +[?] createdAt: { +[?] type: Date, // Type is modeled as string in Nitro which is incorrect +[ ] default: Date.now // Default value is missing in Nitro + }, +[?] updatedAt: { +[?] type: Date, // Type is modeled as string in Nitro which is incorrect +[ ] default: Date.now // Default value is missing in Nitro + } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as category MERGE(c:Category {id: category._id["$oid"]}) ON CREATE SET c.name = category.title, @@ -8,6 +39,7 @@ c.createdAt = category.createdAt.`$date`, c.updatedAt = category.updatedAt.`$date` ; +// Transform icon names MATCH (c:Category) WHERE (c.icon = "categories-justforfun") SET c.icon = 'smile' diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql index 1c8eb9397..1cdc1bfc2 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/comments.cql @@ -1,15 +1,65 @@ -CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as json +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[?] userId: { +[X] type: String, +[ ] required: true, // Not required in Nitro +[-] index: true + }, +[?] contributionId: { +[X] type: String, +[ ] required: true, // Not required in Nitro +[-] index: true + }, +[X] content: { +[X] type: String, +[X] required: true + }, +[?] contentExcerpt: { // Generated from content +[X] type: String, +[ ] required: true // Not required in Nitro + }, +[ ] hasMore: { type: Boolean }, +[ ] upvotes: { +[ ] type: Array, +[ ] default: [] + }, +[ ] upvoteCount: { +[ ] type: Number, +[ ] default: 0 + }, +[?] deleted: { +[X] type: Boolean, +[ ] default: false, // Default value is missing in Nitro +[-] index: true + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] updatedAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } +*/ -MERGE (comment:Comment {id: json._id["$oid"]}) +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as comment +MERGE (c:Comment {id: comment._id["$oid"]}) ON CREATE SET -comment.content = json.content, -comment.contentExcerpt = json.contentExcerpt, -comment.deleted = json.deleted, -comment.disabled = false -WITH comment, json, json.contributionId as postId +c.content = comment.content, +c.contentExcerpt = comment.contentExcerpt, +c.deleted = comment.deleted, +c.disabled = false +WITH c, comment, comment.contributionId as postId MATCH (post:Post {id: postId}) -WITH comment, post, json.userId as userId +WITH c, post, comment.userId as userId MATCH (author:User {id: userId}) -MERGE (comment)-[:COMMENTS]->(post) -MERGE (author)-[:WROTE]->(comment) +MERGE (c)-[:COMMENTS]->(post) +MERGE (author)-[:WROTE]->(c) ; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql index e4e148af3..98d8f24e9 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/contributions.cql @@ -1,4 +1,132 @@ -CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as post +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro +[?] { //Modeled incorrect as Post +[?] userId: { +[X] type: String, +[ ] required: true, // Not required in Nitro +[-] index: true + }, +[ ] organizationId: { +[ ] type: String, +[-] index: true + }, +[X] categoryIds: { +[X] type: Array, +[-] index: true + }, +[X] title: { +[X] type: String, +[X] required: true + }, +[?] slug: { // Generated from title +[X] type: String, +[ ] required: true, // Not required in Nitro +[?] unique: true, // Unique value is not enforced in Nitro? +[-] index: true + }, +[ ] type: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] cando: { +[ ] difficulty: { +[ ] type: String, +[ ] enum: ['easy', 'medium', 'hard'] + }, +[ ] reasonTitle: { type: String }, +[ ] reason: { type: String } + }, +[X] content: { +[X] type: String, +[X] required: true + }, +[?] contentExcerpt: { // Generated from content +[X] type: String, +[?] required: true // Not required in Nitro + }, +[ ] hasMore: { type: Boolean }, +[?] teaserImg: { type: String }, // Path is incorrect in Nitro +[ ] language: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[ ] shoutCount: { +[ ] type: Number, +[ ] default: 0, +[-] index: true + }, +[ ] meta: { +[ ] hasVideo: { +[ ] type: Boolean, +[ ] default: false + }, +[ ] embedds: { +[ ] type: Object, +[ ] default: {} + } + }, +[?] visibility: { +[X] type: String, +[X] enum: ['public', 'friends', 'private'], +[ ] default: 'public', // Default value is missing in Nitro +[-] index: true + }, +[?] isEnabled: { +[X] type: Boolean, +[ ] default: true, // Default value is missing in Nitro +[-] index: true + }, +[?] tags: { type: Array }, // ensure this is working properly +[ ] emotions: { +[ ] type: Object, +[-] index: true, +[ ] default: { +[ ] angry: { +[ ] count: 0, +[ ] percent: 0 +[ ] }, +[ ] cry: { +[ ] count: 0, +[ ] percent: 0 +[ ] }, +[ ] surprised: { +[ ] count: 0, +[ ] percent: 0 + }, +[ ] happy: { +[ ] count: 0, +[ ] percent: 0 + }, +[ ] funny: { +[ ] count: 0, +[ ] percent: 0 + } + } + }, +[?] deleted: { +[X] type: Boolean, +[ ] default: false, // Default value is missing in Nitro +[-] index: true + }, +[?] createdAt: { +[?] type: Date, // Type is modeled as string in Nitro which is incorrect +[ ] default: Date.now // Default value is missing in Nitro + }, +[?] updatedAt: { +[?] type: Date, // Type is modeled as string in Nitro which is incorrect +[ ] default: Date.now // Default value is missing in Nitro + }, +[ ] wasSeeded: { type: Boolean } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as post MERGE (p:Post {id: post._id["$oid"]}) ON CREATE SET p.title = post.title, diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql index bf3837b04..fac858a9a 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/follows.cql @@ -1,4 +1,36 @@ -CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as follow +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[?] userId: { +[-] type: String, +[ ] required: true, +[-] index: true + }, +[?] foreignId: { +[ ] type: String, +[ ] required: true, +[-] index: true + }, +[?] foreignService: { // db.getCollection('follows').distinct('foreignService') returns 'organizations' and 'users' +[ ] type: String, +[ ] required: true, +[ ] index: true + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } + index: +[?] { userId: 1, foreignId: 1, foreignService: 1 },{ unique: true } // is the unique constrain modeled? +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as follow MATCH (u1:User {id: follow.userId}), (u2:User {id: follow.foreignId}) MERGE (u1)-[:FOLLOWS]->(u2) ; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh index 12cc7ce67..470158642 100755 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/import.sh @@ -10,10 +10,10 @@ set +o allexport function import_collection () { for chunk in ${IMPORT_PATH}splits/$1/* do - mv $chunk ${IMPORT_CHUNK_PATH} - NEO4J_COMMAND="$(envsubst '${IMPORT_CHUNK_PATH_CQL}' < $(dirname "$0")/$1.cql)" + export IMPORT_CHUNK_PATH_CQL_FILE="${IMPORT_CHUNK_PATH_CQL}$1/$(basename "${chunk}")" + NEO4J_COMMAND="$(envsubst '${IMPORT_CHUNK_PATH_CQL_FILE}' < $(dirname "$0")/$1.cql)" echo "Import ${chunk}" - echo "${NEO4J_COMMAND}" | "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} + echo "${NEO4J_COMMAND}" | "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} > /dev/null done } @@ -22,7 +22,8 @@ SECONDS=0 # Delete all Neo4J Database content echo "Deleting Database Contents" -echo "MATCH (n) DETACH DELETE n;" | "${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} +"${IMPORT_CYPHERSHELL_BIN}" -u ${NEO4J_USERNAME} -p ${NEO4J_PASSWORD} < $(dirname "$0")/_delete_all.cql > /dev/null +echo "DONE" # Import Data echo "Start Importing Data" @@ -44,5 +45,6 @@ import_collection "comments" #import_collection "systemnotifications" #import_collection "userscandos" #import_collection "usersettings" +echo "DONE" echo "Time elapsed: $SECONDS seconds" diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql index a82a7a33d..d370b4b4a 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/shouts.cql @@ -1,4 +1,36 @@ -CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as shout +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[?] userId: { +[X] type: String, +[ ] required: true, // Not required in Nitro +[-] index: true + }, +[?] foreignId: { +[X] type: String, +[ ] required: true, // Not required in Nitro +[-] index: true + }, +[?] foreignService: { // db.getCollection('shots').distinct('foreignService') returns 'contributions' +[X] type: String, +[ ] required: true, // Not required in Nitro +[-] index: true + }, +[ ] createdAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] wasSeeded: { type: Boolean } + } + index: +[?] { userId: 1, foreignId: 1 },{ unique: true } // is the unique constrain modeled? +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as shout MATCH (u:User {id: shout.userId}), (p:Post {id: shout.foreignId}) MERGE (u)-[:SHOUTED]->(p) ; diff --git a/deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql b/deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql index 693fd75b6..aec5499fc 100644 --- a/deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql +++ b/deployment/legacy-migration/maintenance-worker/migration/neo4j/users.cql @@ -1,4 +1,101 @@ -CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL}") YIELD value as user +/* +// Alpha Model +// [ ] Not modeled in Nitro +// [X] Modeled in Nitro +// [-] Omitted in Nitro +// [?] Unclear / has work to be done for Nitro + { +[?] email: { +[X] type: String, +[-] index: true, +[X] required: true, +[?] unique: true //unique constrain missing in Nitro + }, +[?] password: { // Not required in Alpha -> verify if always present +[X] type: String + }, +[X] name: { type: String }, +[X] slug: { +[X] type: String, +[-] index: true + }, +[ ] gender: { type: String }, +[ ] followersCounts: { +[ ] users: { +[ ] type: Number, +[ ] default: 0 + }, +[ ] organizations: { +[ ] type: Number, +[ ] default: 0 + }, +[ ] projects: { +[ ] type: Number, +[ ] default: 0 + } + }, +[ ] followingCounts: { +[ ] users: { +[ ] type: Number, +[ ] default: 0 + }, +[ ] organizations: { +[ ] type: Number, +[ ] default: 0 + }, +[ ] projects: { +[ ] type: Number, +[ ] default: 0 + } + }, +[ ] timezone: { type: String }, +[?] avatar: { type: String }, // Path is incorrect in Nitro +[?] coverImg: { type: String }, // Path is incorrect in Nitro, was not modeled in latest Nitro - do we want this? +[ ] doiToken: { type: String }, +[ ] confirmedAt: { type: Date }, +[?] badgeIds: [], // Verify this is working properly +[?] deletedAt: { type: Date }, // The Date of deletion is not saved in Nitro +[?] createdAt: { +[?] type: Date, // Modeled as String in Nitro +[ ] default: Date.now // Default value is missing in Nitro + }, +[?] updatedAt: { +[?] type: Date, // Modeled as String in Nitro +[ ] default: Date.now // Default value is missing in Nitro + }, +[ ] lastActiveAt: { +[ ] type: Date, +[ ] default: Date.now + }, +[ ] isVerified: { type: Boolean }, +[?] role: { +[X] type: String, +[-] index: true, +[?] enum: ['admin', 'moderator', 'manager', 'editor', 'user'], // missing roles manager & editor in Nitro +[ ] default: 'user' // Default value is missing in Nitro + }, +[ ] verifyToken: { type: String }, +[ ] verifyShortToken: { type: String }, +[ ] verifyExpires: { type: Date }, +[ ] verifyChanges: { type: Object }, +[ ] resetToken: { type: String }, +[ ] resetShortToken: { type: String }, +[ ] resetExpires: { type: Date }, +[X] wasSeeded: { type: Boolean }, +[X] wasInvited: { type: Boolean }, +[ ] language: { +[ ] type: String, +[ ] default: 'en' + }, +[ ] termsAndConditionsAccepted: { type: Date }, // we display the terms and conditions on registration +[ ] systemNotificationsSeen: { +[ ] type: Array, +[ ] default: [] + } + } +*/ + +CALL apoc.load.json("file:${IMPORT_CHUNK_PATH_CQL_FILE}") YIELD value as user MERGE(u:User {id: user._id["$oid"]}) ON CREATE SET u.name = user.name, @@ -8,6 +105,7 @@ u.password = user.password, u.avatar = user.avatar, u.coverImg = user.coverImg, u.wasInvited = user.wasInvited, +u.wasSeeded = user.wasSeeded, u.role = toLower(user.role), u.createdAt = user.createdAt.`$date`, u.updatedAt = user.updatedAt.`$date`,