From fcc99ab58eb9cfe6a7ee7db2904369178e0cfba9 Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 10 Apr 2025 09:52:49 +0200 Subject: [PATCH 1/4] refactor(backend): clean migrate scripts (#8317) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * clean migrate scripts - refactor migrate:init - separate admin seed - separate categories seed - rework backend README regarding the database - remove `db:clean` command as its a duplicate of `db:reset` - remove `__migrate` helper alias * renamed clean.ts to reset.ts * set indices & constrains in init function * fix comment * disable migrations touching indices * remove obsolete comment * always run init on kubernetes * reset db with or without migrations * lint fixes * Refine 'README.md' * Refine more 'README.md' * fix lint --------- Co-authored-by: Wolfgang Huß --- backend/README.md | 188 +++++++----------- backend/package.json | 13 +- backend/src/db/admin.ts | 51 +++++ backend/src/db/categories.ts | 36 ++++ backend/src/db/factories.ts | 19 +- backend/src/db/migrate/store.ts | 119 +++-------- .../20200207080200-fulltext_index_for_tags.ts | 6 + ...text_indices_and_unique_keys_for_groups.ts | 7 + .../20230320130345-fulltext-search-indexes.ts | 6 + backend/src/db/reset-with-migrations.ts | 20 ++ backend/src/db/{clean.ts => reset.ts} | 0 .../templates/backend/stateful-set.yaml | 2 +- 12 files changed, 249 insertions(+), 218 deletions(-) create mode 100644 backend/src/db/admin.ts create mode 100644 backend/src/db/categories.ts create mode 100644 backend/src/db/reset-with-migrations.ts rename backend/src/db/{clean.ts => reset.ts} (100%) diff --git a/backend/README.md b/backend/README.md index 8fc05779e..bfc875d95 100644 --- a/backend/README.md +++ b/backend/README.md @@ -6,12 +6,12 @@ Run the following command to install everything through docker. The installation takes a bit longer on the first pass or on rebuild ... -```bash +```sh # in main folder -$ docker-compose up +$ docker compose up # or # rebuild the containers for a cleanup -$ docker-compose up --build +$ docker compose up --build ``` Wait a little until your backend is up and running at [http://localhost:4000/](http://localhost:4000/). @@ -26,7 +26,7 @@ some known problems with more recent node versions). You can use the [node version manager](https://github.com/nvm-sh/nvm) `nvm` to switch between different local Node versions: -```bash +```sh # install Node $ cd backend $ nvm install v20.12.1 @@ -35,7 +35,7 @@ $ nvm use v20.12.1 Install node dependencies with [yarn](https://yarnpkg.com/en/): -```bash +```sh # in main folder $ cd backend $ yarn install @@ -47,7 +47,7 @@ $ nvm use && yarn Copy Environment Variables: -```bash +```sh # in backend/ $ cp .env.template .env ``` @@ -57,14 +57,14 @@ a [local Neo4J](http://localhost:7474) instance is up and running. Start the backend for development with: -```bash +```sh # in backend/ $ yarn run dev ``` or start the backend in production environment with: -```bash +```sh # in backend/ $ yarn run start ``` @@ -79,154 +79,120 @@ More details about our GraphQL playground and how to use it with ocelot.social c ![GraphQL Playground](../.gitbook/assets/graphql-playground.png) -### Database Indexes and Constraints +## Database -Database indexes and constraints need to be created and upgraded when the database and the backend are running: +A fresh database needs to be initialized and migrated. -::: tabs -@tab:active Docker - -```bash -# in main folder while docker-compose is running -$ docker exec backend yarn run db:migrate init - -# only once: init admin user and create indexes and constraints in Neo4j database -# for development -$ docker compose exec backend yarn prod:migrate init -# in production mode use command -$ docker compose exec backend /bin/sh -c "yarn prod:migrate init" +```sh +# in folder backend while database is running +yarn db:migrate init +# for docker environments: +docker exec backend yarn db:migrate init +# for docker production: +docker exec backend yarn prod:migrate init ``` -```bash -# in main folder with docker compose running -$ docker exec backend yarn run db:migrate up +```sh +# in backend with database running (In docker or local) +yarn db:migrate up + +# for docker development: +docker exec backend yarn db:migrate up +# for docker production +docker exec backend yarn prod:migrate up ``` -@tab Without Docker +### Optional Data -```bash -# in folder backend/ while database is running -# make sure your database is running on http://localhost:7474/browser/ -yarn run db:migrate init +You can seed some optional data into the database. + +To create the default admin with password `1234` use: + +```sh +# in backend with database running (In docker or local) +yarn db:data:admin ``` -```bash -# in backend/ with database running (In docker or local) -yarn run db:migrate up +When using `CATEGORIES_ACTIVE=true` you also want to seed the categories with: + +```sh +# in backend with database running (In docker or local) +yarn db:data:categories ``` -::: +### Seed Data -#### Seed Database +For a predefined set of test data you can seed the database with: -If you want your backend to return anything else than an empty response, you -need to seed your database: +```sh +# in backend with database running (In docker or local) +yarn db:seed -::: tabs -@tab:active Docker - -In another terminal run: - -```bash -# in main folder while docker-compose is running -$ docker exec backend yarn run db:seed +# for docker +docker exec backend yarn db:seed ``` -To reset the database run: +### Reset Data -```bash -# in main folder while docker-compose is running -$ docker exec backend yarn run db:reset +In order to reset the database you can run: + +```sh +# in backend with database running (In docker or local) +yarn db:reset +# or deleting the migrations as well +yarn db:reset:withmigrations + +# for docker +docker exec backend yarn db:reset +# or deleting the migrations as well +docker exec backend yarn db:reset:withmigrations # you could also wipe out your neo4j database and delete all volumes with: -$ docker-compose down -v -# if container is not running, run this command to set up your database indexes and constraints -$ docker exec backend yarn run db:migrate init -# And then upgrade the indexes and const -$ docker exec backend yarn run db:migrate up +docker compose down -v ``` -@tab Without Docker - -Run: - -```bash -# in backend/ while database is running -$ yarn run db:seed -``` - -To reset the database run: - -```bash -# in backend/ while database is running -$ yarn run db:reset -``` - -::: +> Note: This just deletes the data and not the constraints, hence you do not need to rerun `yarn db:migrate init` or `yarn db:migrate up`. ### Data migrations Although Neo4J is schema-less,you might find yourself in a situation in which you have to migrate your data e.g. because your data modeling has changed. -::: tabs -@tab:active Docker - Generate a data migration file: -```bash -# in main folder while docker-compose is running -$ docker-compose exec backend yarn run db:migrate:create your_data_migration -# Edit the file in ./src/db/migrations/ -``` - -To run the migration: - -```bash -# in main folder while docker-compose is running -$ docker exec backend yarn run db:migrate up -``` - -@tab Without Docker - -Generate a data migration file: - -```bash -# in backend/ +```sh +# in backend $ yarn run db:migrate:create your_data_migration # Edit the file in ./src/db/migrations/ + +# for docker +# in main folder while docker compose is running +$ docker compose exec backend yarn run db:migrate:create your_data_migration +# Edit the file in ./src/db/migrations/ ``` To run the migration: -```bash +```sh # in backend/ while database is running $ yarn run db:migrate up -``` -::: +# for docker +# in main folder while docker compose is running +$ docker exec backend yarn run db:migrate up +``` ## Testing **Beware**: We have no multiple database setup at the moment. We clean the database after each test, running the tests will wipe out all your data! -::: tabs -@tab:active Docker - Run the unit tests: -```bash -# in main folder while docker-compose is running -$ docker exec backend yarn run test -``` - -@tab Without Docker - -Run the unit tests: - -```bash +```sh # in backend/ while database is running $ yarn run test -``` -::: +# for docker +# in main folder while docker compose is running +$ docker exec backend yarn run test +``` diff --git a/backend/package.json b/backend/package.json index f093fff62..3362fe598 100644 --- a/backend/package.json +++ b/backend/package.json @@ -8,19 +8,20 @@ "private": false, "main": "src/index.ts", "scripts": { - "__migrate": "migrate --compiler 'ts:./src/db/compiler.ts' --migrations-dir ./src/db/migrations", - "prod:migrate": "migrate --migrations-dir ./build/src/db/migrations --store ./build/src/db/migrate/store.js", "start": "node build/src/", "build": "tsc && tsc-alias && ./scripts/build.copy.files.sh", "dev": "nodemon --exec ts-node --require tsconfig-paths/register src/ -e js,ts,gql", "dev:debug": "nodemon --exec babel-node --inspect=0.0.0.0:9229 src/ -e js,ts,gql", "lint": "eslint --max-warnings=0 --ext .js,.ts ./src", "test": "cross-env NODE_ENV=test NODE_OPTIONS=--max-old-space-size=8192 jest --runInBand --coverage --forceExit --detectOpenHandles", - "db:clean": "ts-node --require tsconfig-paths/register src/db/clean.ts", - "db:reset": "yarn run db:clean", + "db:reset": "ts-node --require tsconfig-paths/register src/db/reset.ts", + "db:reset:withmigrations": "ts-node --require tsconfig-paths/register src/db/reset-with-migrations.ts", "db:seed": "ts-node --require tsconfig-paths/register src/db/seed.ts", - "db:migrate": "yarn run __migrate --store ./src/db/migrate/store.ts", - "db:migrate:create": "yarn run __migrate --template-file ./src/db/migrate/template.ts --date-format 'yyyymmddHHmmss' create" + "db:data:admin": "ts-node --require tsconfig-paths/register src/db/admin.ts", + "db:data:categories": "ts-node --require tsconfig-paths/register src/db/categories.ts", + "db:migrate": "migrate --compiler 'ts:./src/db/compiler.ts' --migrations-dir ./src/db/migrations --store ./src/db/migrate/store.ts", + "db:migrate:create": "migrate --compiler 'ts:./src/db/compiler.ts' --migrations-dir ./src/db/migrations --template-file ./src/db/migrate/template.ts --date-format 'yyyymmddHHmmss' create", + "prod:migrate": "migrate --migrations-dir ./build/src/db/migrations --store ./build/src/db/migrate/store.js" }, "dependencies": { "@babel/cli": "~7.27.0", diff --git a/backend/src/db/admin.ts b/backend/src/db/admin.ts new file mode 100644 index 000000000..ce6824c0a --- /dev/null +++ b/backend/src/db/admin.ts @@ -0,0 +1,51 @@ +import { hashSync } from 'bcryptjs' +import { v4 as uuid } from 'uuid' + +import { getDriver } from './neo4j' + +const defaultAdmin = { + email: 'admin@example.org', + password: hashSync('1234', 10), + name: 'admin', + id: uuid(), + slug: 'admin', +} + +const createDefaultAdminUser = async () => { + const driver = getDriver() + const session = driver.session() + const createAdminTxResultPromise = session.writeTransaction(async (txc) => { + txc.run( + `MERGE (e:EmailAddress { + email: "${defaultAdmin.email}", + createdAt: toString(datetime()) + })-[:BELONGS_TO]->(u:User { + name: "${defaultAdmin.name}", + encryptedPassword: "${defaultAdmin.password}", + role: "admin", + id: "${defaultAdmin.id}", + slug: "${defaultAdmin.slug}", + createdAt: toString(datetime()), + allowEmbedIframes: false, + showShoutsPublicly: false, + sendNotificationEmails: true, + deleted: false, + disabled: false + })-[:PRIMARY_EMAIL]->(e)`, + ) + }) + try { + await createAdminTxResultPromise + console.log('Successfully created default admin user!') // eslint-disable-line no-console + // eslint-disable-next-line no-catch-all/no-catch-all + } catch (error) { + console.log(error) // eslint-disable-line no-console + } finally { + session.close() + driver.close() + } +} + +;(async function () { + await createDefaultAdminUser() +})() diff --git a/backend/src/db/categories.ts b/backend/src/db/categories.ts new file mode 100644 index 000000000..f550c4d94 --- /dev/null +++ b/backend/src/db/categories.ts @@ -0,0 +1,36 @@ +import { categories } from '@constants/categories' + +import { getDriver } from './neo4j' + +const createCategories = async () => { + const driver = getDriver() + const session = driver.session() + const createCategoriesTxResultPromise = session.writeTransaction(async (txc) => { + categories.forEach(({ icon, name }, index) => { + const id = `cat${index + 1}` + txc.run( + `MERGE (c:Category { + icon: "${icon}", + slug: "${name}", + name: "${name}", + id: "${id}", + createdAt: toString(datetime()) + })`, + ) + }) + }) + try { + await createCategoriesTxResultPromise + console.log('Successfully created categories!') // eslint-disable-line no-console + // eslint-disable-next-line no-catch-all/no-catch-all + } catch (error) { + console.log(`Error creating categories: ${error}`) // eslint-disable-line no-console + } finally { + session.close() + driver.close() + } +} + +;(async function () { + await createCategories() +})() diff --git a/backend/src/db/factories.ts b/backend/src/db/factories.ts index e09a4f921..136f0fe50 100644 --- a/backend/src/db/factories.ts +++ b/backend/src/db/factories.ts @@ -17,18 +17,19 @@ const uniqueImageUrl = (imageUrl) => { return newUrl.toString() } -export const cleanDatabase = async (options: any = {}) => { - const { driver = getDriver() } = options +export const cleanDatabase = async ({ withMigrations } = { withMigrations: false }) => { + const driver = getDriver() const session = driver.session() + + const clean = ` + MATCH (everything) + ${withMigrations ? '' : "WHERE NOT 'Migration' IN labels(everything)"} + DETACH DELETE everything + ` + try { await session.writeTransaction((transaction) => { - return transaction.run( - ` - MATCH (everything) - WHERE NOT 'Migration' IN labels(everything) - DETACH DELETE everything - `, - ) + return transaction.run(clean) }) } finally { session.close() diff --git a/backend/src/db/migrate/store.ts b/backend/src/db/migrate/store.ts index e373c41c0..aa8bd66d1 100644 --- a/backend/src/db/migrate/store.ts +++ b/backend/src/db/migrate/store.ts @@ -1,108 +1,45 @@ -import { hashSync } from 'bcryptjs' -import { v4 as uuid } from 'uuid' - -import CONFIG from '@config/index' -import { categories } from '@constants/categories' import { getDriver, getNeode } from '@db/neo4j' -const defaultAdmin = { - email: 'admin@example.org', - password: hashSync('1234', 10), - name: 'admin', - id: uuid(), - slug: 'admin', -} - -const createCategories = async (session) => { - const createCategoriesTxResultPromise = session.writeTransaction(async (txc) => { - categories.forEach(({ icon, name }, index) => { - const id = `cat${index + 1}` - txc.run( - `MERGE (c:Category { - icon: "${icon}", - slug: "${name}", - name: "${name}", - id: "${id}", - createdAt: toString(datetime()) - })`, - ) - }) - }) - try { - await createCategoriesTxResultPromise - console.log('Successfully created categories!') // eslint-disable-line no-console - // eslint-disable-next-line no-catch-all/no-catch-all - } catch (error) { - console.log(`Error creating categories: ${error}`) // eslint-disable-line no-console - } -} - -const createDefaultAdminUser = async (session) => { - const readTxResultPromise = session.readTransaction(async (txc) => { - const result = await txc.run('MATCH (user:User) RETURN count(user) AS userCount') - return result.records.map((r) => r.get('userCount')) - }) - let createAdmin = false - try { - const userCount = parseInt(String(await readTxResultPromise)) - if (userCount === 0) createAdmin = true - // eslint-disable-next-line no-catch-all/no-catch-all - } catch (error) { - console.log(error) // eslint-disable-line no-console - } - if (createAdmin) { - const createAdminTxResultPromise = session.writeTransaction(async (txc) => { - txc.run( - `MERGE (e:EmailAddress { - email: "${defaultAdmin.email}", - createdAt: toString(datetime()) - })-[:BELONGS_TO]->(u:User { - name: "${defaultAdmin.name}", - encryptedPassword: "${defaultAdmin.password}", - role: "admin", - id: "${defaultAdmin.id}", - slug: "${defaultAdmin.slug}", - createdAt: toString(datetime()), - allowEmbedIframes: false, - showShoutsPublicly: false, - deleted: false, - disabled: false - })-[:PRIMARY_EMAIL]->(e)`, - ) - }) - try { - await createAdminTxResultPromise - console.log('Successfully created default admin user!') // eslint-disable-line no-console - // eslint-disable-next-line no-catch-all/no-catch-all - } catch (error) { - console.log(error) // eslint-disable-line no-console - } - } -} - class Store { - async init(next) { + async init(errFn) { const neode = getNeode() - const { driver } = neode - const session = driver.session() - await createDefaultAdminUser(session) - if (CONFIG.CATEGORIES_ACTIVE) await createCategories(session) - const writeTxResultPromise = session.writeTransaction(async (txc) => { - await txc.run('CALL apoc.schema.assert({},{},true)') // drop all indices and constraints + const session = neode.driver.session() + const txFreshIndicesConstrains = session.writeTransaction(async (txc) => { + // drop all indices and constraints + await txc.run('CALL apoc.schema.assert({},{},true)') + /* + ############################################# + # ADD YOUR CUSTOM INDICES & CONSTRAINS HERE # + ############################################# + */ + // Search indexes (also part of migration 20230320130345-fulltext-search-indexes) + await txc.run( + `CALL db.index.fulltext.createNodeIndex("user_fulltext_search",["User"],["name", "slug"])`, + ) + await txc.run( + `CALL db.index.fulltext.createNodeIndex("post_fulltext_search",["Post"],["title", "content"])`, + ) + await txc.run(`CALL db.index.fulltext.createNodeIndex("tag_fulltext_search",["Tag"],["id"])`) // also part of migration 20200207080200-fulltext_index_for_tags + // Search indexes (also part of migration 20220803060819-create_fulltext_indices_and_unique_keys_for_groups) + await txc.run(` + CALL db.index.fulltext.createNodeIndex("group_fulltext_search",["Group"],["name", "slug", "about", "description"]) + `) }) try { - await writeTxResultPromise + // Due to limitations of neode in combination with the limitations of the community version of neo4j + // we need to have all constraints and indexes defined here. They can not be properly migrated + await txFreshIndicesConstrains + await getNeode().schema.install() // eslint-disable-next-line no-console console.log('Successfully created database indices and constraints!') - next() // eslint-disable-next-line no-catch-all/no-catch-all } catch (error) { console.log(error) // eslint-disable-line no-console - next(error, null) + errFn(error) } finally { session.close() - driver.close() + neode.driver.close() } } diff --git a/backend/src/db/migrations-examples/20200207080200-fulltext_index_for_tags.ts b/backend/src/db/migrations-examples/20200207080200-fulltext_index_for_tags.ts index 8eee22318..79b46a1ff 100644 --- a/backend/src/db/migrations-examples/20200207080200-fulltext_index_for_tags.ts +++ b/backend/src/db/migrations-examples/20200207080200-fulltext_index_for_tags.ts @@ -9,10 +9,13 @@ export async function up(next) { const transaction = session.beginTransaction() try { + // We do do this in /src/db/migrate/store.ts + /* await transaction.run(` CALL db.index.fulltext.createNodeIndex("tag_fulltext_search",["Tag"],["id"]) `) await transaction.commit() + */ next() } catch (error) { const { message } = error @@ -39,10 +42,13 @@ export async function down(next) { try { // Implement your migration here. + // We do do this in /src/db/migrate/store.ts + /* await transaction.run(` CALL db.index.fulltext.drop("tag_fulltext_search") `) await transaction.commit() + */ next() } catch (error) { // eslint-disable-next-line no-console diff --git a/backend/src/db/migrations/20220803060819-create_fulltext_indices_and_unique_keys_for_groups.ts b/backend/src/db/migrations/20220803060819-create_fulltext_indices_and_unique_keys_for_groups.ts index 08dc558fb..c53edb9a0 100644 --- a/backend/src/db/migrations/20220803060819-create_fulltext_indices_and_unique_keys_for_groups.ts +++ b/backend/src/db/migrations/20220803060819-create_fulltext_indices_and_unique_keys_for_groups.ts @@ -18,9 +18,13 @@ export async function up(next) { // await transaction.run(` // CREATE CONSTRAINT ON ( group:Group ) ASSERT group.slug IS UNIQUE // `) + + // We do do this in /src/db/migrate/store.ts + /* await transaction.run(` CALL db.index.fulltext.createNodeIndex("group_fulltext_search",["Group"],["name", "slug", "about", "description"]) `) + */ await transaction.commit() next() } catch (error) { @@ -42,6 +46,8 @@ export async function down(next) { try { // Implement your migration here. + // We do do this in /src/db/migrate/store.ts + /* await transaction.run(` DROP CONSTRAINT ON ( group:Group ) ASSERT group.id IS UNIQUE `) @@ -52,6 +58,7 @@ export async function down(next) { CALL db.index.fulltext.drop("group_fulltext_search") `) await transaction.commit() + */ next() } catch (error) { // eslint-disable-next-line no-console diff --git a/backend/src/db/migrations/20230320130345-fulltext-search-indexes.ts b/backend/src/db/migrations/20230320130345-fulltext-search-indexes.ts index 2239d6d06..765042aad 100644 --- a/backend/src/db/migrations/20230320130345-fulltext-search-indexes.ts +++ b/backend/src/db/migrations/20230320130345-fulltext-search-indexes.ts @@ -8,6 +8,8 @@ export async function up(next) { const transaction = session.beginTransaction() try { + // We do do this in /src/db/migrate/store.ts + /* // Drop indexes if they exist because due to legacy code they might be set already const indexesResponse = await transaction.run(`CALL db.indexes()`) const indexes = indexesResponse.records.map((record) => record.get('name')) @@ -31,6 +33,7 @@ export async function up(next) { `CALL db.index.fulltext.createNodeIndex("tag_fulltext_search",["Tag"],["id"])`, ) await transaction.commit() + */ next() } catch (error) { // eslint-disable-next-line no-console @@ -50,10 +53,13 @@ export async function down(next) { const transaction = session.beginTransaction() try { + // We do do this in /src/db/migrate/store.ts + /* await transaction.run(`CALL db.index.fulltext.drop("user_fulltext_search")`) await transaction.run(`CALL db.index.fulltext.drop("post_fulltext_search")`) await transaction.run(`CALL db.index.fulltext.drop("tag_fulltext_search")`) await transaction.commit() + */ next() } catch (error) { // eslint-disable-next-line no-console diff --git a/backend/src/db/reset-with-migrations.ts b/backend/src/db/reset-with-migrations.ts new file mode 100644 index 000000000..fc3d86b09 --- /dev/null +++ b/backend/src/db/reset-with-migrations.ts @@ -0,0 +1,20 @@ +/* eslint-disable n/no-process-exit */ +import CONFIG from '@config/index' + +import { cleanDatabase } from './factories' + +if (CONFIG.PRODUCTION && !CONFIG.PRODUCTION_DB_CLEAN_ALLOW) { + throw new Error(`You cannot clean the database in a non-staging and real production environment!`) +} + +;(async function () { + try { + await cleanDatabase({ withMigrations: true }) + console.log('Successfully deleted all nodes and relations including!') // eslint-disable-line no-console + process.exit(0) + // eslint-disable-next-line no-catch-all/no-catch-all + } catch (err) { + console.log(`Error occurred deleting the nodes and relations (reset the db)\n\n${err}`) // eslint-disable-line no-console + process.exit(1) + } +})() diff --git a/backend/src/db/clean.ts b/backend/src/db/reset.ts similarity index 100% rename from backend/src/db/clean.ts rename to backend/src/db/reset.ts diff --git a/deployment/helm/charts/ocelot-social/templates/backend/stateful-set.yaml b/deployment/helm/charts/ocelot-social/templates/backend/stateful-set.yaml index 98eb3fcad..618a99f7f 100644 --- a/deployment/helm/charts/ocelot-social/templates/backend/stateful-set.yaml +++ b/deployment/helm/charts/ocelot-social/templates/backend/stateful-set.yaml @@ -18,7 +18,7 @@ spec: - name: {{ .Release.Name }}-backend-migrations image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag | default (include "defaultTag" .) }}" imagePullPolicy: {{ quote .Values.global.image.pullPolicy }} - command: ["/bin/sh", "-c", "yarn prod:migrate up"] + command: ["/bin/sh", "-c", "yarn prod:migrate init && yarn prod:migrate up"] {{- include "resources" .Values.backend.resources | indent 10 }} envFrom: - configMapRef: From ab6fe37c3e1c3c18b19f1d200245860e3b7a154a Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 10 Apr 2025 18:38:51 +0200 Subject: [PATCH 2/4] refactor(backend): new chat message notification email (#8357) * new chat message notification email - new mail features name in subject and text * fix english version * typo * fix typos * adjust tests --- .../helpers/email/templateBuilder.spec.ts | 16 +++++++++---- .../helpers/email/templateBuilder.ts | 17 +++++++++---- .../helpers/email/templates/chatMessage.html | 8 +++---- .../middleware/helpers/isUserOnline.spec.ts | 24 +++++++++---------- .../src/middleware/helpers/isUserOnline.ts | 6 ++--- .../notifications/notificationsMiddleware.ts | 15 ++++++------ 6 files changed, 49 insertions(+), 37 deletions(-) diff --git a/backend/src/middleware/helpers/email/templateBuilder.spec.ts b/backend/src/middleware/helpers/email/templateBuilder.spec.ts index 48e8b4c99..9dbfca91f 100644 --- a/backend/src/middleware/helpers/email/templateBuilder.spec.ts +++ b/backend/src/middleware/helpers/email/templateBuilder.spec.ts @@ -39,7 +39,12 @@ const resetPasswordTemplateData = () => ({ const chatMessageTemplateData = { email: 'test@example.org', variables: { - name: 'Mr Example', + senderUser: { + name: 'Sender', + }, + recipientUser: { + name: 'Recipient', + }, }, } const wrongAccountTemplateData = () => ({ @@ -174,10 +179,10 @@ describe('templateBuilder', () => { describe('chatMessageTemplate', () => { describe('multi language', () => { it('e-mail is build with all data', () => { - const subject = 'Neue Chatnachricht | New chat message' + const subject = `Neue Chat-Nachricht | New chat message - ${chatMessageTemplateData.variables.senderUser.name}` const actionUrl = new URL('/chat', CONFIG.CLIENT_URI).toString() - const enContent = 'You have received a new chat message.' - const deContent = 'Du hast eine neue Chatnachricht erhalten.' + const enContent = `You have received a new chat message from ${chatMessageTemplateData.variables.senderUser.name}.` + const deContent = `Du hast eine neue Chat-Nachricht von ${chatMessageTemplateData.variables.senderUser.name} erhalten.` testEmailData(null, chatMessageTemplate, chatMessageTemplateData, [ ...textsStandard, { @@ -187,7 +192,8 @@ describe('templateBuilder', () => { }, englishHint, actionUrl, - chatMessageTemplateData.variables.name, + chatMessageTemplateData.variables.senderUser, + chatMessageTemplateData.variables.recipientUser, enContent, deContent, supportUrl, diff --git a/backend/src/middleware/helpers/email/templateBuilder.ts b/backend/src/middleware/helpers/email/templateBuilder.ts index c091bf1f8..bd44716fe 100644 --- a/backend/src/middleware/helpers/email/templateBuilder.ts +++ b/backend/src/middleware/helpers/email/templateBuilder.ts @@ -1,9 +1,9 @@ /* eslint-disable import/no-namespace */ import mustache from 'mustache' -import logosWebapp from '@config//logos' -import metadata from '@config//metadata' import CONFIG from '@config/index' +import logosWebapp from '@config/logos' +import metadata from '@config/metadata' import * as templates from './templates' import * as templatesDE from './templates/de' @@ -73,10 +73,17 @@ export const resetPasswordTemplate = ({ email, variables: { nonce, name } }) => } } -export const chatMessageTemplate = ({ email, variables: { name } }) => { - const subject = 'Neue Chatnachricht | New chat message' +export const chatMessageTemplate = ({ email, variables: { senderUser, recipientUser } }) => { + const subject = `Neue Chat-Nachricht | New chat message - ${senderUser.name}` const actionUrl = new URL('/chat', CONFIG.CLIENT_URI) - const renderParams = { ...defaultParams, englishHint, actionUrl, name, subject } + const renderParams = { + ...defaultParams, + subject, + englishHint, + actionUrl, + senderUser, + recipientUser, + } return { from, diff --git a/backend/src/middleware/helpers/email/templates/chatMessage.html b/backend/src/middleware/helpers/email/templates/chatMessage.html index 0b1bacb08..49fc69bf2 100644 --- a/backend/src/middleware/helpers/email/templates/chatMessage.html +++ b/backend/src/middleware/helpers/email/templates/chatMessage.html @@ -23,8 +23,8 @@ style="padding: 20px; padding-top: 0; font-family: Lato, sans-serif; font-size: 16px; line-height: 22px; color: #555555;">

- Hallo {{ name }}!

-

Du hast eine neue Chatnachricht erhalten.

+ Hallo {{ recipientUser.name }}! +

Du hast eine neue Chat-Nachricht von {{ senderUser.name }} erhalten.

@@ -78,8 +78,8 @@ style="padding: 20px; padding-top: 0; font-family: Lato, sans-serif; font-size: 16px; line-height: 22px; color: #555555;">

- Hello {{ name }}!

-

You have received a new chat message.

+ Hello {{ recipientUser.name }}! +

You have received a new chat message from {{ senderUser.name }}.

diff --git a/backend/src/middleware/helpers/isUserOnline.spec.ts b/backend/src/middleware/helpers/isUserOnline.spec.ts index bf2cb8d17..62ed17f79 100644 --- a/backend/src/middleware/helpers/isUserOnline.spec.ts +++ b/backend/src/middleware/helpers/isUserOnline.spec.ts @@ -5,35 +5,33 @@ let user describe('isUserOnline', () => { beforeEach(() => { user = { - properties: { - lastActiveAt: null, - awaySince: null, - lastOnlineStatus: null, - }, + lastActiveAt: null, + awaySince: null, + lastOnlineStatus: null, } }) describe('user has lastOnlineStatus `online`', () => { it('returns true if he was active within the last 90 seconds', () => { - user.properties.lastOnlineStatus = 'online' - user.properties.lastActiveAt = new Date() + user.lastOnlineStatus = 'online' + user.lastActiveAt = new Date() expect(isUserOnline(user)).toBe(true) }) it('returns false if he was not active within the last 90 seconds', () => { - user.properties.lastOnlineStatus = 'online' - user.properties.lastActiveAt = new Date().getTime() - 90001 + user.lastOnlineStatus = 'online' + user.lastActiveAt = new Date().getTime() - 90001 expect(isUserOnline(user)).toBe(false) }) }) describe('user has lastOnlineStatus `away`', () => { it('returns true if he went away less then 180 seconds ago', () => { - user.properties.lastOnlineStatus = 'away' - user.properties.awaySince = new Date() + user.lastOnlineStatus = 'away' + user.awaySince = new Date() expect(isUserOnline(user)).toBe(true) }) it('returns false if he went away more then 180 seconds ago', () => { - user.properties.lastOnlineStatus = 'away' - user.properties.awaySince = new Date().getTime() - 180001 + user.lastOnlineStatus = 'away' + user.awaySince = new Date().getTime() - 180001 expect(isUserOnline(user)).toBe(false) }) }) diff --git a/backend/src/middleware/helpers/isUserOnline.ts b/backend/src/middleware/helpers/isUserOnline.ts index 679953f81..23ddeb0dc 100644 --- a/backend/src/middleware/helpers/isUserOnline.ts +++ b/backend/src/middleware/helpers/isUserOnline.ts @@ -1,9 +1,9 @@ export const isUserOnline = (user) => { // Is Recipient considered online - const lastActive = new Date(user.properties.lastActiveAt).getTime() - const awaySince = new Date(user.properties.awaySince).getTime() + const lastActive = new Date(user.lastActiveAt).getTime() + const awaySince = new Date(user.awaySince).getTime() const now = new Date().getTime() - const status = user.properties.lastOnlineStatus + const status = user.lastOnlineStatus if ( // online & last active less than 1.5min -> online (status === 'online' && now - lastActive < 90000) || diff --git a/backend/src/middleware/notifications/notificationsMiddleware.ts b/backend/src/middleware/notifications/notificationsMiddleware.ts index faf4fd994..d24ddc8ef 100644 --- a/backend/src/middleware/notifications/notificationsMiddleware.ts +++ b/backend/src/middleware/notifications/notificationsMiddleware.ts @@ -355,12 +355,12 @@ const handleCreateMessage = async (resolve, root, args, context, resolveInfo) => const session = context.driver.session() const messageRecipient = session.readTransaction(async (transaction) => { const messageRecipientCypher = ` - MATCH (currentUser:User { id: $currentUserId })-[:CHATS_IN]->(room:Room { id: $roomId }) + MATCH (senderUser:User { id: $currentUserId })-[:CHATS_IN]->(room:Room { id: $roomId }) MATCH (room)<-[:CHATS_IN]-(recipientUser:User)-[:PRIMARY_EMAIL]->(emailAddress:EmailAddress) WHERE NOT recipientUser.id = $currentUserId - AND NOT (recipientUser)-[:BLOCKED]-(currentUser) + AND NOT (recipientUser)-[:BLOCKED]-(senderUser) AND NOT recipientUser.emailNotificationsChatMessage = false - RETURN recipientUser, emailAddress {.email} + RETURN senderUser {.*}, recipientUser {.*}, emailAddress {.email} ` const txResponse = await transaction.run(messageRecipientCypher, { currentUserId, @@ -368,18 +368,19 @@ const handleCreateMessage = async (resolve, root, args, context, resolveInfo) => }) return { - user: await txResponse.records.map((record) => record.get('recipientUser'))[0], + senderUser: await txResponse.records.map((record) => record.get('senderUser'))[0], + recipientUser: await txResponse.records.map((record) => record.get('recipientUser'))[0], email: await txResponse.records.map((record) => record.get('emailAddress'))[0]?.email, } }) try { // Execute Query - const { user, email } = await messageRecipient + const { senderUser, recipientUser, email } = await messageRecipient // Send EMail if we found a user(not blocked) and he is not considered online - if (user && !isUserOnline(user)) { - void sendMail(chatMessageTemplate({ email, variables: { name: user.properties.name } })) + if (recipientUser && !isUserOnline(recipientUser)) { + void sendMail(chatMessageTemplate({ email, variables: { senderUser, recipientUser } })) } // Return resolver result to client From abd13a6cff650325b1069a8794194d9126a2a25f Mon Sep 17 00:00:00 2001 From: Moriz Wahl Date: Thu, 10 Apr 2025 19:36:26 +0200 Subject: [PATCH 3/4] migrate commenting users to observe commented posts (#8308) --- .../20250331140313-commenter-observes-post.ts | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 backend/src/db/migrations/20250331140313-commenter-observes-post.ts diff --git a/backend/src/db/migrations/20250331140313-commenter-observes-post.ts b/backend/src/db/migrations/20250331140313-commenter-observes-post.ts new file mode 100644 index 000000000..cc9a82160 --- /dev/null +++ b/backend/src/db/migrations/20250331140313-commenter-observes-post.ts @@ -0,0 +1,62 @@ +import { getDriver } from '@db/neo4j' + +export const description = ` +All users commenting a post observe the post. +` + +export async function up(next) { + const driver = getDriver() + const session = driver.session() + const transaction = session.beginTransaction() + + try { + // Implement your migration here. + await transaction.run(` + MATCH (commenter:User)-[:WROTE]->(:Comment)-[:COMMENTS]->(post:Post) + MERGE (commenter)-[obs:OBSERVES]->(post) + ON CREATE SET + obs.active = true, + obs.createdAt = toString(datetime()), + obs.updatedAt = toString(datetime()) + RETURN post + `) + await transaction.commit() + next() + } catch (error) { + // eslint-disable-next-line no-console + console.log(error) + await transaction.rollback() + // eslint-disable-next-line no-console + console.log('rolled back') + throw new Error(error) + } finally { + session.close() + } +} + +export async function down(next) { + const driver = getDriver() + const session = driver.session() + const transaction = session.beginTransaction() + + try { + // Implement your migration here. + await transaction.run(` + MATCH (u:User)-[obs:OBSERVES]->(p:Post)<-[:COMMENTS]-(:Comment)<-[:WROTE]-(u) + WHERE NOT (u)-[:WROTE]->(post) + DELETE obs + RETURN p + `) + await transaction.commit() + next() + } catch (error) { + // eslint-disable-next-line no-console + console.log(error) + await transaction.rollback() + // eslint-disable-next-line no-console + console.log('rolled back') + throw new Error(error) + } finally { + session.close() + } +} From 711061b0c06500085cabecfc34026391444dc07e Mon Sep 17 00:00:00 2001 From: Ulf Gebhardt Date: Thu, 10 Apr 2025 20:17:51 +0200 Subject: [PATCH 4/4] fix(backend): error when there is an abandoned email (#8315) * fix error when there is an abandoned email We check beforehand if an email has an user attached, but we never delete an email without user. This will violate a unique constraint blocking that particular email from ever being used again. * fix tests --- backend/src/schema/resolvers/emails.spec.ts | 22 +++++++++++++++++++-- backend/src/schema/resolvers/emails.ts | 2 ++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/backend/src/schema/resolvers/emails.spec.ts b/backend/src/schema/resolvers/emails.spec.ts index c594f99f7..63141a3fc 100644 --- a/backend/src/schema/resolvers/emails.spec.ts +++ b/backend/src/schema/resolvers/emails.spec.ts @@ -294,9 +294,9 @@ describe('VerifyEmailAddress', () => { await expect(email).toBe(false) }) - describe('Edge case: In the meantime someone created an `EmailAddress` node with the given email', () => { + describe('Edge case: In the meantime someone created an `EmailAddress` node with the given email belonging to a user', () => { beforeEach(async () => { - await Factory.build('emailAddress', { email: 'to-be-verified@example.org' }) + await Factory.build('user', { id: '568' }, { email: 'to-be-verified@example.org' }) }) it('throws UserInputError because of unique constraints', async () => { @@ -306,6 +306,24 @@ describe('VerifyEmailAddress', () => { }) }) }) + + describe('Edge case: We have an abandoned `EmailAddress` node with the given email', () => { + beforeEach(async () => { + await Factory.build('emailAddress', { email: 'to-be-verified@example.org' }) + }) + + it('connects the new `EmailAddress` as PRIMARY', async () => { + await mutate({ mutation, variables }) + const result = await neode.cypher(` + MATCH(u:User {id: "567"})-[:PRIMARY_EMAIL]->(e:EmailAddress {email: "to-be-verified@example.org"}) + RETURN e + `) + const email = neode.hydrateFirst(result, 'e', neode.model('EmailAddress')) + await expect(email.toJson()).resolves.toMatchObject({ + email: 'to-be-verified@example.org', + }) + }) + }) }) }) }) diff --git a/backend/src/schema/resolvers/emails.ts b/backend/src/schema/resolvers/emails.ts index d4de9c87b..0638ec634 100644 --- a/backend/src/schema/resolvers/emails.ts +++ b/backend/src/schema/resolvers/emails.ts @@ -87,6 +87,8 @@ export default { ` MATCH (user:User {id: $userId})-[:PRIMARY_EMAIL]->(previous:EmailAddress) MATCH (user)<-[:BELONGS_TO]-(email:UnverifiedEmailAddress {email: $email, nonce: $nonce}) + OPTIONAL MATCH (abandonedEmail:EmailAddress{email: $email}) WHERE NOT EXISTS ((abandonedEmail)<-[]-()) + DELETE abandonedEmail MERGE (user)-[:PRIMARY_EMAIL]->(email) SET email:EmailAddress SET email.verifiedAt = toString(datetime())