From d6a8de478bc604cdbf47ac3070aa8cd74159364d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Sun, 1 Jun 2025 17:53:31 +0800 Subject: [PATCH] feat(backend): migrate to s3 (#8545) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 🍰 Pullrequest This will migrate our assets to an objectstorage via S3. Before this PR is rolled out, the S3 credentials need to be configured in the respective infrastructure repository. The migration is implemented in a backend migration, i.e. I expect the `initContainer` to take a little longer but I hope then it's going to be fine. If any errors occcur, the migration should be repeatable, since the disk volume is still there. ### Issues The backend having direct access on disk. ### Todo - [ ] Configure backend environment variables in every infrastructure repo - [ ] Remove kubernetes uploads volume in a future PR Commits: * refactor: follow @ulfgebhardt Here: https://github.com/Ocelot-Social-Community/Ocelot-Social/pull/8545#pullrequestreview-2846163417 I don't know why the PR didn't include these changes already, I believe I made a mistake during rebase and lost the relevant commits. * refactor: use typescript assertions I found it a better way to react to this comment: https://github.com/Ocelot-Social-Community/Ocelot-Social/pull/8545/files#r2092766596 * add S3 credentials * refactor: easier to remember credentials It's for local development only * give init container necessary file access * fix: wrong upload location on production * refactor: follow @ulfgebhardt's review See: https://github.com/Ocelot-Social-Community/Ocelot-Social/pull/8545#pullrequestreview-2881626504 --- .github/workflows/test-backend.yml | 2 +- .github/workflows/test-e2e.yml | 4 +- backend/.dockerignore | 1 - backend/.env.template | 11 +- backend/.gitignore | 3 +- backend/src/config/index.ts | 26 +- .../20250502230521-migrate-to-s3.ts | 91 ++++ backend/src/graphql/resolvers/groups.ts | 4 +- .../src/graphql/resolvers/images/images.ts | 245 ++--------- .../{images.spec.ts => imagesLocal.spec.ts} | 36 +- .../graphql/resolvers/images/imagesLocal.ts | 134 ++++++ .../graphql/resolvers/images/imagesS3.spec.ts | 407 ++++++++++++++++++ .../src/graphql/resolvers/images/imagesS3.ts | 153 +++++++ .../images/sanitizeRelationshipTypes.ts | 9 + .../resolvers/images/wrapTransaction.ts | 23 + backend/src/graphql/resolvers/posts.ts | 8 +- backend/src/graphql/resolvers/users.ts | 8 +- .../templates/backend/stateful-set.yaml | 3 + deployment/helm/helmfile/secrets/ocelot.yaml | 16 +- docker-compose.override.yml | 39 ++ docker-compose.test.yml | 39 ++ docker-compose.yml | 3 - minio/readonly-policy.json | 19 + 23 files changed, 1025 insertions(+), 259 deletions(-) create mode 100644 backend/src/db/migrations/20250502230521-migrate-to-s3.ts rename backend/src/graphql/resolvers/images/{images.spec.ts => imagesLocal.spec.ts} (93%) create mode 100644 backend/src/graphql/resolvers/images/imagesLocal.ts create mode 100644 backend/src/graphql/resolvers/images/imagesS3.spec.ts create mode 100644 backend/src/graphql/resolvers/images/imagesS3.ts create mode 100644 backend/src/graphql/resolvers/images/sanitizeRelationshipTypes.ts create mode 100644 backend/src/graphql/resolvers/images/wrapTransaction.ts create mode 100644 minio/readonly-policy.json diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 766717a97..a7eff0ef2 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -113,7 +113,7 @@ jobs: - name: backend | docker compose # doesn't work without the --build flag - this either means we should not load the cached images or cache the correct image - run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach --no-deps neo4j backend --build + run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach backend --build - name: backend | Initialize Database run: docker compose exec -T backend yarn db:migrate init diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index d232e2909..fc6a2ea65 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -135,7 +135,7 @@ jobs: docker load < /tmp/neo4j.tar docker load < /tmp/backend.tar docker load < /tmp/webapp.tar - docker compose -f docker-compose.yml -f docker-compose.test.yml up --build --detach --no-deps webapp neo4j backend mailserver + docker compose -f docker-compose.yml -f docker-compose.test.yml up --build --detach webapp mailserver sleep 90s - name: Full stack tests | run tests @@ -176,4 +176,4 @@ jobs: done echo "Done" env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/backend/.dockerignore b/backend/.dockerignore index a0883bf4d..af1b934a3 100644 --- a/backend/.dockerignore +++ b/backend/.dockerignore @@ -17,5 +17,4 @@ build/ maintenance-worker/ neo4j/ -public/uploads/* !.gitkeep diff --git a/backend/.env.template b/backend/.env.template index 5aa44036b..e8c1f4168 100644 --- a/backend/.env.template +++ b/backend/.env.template @@ -40,11 +40,12 @@ COMMIT= PUBLIC_REGISTRATION=false INVITE_REGISTRATION=true -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= -AWS_ENDPOINT= -AWS_REGION= -AWS_BUCKET= +AWS_ACCESS_KEY_ID=minio +AWS_SECRET_ACCESS_KEY=12341234 +AWS_ENDPOINT=http://localhost:9000 +AWS_REGION=local +AWS_BUCKET=ocelot +S3_PUBLIC_GATEWAY=http://localhost:8000 CATEGORIES_ACTIVE=false MAX_PINNED_POSTS=1 diff --git a/backend/.gitignore b/backend/.gitignore index 833f7e34e..a5d3db1fa 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -6,8 +6,7 @@ yarn-error.log build/* coverage.lcov .nyc_output/ -public/uploads/* !.gitkeep # Apple macOS folder attribute file -.DS_Store \ No newline at end of file +.DS_Store diff --git a/backend/src/config/index.ts b/backend/src/config/index.ts index 82f0f674e..0aee79626 100644 --- a/backend/src/config/index.ts +++ b/backend/src/config/index.ts @@ -102,12 +102,26 @@ const s3 = { AWS_ENDPOINT: env.AWS_ENDPOINT, AWS_REGION: env.AWS_REGION, AWS_BUCKET: env.AWS_BUCKET, - S3_CONFIGURED: - env.AWS_ACCESS_KEY_ID && - env.AWS_SECRET_ACCESS_KEY && - env.AWS_ENDPOINT && - env.AWS_REGION && - env.AWS_BUCKET, + S3_PUBLIC_GATEWAY: env.S3_PUBLIC_GATEWAY, +} + +export interface S3Configured { + AWS_ACCESS_KEY_ID: string + AWS_SECRET_ACCESS_KEY: string + AWS_ENDPOINT: string + AWS_REGION: string + AWS_BUCKET: string + S3_PUBLIC_GATEWAY: string | undefined +} + +export const isS3configured = (config: typeof s3): config is S3Configured => { + return !!( + config.AWS_ACCESS_KEY_ID && + config.AWS_SECRET_ACCESS_KEY && + config.AWS_ENDPOINT && + config.AWS_REGION && + config.AWS_BUCKET + ) } const options = { diff --git a/backend/src/db/migrations/20250502230521-migrate-to-s3.ts b/backend/src/db/migrations/20250502230521-migrate-to-s3.ts new file mode 100644 index 000000000..243b7f4e9 --- /dev/null +++ b/backend/src/db/migrations/20250502230521-migrate-to-s3.ts @@ -0,0 +1,91 @@ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ + +import { open } from 'node:fs/promises' +import path from 'node:path' + +import { S3Client, ObjectCannedACL } from '@aws-sdk/client-s3' +import { Upload } from '@aws-sdk/lib-storage' +import { lookup } from 'mime-types' + +import CONFIG from '@config/index' +import { getDriver } from '@db/neo4j' + +export const description = + 'Upload all image files to a S3 compatible object storage in order to reduce load on our backend.' + +export async function up(_next) { + const { AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_ENDPOINT, AWS_BUCKET } = CONFIG + if (!(AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY && AWS_ENDPOINT && AWS_BUCKET)) { + throw new Error('No S3 configuration given, cannot upload image files') + } + + const driver = getDriver() + const session = driver.session() + const transaction = session.beginTransaction() + + const s3 = new S3Client({ + credentials: { + accessKeyId: AWS_ACCESS_KEY_ID, + secretAccessKey: AWS_SECRET_ACCESS_KEY, + }, + endpoint: AWS_ENDPOINT, + forcePathStyle: true, + }) + try { + const { records } = await transaction.run('MATCH (image:Image) RETURN image.url as url') + let urls: string[] = records.map((r) => r.get('url') as string) + urls = urls.filter((url) => url.startsWith('/uploads')) + // eslint-disable-next-line no-console + console.log('URLS uploaded:') + await Promise.all( + urls + .map((url) => async () => { + const { pathname } = new URL(url, 'http://example.org') + // TODO: find a better way to do this - this is quite a hack + const fileLocation = + CONFIG.NODE_ENV === 'production' + ? path.join(__dirname, `../../../../public/${pathname}`) // we're in the /build folder + : path.join(__dirname, `../../../public/${pathname}`) + const s3Location = `original${pathname}` + const mimeType = lookup(fileLocation) + // eslint-disable-next-line security/detect-non-literal-fs-filename + const fileHandle = await open(fileLocation) + const params = { + Bucket: AWS_BUCKET, + Key: s3Location, + ACL: ObjectCannedACL.public_read, + ContentType: mimeType || 'image/jpeg', + Body: fileHandle.createReadStream(), + } + const command = new Upload({ client: s3, params }) + + const data = await command.done() + const { Location: spacesUrl } = data + + const updatedRecord = await transaction.run( + 'MATCH (image:Image {url: $url}) SET image.url = $spacesUrl RETURN image.url as url', + { url, spacesUrl }, + ) + const [updatedUrl] = updatedRecord.records.map((record) => record.get('url') as string) + // eslint-disable-next-line no-console + console.log(updatedUrl) + return updatedUrl + }) + .map((p) => p()), + ) + await transaction.commit() + } catch (error) { + // eslint-disable-next-line no-console + console.log(error) + await transaction.rollback() + // eslint-disable-next-line no-console + console.log('rolled back') + throw new Error(error) + } finally { + await session.close() + } +} + +export function down(_next) { + throw new Error('This migration is irreversible: The backend does not have disk access anymore.') +} diff --git a/backend/src/graphql/resolvers/groups.ts b/backend/src/graphql/resolvers/groups.ts index a3ce3285a..9e330bade 100644 --- a/backend/src/graphql/resolvers/groups.ts +++ b/backend/src/graphql/resolvers/groups.ts @@ -18,7 +18,7 @@ import Resolver, { removeUndefinedNullValuesFromObject, convertObjectToCypherMapLiteral, } from './helpers/Resolver' -import { mergeImage } from './images/images' +import { images } from './images/images' import { createOrUpdateLocations } from './users/location' export default { @@ -260,7 +260,7 @@ export default { }) const [group] = transactionResponse.records.map((record) => record.get('group')) if (avatarInput) { - await mergeImage(group, 'AVATAR_IMAGE', avatarInput, { transaction }) + await images.mergeImage(group, 'AVATAR_IMAGE', avatarInput, { transaction }) } return group }) diff --git a/backend/src/graphql/resolvers/images/images.ts b/backend/src/graphql/resolvers/images/images.ts index f437b3c85..6c2fa8b3a 100644 --- a/backend/src/graphql/resolvers/images/images.ts +++ b/backend/src/graphql/resolvers/images/images.ts @@ -1,32 +1,27 @@ -/* eslint-disable @typescript-eslint/require-await */ -/* eslint-disable @typescript-eslint/restrict-template-expressions */ -/* eslint-disable @typescript-eslint/no-unsafe-call */ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -/* eslint-disable @typescript-eslint/no-unsafe-argument */ -/* eslint-disable promise/avoid-new */ -/* eslint-disable security/detect-non-literal-fs-filename */ +import CONFIG, { isS3configured } from '@config/index' -import { existsSync, unlinkSync, createWriteStream } from 'node:fs' -import path from 'node:path' - -import { S3Client, DeleteObjectCommand, ObjectCannedACL } from '@aws-sdk/client-s3' -import { Upload } from '@aws-sdk/lib-storage' -import { UserInputError } from 'apollo-server' -import slug from 'slug' -import { v4 as uuid } from 'uuid' - -import CONFIG from '@config/index' -import { getDriver } from '@db/neo4j' +import { images as imagesLocal } from './imagesLocal' +import { images as imagesS3 } from './imagesS3' import type { FileUpload } from 'graphql-upload' import type { Transaction } from 'neo4j-driver' -type FileDeleteCallback = (url: string) => Promise -type FileUploadCallback = ( +export type FileDeleteCallback = (url: string) => Promise + +export type FileUploadCallback = ( upload: Pick & { uniqueFilename: string }, ) => Promise +export interface DeleteImageOpts { + transaction?: Transaction + deleteCallback?: FileDeleteCallback +} + +export interface MergeImageOpts { + transaction?: Transaction + uploadCallback?: FileUploadCallback + deleteCallback?: FileDeleteCallback +} + export interface ImageInput { upload?: Promise alt?: string @@ -35,191 +30,29 @@ export interface ImageInput { type?: string } -// const widths = [34, 160, 320, 640, 1024] -const { AWS_BUCKET: Bucket, S3_CONFIGURED } = CONFIG - -const createS3Client = () => { - const { AWS_ENDPOINT, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY } = CONFIG - if (!(AWS_ENDPOINT && AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY)) { - throw new Error('Missing AWS credentials.') - } - return new S3Client({ - credentials: { - accessKeyId: AWS_ACCESS_KEY_ID, - secretAccessKey: AWS_SECRET_ACCESS_KEY, - }, - endpoint: AWS_ENDPOINT, - forcePathStyle: true, - }) +export interface Image { + url: string + alt?: string + sensitive?: boolean + aspectRatio?: number + type?: string } -interface DeleteImageOpts { - transaction?: Transaction - deleteCallback?: FileDeleteCallback -} -export async function deleteImage(resource, relationshipType, opts: DeleteImageOpts = {}) { - sanitizeRelationshipType(relationshipType) - const { transaction, deleteCallback } = opts - if (!transaction) return wrapTransaction(deleteImage, [resource, relationshipType], opts) - const txResult = await transaction.run( - ` - MATCH (resource {id: $resource.id})-[rel:${relationshipType}]->(image:Image) - WITH image, image {.*} as imageProps - DETACH DELETE image - RETURN imageProps - `, - { resource }, - ) - const [image] = txResult.records.map((record) => record.get('imageProps')) - // This behaviour differs from `mergeImage`. If you call `mergeImage` - // with metadata for an image that does not exist, it's an indicator - // of an error (so throw an error). If we bulk delete an image, it - // could very well be that there is no image for the resource. - if (image) deleteImageFile(image, deleteCallback) - return image +export interface Images { + deleteImage: ( + resource: { id: string }, + relationshipType: 'HERO_IMAGE' | 'AVATAR_IMAGE', + opts?: DeleteImageOpts, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ) => Promise + + mergeImage: ( + resource: { id: string }, + relationshipType: 'HERO_IMAGE' | 'AVATAR_IMAGE', + imageInput: ImageInput | null | undefined, + opts?: MergeImageOpts, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ) => Promise } -interface MergeImageOpts { - transaction?: Transaction - uploadCallback?: FileUploadCallback - deleteCallback?: FileDeleteCallback -} - -export async function mergeImage( - resource, - relationshipType, - imageInput: ImageInput | null | undefined, - opts: MergeImageOpts = {}, -) { - if (typeof imageInput === 'undefined') return - if (imageInput === null) return deleteImage(resource, relationshipType, opts) - sanitizeRelationshipType(relationshipType) - const { transaction, uploadCallback, deleteCallback } = opts - if (!transaction) - return wrapTransaction(mergeImage, [resource, relationshipType, imageInput], opts) - - let txResult - txResult = await transaction.run( - ` - MATCH (resource {id: $resource.id})-[:${relationshipType}]->(image:Image) - RETURN image {.*} - `, - { resource }, - ) - const [existingImage] = txResult.records.map((record) => record.get('image')) - const { upload } = imageInput - if (!(existingImage || upload)) throw new UserInputError('Cannot find image for given resource') - if (existingImage && upload) deleteImageFile(existingImage, deleteCallback) - const url = await uploadImageFile(upload, uploadCallback) - const { alt, sensitive, aspectRatio, type } = imageInput - const image = { alt, sensitive, aspectRatio, url, type } - txResult = await transaction.run( - ` - MATCH (resource {id: $resource.id}) - MERGE (resource)-[:${relationshipType}]->(image:Image) - ON CREATE SET image.createdAt = toString(datetime()) - ON MATCH SET image.updatedAt = toString(datetime()) - SET image += $image - RETURN image {.*} - `, - { resource, image }, - ) - const [mergedImage] = txResult.records.map((record) => record.get('image')) - return mergedImage -} - -const wrapTransaction = async (wrappedCallback, args, opts) => { - const session = getDriver().session() - try { - const result = await session.writeTransaction(async (transaction) => { - return wrappedCallback(...args, { ...opts, transaction }) - }) - return result - } finally { - await session.close() - } -} - -const deleteImageFile = (image, deleteCallback: FileDeleteCallback | undefined) => { - if (!deleteCallback) { - deleteCallback = S3_CONFIGURED ? s3Delete : localFileDelete - } - const { url } = image - // eslint-disable-next-line @typescript-eslint/no-floating-promises - deleteCallback(url) - return url -} - -const uploadImageFile = async ( - upload: Promise | undefined, - uploadCallback: FileUploadCallback | undefined, -) => { - if (!upload) return undefined - if (!uploadCallback) { - uploadCallback = S3_CONFIGURED ? s3Upload : localFileUpload - } - // eslint-disable-next-line @typescript-eslint/unbound-method - const { createReadStream, filename, mimetype } = await upload - const { name, ext } = path.parse(filename) - const uniqueFilename = `${uuid()}-${slug(name)}${ext}` - return uploadCallback({ createReadStream, uniqueFilename, mimetype }) -} - -const sanitizeRelationshipType = (relationshipType) => { - // Cypher query language does not allow to parameterize relationship types - // See: https://github.com/neo4j/neo4j/issues/340 - if (!['HERO_IMAGE', 'AVATAR_IMAGE'].includes(relationshipType)) { - throw new Error(`Unknown relationship type ${relationshipType}`) - } -} - -const localFileUpload: FileUploadCallback = ({ createReadStream, uniqueFilename }) => { - const destination = `/uploads/${uniqueFilename}` - return new Promise((resolve, reject) => - createReadStream().pipe( - createWriteStream(`public${destination}`) - .on('finish', () => resolve(destination)) - .on('error', (error) => reject(error)), - ), - ) -} - -const s3Upload: FileUploadCallback = async ({ createReadStream, uniqueFilename, mimetype }) => { - const s3Location = `original/${uniqueFilename}` - const params = { - Bucket, - Key: s3Location, - ACL: ObjectCannedACL.public_read, - ContentType: mimetype, - Body: createReadStream(), - } - const s3 = createS3Client() - const command = new Upload({ client: s3, params }) - const data = await command.done() - const { Location } = data - if (!Location) { - throw new Error('File upload did not return `Location`') - } - return Location -} - -const localFileDelete: FileDeleteCallback = async (url) => { - const location = `public${url}` - // eslint-disable-next-line n/no-sync - if (existsSync(location)) unlinkSync(location) -} - -const s3Delete: FileDeleteCallback = async (url) => { - let { pathname } = new URL(url, 'http://example.org') // dummy domain to avoid invalid URL error - pathname = pathname.substring(1) // remove first character '/' - const prefix = `${Bucket}/` - if (pathname.startsWith(prefix)) { - pathname = pathname.slice(prefix.length) - } - const params = { - Bucket, - Key: pathname, - } - const s3 = createS3Client() - await s3.send(new DeleteObjectCommand(params)) -} +export const images = isS3configured(CONFIG) ? imagesS3(CONFIG) : imagesLocal diff --git a/backend/src/graphql/resolvers/images/images.spec.ts b/backend/src/graphql/resolvers/images/imagesLocal.spec.ts similarity index 93% rename from backend/src/graphql/resolvers/images/images.spec.ts rename to backend/src/graphql/resolvers/images/imagesLocal.spec.ts index 75f5f4feb..257a24e78 100644 --- a/backend/src/graphql/resolvers/images/images.spec.ts +++ b/backend/src/graphql/resolvers/images/imagesLocal.spec.ts @@ -11,7 +11,7 @@ import { UserInputError } from 'apollo-server' import Factory, { cleanDatabase } from '@db/factories' import { getNeode, getDriver } from '@db/neo4j' -import { deleteImage, mergeImage } from './images' +import { images } from './imagesLocal' import type { ImageInput } from './images' import type { FileUpload } from 'graphql-upload' @@ -42,10 +42,12 @@ afterEach(async () => { }) describe('deleteImage', () => { + const { deleteImage } = images + describe('given a resource with an image', () => { - let user + let user: { id: string } beforeEach(async () => { - user = await Factory.build( + const u = await Factory.build( 'user', {}, { @@ -55,7 +57,7 @@ describe('deleteImage', () => { }), }, ) - user = await user.toJson() + user = await u.toJson() }) it('deletes `Image` node', async () => { @@ -65,8 +67,8 @@ describe('deleteImage', () => { }) it('calls deleteCallback', async () => { - user = await Factory.build('user') - user = await user.toJson() + const u = await Factory.build('user') + user = await u.toJson() await deleteImage(user, 'AVATAR_IMAGE', { deleteCallback }) expect(deleteCallback).toHaveBeenCalled() }) @@ -117,8 +119,9 @@ describe('deleteImage', () => { }) describe('mergeImage', () => { + const { mergeImage } = images let imageInput: ImageInput - let post + let post: { id: string } beforeEach(() => { imageInput = { alt: 'A description of the new image', @@ -145,7 +148,7 @@ describe('mergeImage', () => { describe('on existing resource', () => { beforeEach(async () => { - post = await Factory.build( + const p = await Factory.build( 'post', { id: 'p99' }, { @@ -153,7 +156,7 @@ describe('mergeImage', () => { image: null, }, ) - post = await post.toJson() + post = await p.toJson() }) it('returns new image', async () => { @@ -196,7 +199,7 @@ describe('mergeImage', () => { `MATCH(p:Post {id: "p99"})-[:HERO_IMAGE]->(i:Image) RETURN i,p`, {}, ) - post = neode.hydrateFirst(result, 'p', neode.model('Post')) + post = neode.hydrateFirst<{ id: string }>(result, 'p', neode.model('Post')).properties() const image = neode.hydrateFirst(result, 'i', neode.model('Image')) expect(post).toBeTruthy() expect(image).toBeTruthy() @@ -204,7 +207,10 @@ describe('mergeImage', () => { it('whitelists relationship types', async () => { await expect( - mergeImage(post, 'WHATEVER', imageInput, { uploadCallback, deleteCallback }), + mergeImage(post, 'WHATEVER' as 'HERO_IMAGE', imageInput, { + uploadCallback, + deleteCallback, + }), ).rejects.toEqual(new Error('Unknown relationship type WHATEVER')) }) @@ -311,8 +317,8 @@ describe('mergeImage', () => { describe('without image.upload', () => { it('throws UserInputError', async () => { - post = await Factory.build('post', { id: 'p99' }, { image: null }) - post = await post.toJson() + const p = await Factory.build('post', { id: 'p99' }, { image: null }) + post = await p.toJson() await expect(mergeImage(post, 'HERO_IMAGE', imageInput)).rejects.toEqual( new UserInputError('Cannot find image for given resource'), ) @@ -320,7 +326,7 @@ describe('mergeImage', () => { describe('if resource has an image already', () => { beforeEach(async () => { - post = await Factory.build( + const p = await Factory.build( 'post', { id: 'p99', @@ -339,7 +345,7 @@ describe('mergeImage', () => { }), }, ) - post = await post.toJson() + post = await p.toJson() }) it('does not call deleteCallback', async () => { diff --git a/backend/src/graphql/resolvers/images/imagesLocal.ts b/backend/src/graphql/resolvers/images/imagesLocal.ts new file mode 100644 index 000000000..c9f575777 --- /dev/null +++ b/backend/src/graphql/resolvers/images/imagesLocal.ts @@ -0,0 +1,134 @@ +/* eslint-disable @typescript-eslint/require-await */ + +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ +/* eslint-disable promise/avoid-new */ +/* eslint-disable security/detect-non-literal-fs-filename */ + +import { existsSync, unlinkSync, createWriteStream } from 'node:fs' +import path from 'node:path' + +import { UserInputError } from 'apollo-server' +import slug from 'slug' +import { v4 as uuid } from 'uuid' + +import { sanitizeRelationshipType } from './sanitizeRelationshipTypes' +import { wrapTransaction } from './wrapTransaction' + +import type { Images, FileDeleteCallback, FileUploadCallback } from './images' +import type { FileUpload } from 'graphql-upload' + +const deleteImage: Images['deleteImage'] = async (resource, relationshipType, opts = {}) => { + sanitizeRelationshipType(relationshipType) + const { transaction, deleteCallback } = opts + if (!transaction) return wrapTransaction(deleteImage, [resource, relationshipType], opts) + const txResult = await transaction.run( + ` + MATCH (resource {id: $resource.id})-[rel:${relationshipType}]->(image:Image) + WITH image, image {.*} as imageProps + DETACH DELETE image + RETURN imageProps + `, + { resource }, + ) + const [image] = txResult.records.map((record) => record.get('imageProps')) + // This behaviour differs from `mergeImage`. If you call `mergeImage` + // with metadata for an image that does not exist, it's an indicator + // of an error (so throw an error). If we bulk delete an image, it + // could very well be that there is no image for the resource. + if (image) deleteImageFile(image, deleteCallback) + return image +} + +const mergeImage: Images['mergeImage'] = async ( + resource, + relationshipType, + imageInput, + opts = {}, +) => { + if (typeof imageInput === 'undefined') return + if (imageInput === null) return deleteImage(resource, relationshipType, opts) + sanitizeRelationshipType(relationshipType) + const { transaction, uploadCallback, deleteCallback } = opts + if (!transaction) + return wrapTransaction(mergeImage, [resource, relationshipType, imageInput], opts) + + let txResult + txResult = await transaction.run( + ` + MATCH (resource {id: $resource.id})-[:${relationshipType}]->(image:Image) + RETURN image {.*} + `, + { resource }, + ) + const [existingImage] = txResult.records.map((record) => record.get('image')) + const { upload } = imageInput + if (!(existingImage || upload)) throw new UserInputError('Cannot find image for given resource') + if (existingImage && upload) deleteImageFile(existingImage, deleteCallback) + const url = await uploadImageFile(upload, uploadCallback) + const { alt, sensitive, aspectRatio, type } = imageInput + const image = { alt, sensitive, aspectRatio, url, type } + txResult = await transaction.run( + ` + MATCH (resource {id: $resource.id}) + MERGE (resource)-[:${relationshipType}]->(image:Image) + ON CREATE SET image.createdAt = toString(datetime()) + ON MATCH SET image.updatedAt = toString(datetime()) + SET image += $image + RETURN image {.*} + `, + { resource, image }, + ) + const [mergedImage] = txResult.records.map((record) => record.get('image')) + return mergedImage +} + +const localFileDelete: FileDeleteCallback = async (url) => { + const location = `public${url}` + // eslint-disable-next-line n/no-sync + if (existsSync(location)) unlinkSync(location) +} + +const deleteImageFile = (image, deleteCallback: FileDeleteCallback | undefined) => { + if (!deleteCallback) { + deleteCallback = localFileDelete + } + const { url } = image + // eslint-disable-next-line @typescript-eslint/no-floating-promises + deleteCallback(url) + return url +} + +const uploadImageFile = async ( + upload: Promise | undefined, + uploadCallback: FileUploadCallback | undefined, +) => { + if (!upload) return undefined + if (!uploadCallback) { + uploadCallback = localFileUpload + } + // eslint-disable-next-line @typescript-eslint/unbound-method + const { createReadStream, filename, mimetype } = await upload + const { name, ext } = path.parse(filename) + const uniqueFilename = `${uuid()}-${slug(name)}${ext}` + return uploadCallback({ createReadStream, uniqueFilename, mimetype }) +} + +const localFileUpload: FileUploadCallback = ({ createReadStream, uniqueFilename }) => { + const destination = `/uploads/${uniqueFilename}` + return new Promise((resolve, reject) => + createReadStream().pipe( + createWriteStream(`public${destination}`) + .on('finish', () => resolve(destination)) + .on('error', (error) => reject(error)), + ), + ) +} + +export const images: Images = { + deleteImage, + mergeImage, +} diff --git a/backend/src/graphql/resolvers/images/imagesS3.spec.ts b/backend/src/graphql/resolvers/images/imagesS3.spec.ts new file mode 100644 index 000000000..2bedec3cd --- /dev/null +++ b/backend/src/graphql/resolvers/images/imagesS3.spec.ts @@ -0,0 +1,407 @@ +/* eslint-disable @typescript-eslint/require-await */ + +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable @typescript-eslint/restrict-template-expressions */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable promise/prefer-await-to-callbacks */ +import { UserInputError } from 'apollo-server' + +import Factory, { cleanDatabase } from '@db/factories' +import { getNeode, getDriver } from '@db/neo4j' +import type { S3Configured } from '@src/config' + +import { images } from './imagesS3' + +import type { ImageInput } from './images' +import type { FileUpload } from 'graphql-upload' + +const driver = getDriver() +const neode = getNeode() +const uuid = '[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}' +let uploadCallback +let deleteCallback + +const config: S3Configured = { + AWS_ACCESS_KEY_ID: 'AWS_ACCESS_KEY_ID', + AWS_SECRET_ACCESS_KEY: 'AWS_SECRET_ACCESS_KEY', + AWS_BUCKET: 'AWS_BUCKET', + AWS_ENDPOINT: 'AWS_ENDPOINT', + AWS_REGION: 'AWS_REGION', + S3_PUBLIC_GATEWAY: undefined, +} + +beforeAll(async () => { + await cleanDatabase() +}) + +afterAll(async () => { + await cleanDatabase() + await driver.close() +}) + +beforeEach(async () => { + uploadCallback = jest.fn( + ({ uniqueFilename }) => `http://your-objectstorage.com/bucket/${uniqueFilename}`, + ) + deleteCallback = jest.fn() +}) + +// TODO: avoid database clean after each test in the future if possible for performance and flakyness reasons by filling the database step by step, see issue https://github.com/Ocelot-Social-Community/Ocelot-Social/issues/4543 +afterEach(async () => { + await cleanDatabase() +}) + +describe('deleteImage', () => { + const { deleteImage } = images(config) + describe('given a resource with an image', () => { + let user: { id: string } + beforeEach(async () => { + const u = await Factory.build( + 'user', + {}, + { + avatar: Factory.build('image', { + url: 'http://localhost/some/avatar/url/', + alt: 'This is the avatar image of a user', + }), + }, + ) + user = await u.toJson() + }) + + it('deletes `Image` node', async () => { + await expect(neode.all('Image')).resolves.toHaveLength(1) + await deleteImage(user, 'AVATAR_IMAGE', { deleteCallback }) + await expect(neode.all('Image')).resolves.toHaveLength(0) + }) + + it('calls deleteCallback', async () => { + await deleteImage(user, 'AVATAR_IMAGE', { deleteCallback }) + expect(deleteCallback).toHaveBeenCalled() + }) + + describe('given a transaction parameter', () => { + it('executes cypher statements within the transaction', async () => { + const session = driver.session() + let someString: string + try { + someString = await session.writeTransaction(async (transaction) => { + await deleteImage(user, 'AVATAR_IMAGE', { + deleteCallback, + transaction, + }) + const txResult = await transaction.run('RETURN "Hello" as result') + const [result] = txResult.records.map((record) => record.get('result')) + return result + }) + } finally { + await session.close() + } + await expect(neode.all('Image')).resolves.toHaveLength(0) + expect(someString).toEqual('Hello') + }) + + it('rolls back the transaction in case of errors', async () => { + await expect(neode.all('Image')).resolves.toHaveLength(1) + const session = driver.session() + try { + await session.writeTransaction(async (transaction) => { + await deleteImage(user, 'AVATAR_IMAGE', { + deleteCallback, + transaction, + }) + throw new Error('Ouch!') + }) + // eslint-disable-next-line no-catch-all/no-catch-all + } catch (err) { + // nothing has been deleted + await expect(neode.all('Image')).resolves.toHaveLength(1) + // all good + } finally { + await session.close() + } + }) + }) + }) +}) + +describe('mergeImage', () => { + const { mergeImage } = images(config) + let imageInput: ImageInput + let post: { id: string } + beforeEach(() => { + imageInput = { + alt: 'A description of the new image', + } + }) + + describe('given image.upload', () => { + beforeEach(() => { + const createReadStream: FileUpload['createReadStream'] = (() => ({ + pipe: () => ({ + on: (_, callback) => callback(), + }), + })) as unknown as FileUpload['createReadStream'] + imageInput = { + ...imageInput, + upload: Promise.resolve({ + filename: 'image.jpg', + mimetype: 'image/jpeg', + encoding: '7bit', + createReadStream, + }), + } + }) + + describe('on existing resource', () => { + beforeEach(async () => { + const p = await Factory.build( + 'post', + { id: 'p99' }, + { + author: Factory.build('user', {}, { avatar: null }), + image: null, + }, + ) + post = await p.toJson() + }) + + it('returns new image', async () => { + await expect( + mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }), + ).resolves.toMatchObject({ + url: expect.any(String), + alt: 'A description of the new image', + }) + }) + + it('calls upload callback', async () => { + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + expect(uploadCallback).toHaveBeenCalled() + }) + + it('creates `:Image` node', async () => { + await expect(neode.all('Image')).resolves.toHaveLength(0) + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + await expect(neode.all('Image')).resolves.toHaveLength(1) + }) + + it('creates a url safe name', async () => { + if (!imageInput.upload) { + throw new Error('Test imageInput was not setup correctly.') + } + const upload = await imageInput.upload + upload.filename = '/path/to/awkward?/ file-location/?foo- bar-avatar.jpg' + imageInput.upload = Promise.resolve(upload) + await expect( + mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }), + ).resolves.toMatchObject({ + url: expect.stringMatching( + new RegExp(`^http://your-objectstorage.com/bucket/${uuid}-foo-bar-avatar.jpg`), + ), + }) + }) + + describe('given a `S3_PUBLIC_GATEWAY` configuration', () => { + const { mergeImage } = images({ + ...config, + S3_PUBLIC_GATEWAY: 'http://s3-public-gateway.com', + }) + + it('changes the domain of the URL to a server that could e.g. apply image transformations', async () => { + if (!imageInput.upload) { + throw new Error('Test imageInput was not setup correctly.') + } + const upload = await imageInput.upload + upload.filename = '/path/to/file-location/foo-bar-avatar.jpg' + imageInput.upload = Promise.resolve(upload) + await expect( + mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }), + ).resolves.toMatchObject({ + url: expect.stringMatching( + new RegExp(`^http://s3-public-gateway.com/bucket/${uuid}-foo-bar-avatar.jpg`), + ), + }) + }) + }) + + it('connects resource with image via given image type', async () => { + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + const result = await neode.cypher( + `MATCH(p:Post {id: "p99"})-[:HERO_IMAGE]->(i:Image) RETURN i,p`, + {}, + ) + post = neode.hydrateFirst<{ id: string }>(result, 'p', neode.model('Post')).properties() + const image = neode.hydrateFirst(result, 'i', neode.model('Image')) + expect(post).toBeTruthy() + expect(image).toBeTruthy() + }) + + it('whitelists relationship types', async () => { + await expect( + mergeImage(post, 'WHATEVER' as 'HERO_IMAGE', imageInput, { + uploadCallback, + deleteCallback, + }), + ).rejects.toEqual(new Error('Unknown relationship type WHATEVER')) + }) + + it('sets metadata', async () => { + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + const image = await neode.first('Image', {}, undefined) + await expect(image.toJson()).resolves.toMatchObject({ + alt: 'A description of the new image', + createdAt: expect.any(String), + url: expect.any(String), + }) + }) + + describe('given a transaction parameter', () => { + it('executes cypher statements within the transaction', async () => { + const session = driver.session() + try { + await session.writeTransaction(async (transaction) => { + const image = await mergeImage(post, 'HERO_IMAGE', imageInput, { + uploadCallback, + deleteCallback, + transaction, + }) + return transaction.run( + ` + MATCH(image:Image {url: $image.url}) + SET image.alt = 'This alt text gets overwritten' + RETURN image {.*} + `, + { image }, + ) + }) + } finally { + await session.close() + } + const image = await neode.first( + 'Image', + { alt: 'This alt text gets overwritten' }, + undefined, + ) + await expect(image.toJson()).resolves.toMatchObject({ + alt: 'This alt text gets overwritten', + }) + }) + + it('rolls back the transaction in case of errors', async () => { + const session = driver.session() + try { + await session.writeTransaction(async (transaction) => { + const image = await mergeImage(post, 'HERO_IMAGE', imageInput, { + uploadCallback, + deleteCallback, + transaction, + }) + return transaction.run('Ooops invalid cypher!', { image }) + }) + // eslint-disable-next-line no-catch-all/no-catch-all + } catch (err) { + // nothing has been created + await expect(neode.all('Image')).resolves.toHaveLength(0) + // all good + } finally { + await session.close() + } + }) + }) + + describe('if resource has an image already', () => { + beforeEach(async () => { + const [post, image] = await Promise.all([ + neode.find('Post', 'p99'), + Factory.build('image'), + ]) + await post.relateTo(image, 'image') + }) + + it('calls deleteCallback', async () => { + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + expect(deleteCallback).toHaveBeenCalled() + }) + + it('calls uploadCallback', async () => { + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + expect(uploadCallback).toHaveBeenCalled() + }) + + it('updates metadata of existing image node', async () => { + await expect(neode.all('Image')).resolves.toHaveLength(1) + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + await expect(neode.all('Image')).resolves.toHaveLength(1) + const image = await neode.first('Image', {}, undefined) + await expect(image.toJson()).resolves.toMatchObject({ + alt: 'A description of the new image', + createdAt: expect.any(String), + url: expect.any(String), + // TODO + // width: + // height: + }) + }) + }) + }) + }) + + describe('without image.upload', () => { + it('throws UserInputError', async () => { + const p = await Factory.build('post', { id: 'p99' }, { image: null }) + post = await p.toJson() + await expect(mergeImage(post, 'HERO_IMAGE', imageInput)).rejects.toEqual( + new UserInputError('Cannot find image for given resource'), + ) + }) + + describe('if resource has an image already', () => { + beforeEach(async () => { + const p = await Factory.build( + 'post', + { + id: 'p99', + }, + { + author: Factory.build( + 'user', + {}, + { + avatar: null, + }, + ), + image: Factory.build('image', { + alt: 'This is the previous, not updated image', + url: 'http://localhost/some/original/url', + }), + }, + ) + post = await p.toJson() + }) + + it('does not call deleteCallback', async () => { + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + expect(deleteCallback).not.toHaveBeenCalled() + }) + + it('does not call uploadCallback', async () => { + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + expect(uploadCallback).not.toHaveBeenCalled() + }) + + it('updates metadata', async () => { + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + const images = await neode.all('Image') + expect(images).toHaveLength(1) + await expect(images.first().toJson()).resolves.toMatchObject({ + createdAt: expect.any(String), + url: expect.any(String), + alt: 'A description of the new image', + }) + }) + }) + }) +}) diff --git a/backend/src/graphql/resolvers/images/imagesS3.ts b/backend/src/graphql/resolvers/images/imagesS3.ts new file mode 100644 index 000000000..66c4a0a69 --- /dev/null +++ b/backend/src/graphql/resolvers/images/imagesS3.ts @@ -0,0 +1,153 @@ +import path from 'node:path' + +import { S3Client, DeleteObjectCommand, ObjectCannedACL } from '@aws-sdk/client-s3' +import { Upload } from '@aws-sdk/lib-storage' +import { UserInputError } from 'apollo-server' +import slug from 'slug' +import { v4 as uuid } from 'uuid' + +import { S3Configured } from '@config/index' + +import { sanitizeRelationshipType } from './sanitizeRelationshipTypes' +import { wrapTransaction } from './wrapTransaction' + +import type { Image, Images, FileDeleteCallback, FileUploadCallback } from './images' +import type { FileUpload } from 'graphql-upload' + +export const images = (config: S3Configured) => { + // const widths = [34, 160, 320, 640, 1024] + const { AWS_BUCKET: Bucket, S3_PUBLIC_GATEWAY } = config + + const { AWS_ENDPOINT, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY } = config + const s3 = new S3Client({ + credentials: { + accessKeyId: AWS_ACCESS_KEY_ID, + secretAccessKey: AWS_SECRET_ACCESS_KEY, + }, + endpoint: AWS_ENDPOINT, + forcePathStyle: true, + }) + + const deleteImage: Images['deleteImage'] = async (resource, relationshipType, opts = {}) => { + sanitizeRelationshipType(relationshipType) + const { transaction, deleteCallback = s3Delete } = opts + if (!transaction) return wrapTransaction(deleteImage, [resource, relationshipType], opts) + const txResult = await transaction.run( + ` + MATCH (resource {id: $resource.id})-[rel:${relationshipType}]->(image:Image) + WITH image, image {.*} as imageProps + DETACH DELETE image + RETURN imageProps + `, + { resource }, + ) + const [image] = txResult.records.map((record) => record.get('imageProps') as Image) + // This behaviour differs from `mergeImage`. If you call `mergeImage` + // with metadata for an image that does not exist, it's an indicator + // of an error (so throw an error). If we bulk delete an image, it + // could very well be that there is no image for the resource. + if (image) { + await deleteCallback(image.url) + } + return image + } + + const mergeImage: Images['mergeImage'] = async ( + resource, + relationshipType, + imageInput, + opts = {}, + ) => { + if (typeof imageInput === 'undefined') return + if (imageInput === null) return deleteImage(resource, relationshipType, opts) + sanitizeRelationshipType(relationshipType) + const { transaction, uploadCallback, deleteCallback = s3Delete } = opts + if (!transaction) + return wrapTransaction(mergeImage, [resource, relationshipType, imageInput], opts) + + let txResult = await transaction.run( + ` + MATCH (resource {id: $resource.id})-[:${relationshipType}]->(image:Image) + RETURN image {.*} + `, + { resource }, + ) + const [existingImage] = txResult.records.map((record) => record.get('image') as Image) + const { upload } = imageInput + if (!(existingImage || upload)) throw new UserInputError('Cannot find image for given resource') + if (existingImage && upload) { + await deleteCallback(existingImage.url) + } + const url = await uploadImageFile(upload, uploadCallback) + const { alt, sensitive, aspectRatio, type } = imageInput + const image = { alt, sensitive, aspectRatio, url, type } + txResult = await transaction.run( + ` + MATCH (resource {id: $resource.id}) + MERGE (resource)-[:${relationshipType}]->(image:Image) + ON CREATE SET image.createdAt = toString(datetime()) + ON MATCH SET image.updatedAt = toString(datetime()) + SET image += $image + RETURN image {.*} + `, + { resource, image }, + ) + const [mergedImage] = txResult.records.map((record) => record.get('image') as Image) + return mergedImage + } + + const uploadImageFile = async ( + uploadPromise: Promise | undefined, + uploadCallback: FileUploadCallback | undefined = s3Upload, + ) => { + if (!uploadPromise) return undefined + const upload = await uploadPromise + const { name, ext } = path.parse(upload.filename) + const uniqueFilename = `${uuid()}-${slug(name)}${ext}` + const Location = await uploadCallback({ ...upload, uniqueFilename }) + if (!S3_PUBLIC_GATEWAY) { + return Location + } + const publicLocation = new URL(S3_PUBLIC_GATEWAY) + publicLocation.pathname = new URL(Location).pathname + return publicLocation.href + } + + const s3Upload: FileUploadCallback = async ({ createReadStream, uniqueFilename, mimetype }) => { + const s3Location = `original/${uniqueFilename}` + const params = { + Bucket, + Key: s3Location, + ACL: ObjectCannedACL.public_read, + ContentType: mimetype, + Body: createReadStream(), + } + const command = new Upload({ client: s3, params }) + const data = await command.done() + const { Location } = data + if (!Location) { + throw new Error('File upload did not return `Location`') + } + return Location + } + + const s3Delete: FileDeleteCallback = async (url) => { + let { pathname } = new URL(url, 'http://example.org') // dummy domain to avoid invalid URL error + pathname = pathname.substring(1) // remove first character '/' + const prefix = `${Bucket}/` + if (pathname.startsWith(prefix)) { + pathname = pathname.slice(prefix.length) + } + const params = { + Bucket, + Key: pathname, + } + await s3.send(new DeleteObjectCommand(params)) + } + + const images: Images = { + deleteImage, + mergeImage, + } + return images +} diff --git a/backend/src/graphql/resolvers/images/sanitizeRelationshipTypes.ts b/backend/src/graphql/resolvers/images/sanitizeRelationshipTypes.ts new file mode 100644 index 000000000..a6b984a13 --- /dev/null +++ b/backend/src/graphql/resolvers/images/sanitizeRelationshipTypes.ts @@ -0,0 +1,9 @@ +export function sanitizeRelationshipType( + relationshipType: string, +): asserts relationshipType is 'HERO_IMAGE' | 'AVATAR_IMAGE' { + // Cypher query language does not allow to parameterize relationship types + // See: https://github.com/neo4j/neo4j/issues/340 + if (!['HERO_IMAGE', 'AVATAR_IMAGE'].includes(relationshipType)) { + throw new Error(`Unknown relationship type ${relationshipType}`) + } +} diff --git a/backend/src/graphql/resolvers/images/wrapTransaction.ts b/backend/src/graphql/resolvers/images/wrapTransaction.ts new file mode 100644 index 000000000..bcc17877d --- /dev/null +++ b/backend/src/graphql/resolvers/images/wrapTransaction.ts @@ -0,0 +1,23 @@ +import { getDriver } from '@db/neo4j' + +import type { DeleteImageOpts, MergeImageOpts } from './images' + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type AsyncFunc = (...args: any[]) => Promise +export const wrapTransaction = async ( + wrappedCallback: F, + args: unknown[], + opts: DeleteImageOpts | MergeImageOpts, +) => { + const session = getDriver().session() + try { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const result = await session.writeTransaction((transaction) => { + return wrappedCallback(...args, { ...opts, transaction }) + }) + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return result + } finally { + await session.close() + } +} diff --git a/backend/src/graphql/resolvers/posts.ts b/backend/src/graphql/resolvers/posts.ts index 5190cabf8..34d190ba5 100644 --- a/backend/src/graphql/resolvers/posts.ts +++ b/backend/src/graphql/resolvers/posts.ts @@ -17,7 +17,7 @@ import { filterForMutedUsers } from './helpers/filterForMutedUsers' import { filterInvisiblePosts } from './helpers/filterInvisiblePosts' import { filterPostsOfMyGroups } from './helpers/filterPostsOfMyGroups' import Resolver from './helpers/Resolver' -import { mergeImage, deleteImage } from './images/images' +import { images } from './images/images' import { createOrUpdateLocations } from './users/location' const maintainPinnedPosts = (params) => { @@ -176,7 +176,7 @@ export default { ) const [post] = createPostTransactionResponse.records.map((record) => record.get('post')) if (imageInput) { - await mergeImage(post, 'HERO_IMAGE', imageInput, { transaction }) + await images.mergeImage(post, 'HERO_IMAGE', imageInput, { transaction }) } return post }) @@ -247,7 +247,7 @@ export default { updatePostVariables, ) const [post] = updatePostTransactionResponse.records.map((record) => record.get('post')) - await mergeImage(post, 'HERO_IMAGE', imageInput, { transaction }) + await images.mergeImage(post, 'HERO_IMAGE', imageInput, { transaction }) return post }) const post = await writeTxResultPromise @@ -277,7 +277,7 @@ export default { { postId: args.id }, ) const [post] = deletePostTransactionResponse.records.map((record) => record.get('post')) - await deleteImage(post, 'HERO_IMAGE', { transaction }) + await images.deleteImage(post, 'HERO_IMAGE', { transaction }) return post }) try { diff --git a/backend/src/graphql/resolvers/users.ts b/backend/src/graphql/resolvers/users.ts index a2f4b9dcd..9418ef3e6 100644 --- a/backend/src/graphql/resolvers/users.ts +++ b/backend/src/graphql/resolvers/users.ts @@ -15,7 +15,7 @@ import { Context } from '@src/server' import { defaultTrophyBadge, defaultVerificationBadge } from './badges' import normalizeEmail from './helpers/normalizeEmail' import Resolver from './helpers/Resolver' -import { mergeImage, deleteImage } from './images/images' +import { images } from './images/images' import { createOrUpdateLocations } from './users/location' const neode = getNeode() @@ -210,7 +210,7 @@ export default { ) const [user] = updateUserTransactionResponse.records.map((record) => record.get('user')) if (avatarInput) { - await mergeImage(user, 'AVATAR_IMAGE', avatarInput, { transaction }) + await images.mergeImage(user, 'AVATAR_IMAGE', avatarInput, { transaction }) } return user }) @@ -253,7 +253,7 @@ export default { return Promise.all( txResult.records .map((record) => record.get('resource')) - .map((resource) => deleteImage(resource, 'HERO_IMAGE', { transaction })), + .map((resource) => images.deleteImage(resource, 'HERO_IMAGE', { transaction })), ) }), ) @@ -281,7 +281,7 @@ export default { { userId }, ) const [user] = deleteUserTransactionResponse.records.map((record) => record.get('user')) - await deleteImage(user, 'AVATAR_IMAGE', { transaction }) + await images.deleteImage(user, 'AVATAR_IMAGE', { transaction }) return user }) try { diff --git a/deployment/helm/charts/ocelot-social/templates/backend/stateful-set.yaml b/deployment/helm/charts/ocelot-social/templates/backend/stateful-set.yaml index 618a99f7f..604b79826 100644 --- a/deployment/helm/charts/ocelot-social/templates/backend/stateful-set.yaml +++ b/deployment/helm/charts/ocelot-social/templates/backend/stateful-set.yaml @@ -25,6 +25,9 @@ spec: name: {{ .Release.Name }}-backend-env - secretRef: name: {{ .Release.Name }}-backend-secret-env + volumeMounts: + - mountPath: /app/public/uploads + name: uploads containers: - name: {{ .Release.Name }}-backend image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag | default (include "defaultTag" .) }}" diff --git a/deployment/helm/helmfile/secrets/ocelot.yaml b/deployment/helm/helmfile/secrets/ocelot.yaml index 3965bc09e..41eff134c 100644 --- a/deployment/helm/helmfile/secrets/ocelot.yaml +++ b/deployment/helm/helmfile/secrets/ocelot.yaml @@ -23,15 +23,16 @@ secrets: NEO4J_USERNAME: null NEO4J_PASSWORD: null REDIS_PASSWORD: null + AWS_ACCESS_KEY_ID: ENC[AES256_GCM,data:iiN5ueqyo60VHb9e2bnhc19iGTg=,iv:zawYpKrFafgsu1+YRet1hzZf1G3a6BIlZgsh7xNADaE=,tag:rTsmm8cqei34b6cT6vn08w==,type:str] + AWS_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:Zl4LRXdDh/6Q8F9RVp+3L7NXGZ0F2cgFMKPhl/TVeuD5Bhy68W5ekg==,iv:AmPoinGISrSOZdoBKdeFFXfr2hwOK4nWMnniz8K5qgU=,tag:K8Q7M7e+6G9T0Oh3Sp4OzA==,type:str] + AWS_ENDPOINT: ENC[AES256_GCM,data:/waEqUgcOmldZ+peFTNVsDQf2KrpWY8ZZMt1nT5117SkbY4=,iv:n+Kvidjb/TM4bQYKqTaFxt8GkHo02PuxEGpzgOcywr4=,tag:lrGPgCWWy3GMIcTv75IYTg==,type:str] + AWS_REGION: ENC[AES256_GCM,data:kBPpHZ8zw4PMpg==,iv:R+QZe303do37Hd/97NpS1pt9VaBE/gqZDY2/qlIvvps=,tag:0WduW8wfJXtBqlh4qfRGNA==,type:str] + AWS_BUCKET: ENC[AES256_GCM,data:0fAspN/PoRVPlSbz+qDBRUOieeC4,iv:JGJ/LyLpMymN0tpZmW6DjPT3xqXzK/KhYQsy9sgPd60=,tag:Y6PBs0916JkHRHSe7hqSMA==,type:str] neo4j: env: NEO4J_USERNAME: "" NEO4J_PASSWORD: "" sops: - kms: [] - gcp_kms: [] - azure_kv: [] - hc_vault: [] age: - recipient: age1llp6k66265q3rzqemxpnq0x3562u20989vcjf65fl9s3hjhgcscq6mhnjw enc: | @@ -69,8 +70,7 @@ sops: aGNFeXZZRmlJM041OHdTM0pmM3BBdGMKGvFgYY1jhKwciAOZKyw0hlFVNbOk7CM7 041g17JXNV1Wk6WgMZ4w8p54RKQVaWCT4wxChy6wNNdQ3IeKgqEU2w== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-10-29T14:26:49Z" - mac: ENC[AES256_GCM,data:YXX7MEAK0wmuxLTmdr7q5uVd6DG6FhGUeE+EzbhWe/OovH6n+CjKZGklnEX+5ztDO0IgZh/T9Hx1CgFYuVbcOkvDoFBDwNpRA/QOQrM0p/+tRlMNCypC/Wh2xL0DhA4A/Qum2oyE/BDkt1Yy8N5wZDZn575+ZAjXEgAzlhpT5qk=,iv:ire3gkHTY6+0lgbV1Es6Lf8bcKTg4WKnq46M+b/VRcU=,tag:MkZULKcwROvIw/C0YtcUbA==,type:str] - pgp: [] + lastmodified: "2025-05-29T06:57:01Z" + mac: ENC[AES256_GCM,data:0eWUqVsJrolrVYFsG4aigAYSjBW35Z+64y/ZE8Az15GmI0E4p/1kZPIY6hv9SUiCD1R2Ro0nm1QsV9A/hvNeWla0EdARNnARxIphxMJWKRGwcVkFVk28Jh4g4jE/rTggWx/wLbR6bza1RLHA1wRMb1PYuLfZybsb/whN4+gTMfg=,iv:irQkLpj1+3egSsiV9HqZP4tgG1fotCOizubL42gRjSQ=,tag:DC0xzG/VqcL4ib6ijxQZnA==,type:str] unencrypted_suffix: _unencrypted - version: 3.9.0 + version: 3.10.2 diff --git a/docker-compose.override.yml b/docker-compose.override.yml index ae77abd5e..a9e957dca 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -25,6 +25,9 @@ services: backend: image: ghcr.io/ocelot-social-community/ocelot-social/backend:local-development + depends_on: + - minio + - minio-mc build: target: development environment: @@ -32,6 +35,12 @@ services: - DEBUG=true - SMTP_PORT=1025 - SMTP_HOST=mailserver + - AWS_ACCESS_KEY_ID=minio + - AWS_SECRET_ACCESS_KEY=12341234 + - AWS_ENDPOINT=http:/minio:9000 + - AWS_REGION=local + - AWS_BUCKET=ocelot + - S3_PUBLIC_GATEWAY=http:/localhost:9000 volumes: - ./backend:/app @@ -46,3 +55,33 @@ services: ports: - 1080:1080 - 1025:1025 + + minio: + image: quay.io/minio/minio + ports: + - 9000:9000 + - 9001:9001 + volumes: + - minio_data:/data + environment: + - MINIO_ROOT_USER=minio + - MINIO_ROOT_PASSWORD=12341234 + command: server /data --console-address ":9001" + + minio-mc: + image: quay.io/minio/mc + depends_on: + - minio + restart: on-failure + volumes: + - ./minio/readonly-policy.json:/tmp/readonly-policy.json + entrypoint: > + /bin/sh -c " + sleep 5; + /usr/bin/mc alias set dockerminio http://minio:9000 minio 12341234; + /usr/bin/mc mb --ignore-existing dockerminio/ocelot; + /usr/bin/mc anonymous set-json /tmp/readonly-policy.json dockerminio/ocelot; + " + +volumes: + minio_data: diff --git a/docker-compose.test.yml b/docker-compose.test.yml index cc7490106..edd2f30d2 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -13,10 +13,19 @@ services: backend: # name the image so that it cannot be found in a DockerHub repository, otherwise it will not be built locally from the 'dockerfile' but pulled from there image: ghcr.io/ocelot-social-community/ocelot-social/backend:test + depends_on: + - minio + - minio-mc + - mailserver build: target: test environment: - NODE_ENV="test" + - AWS_ACCESS_KEY_ID=minio + - AWS_SECRET_ACCESS_KEY=12341234 + - AWS_ENDPOINT=http:/minio:9000 + - AWS_REGION=local + - AWS_BUCKET=ocelot volumes: - ./coverage:/app/coverage @@ -41,3 +50,33 @@ services: ports: - 1080:1080 - 1025:1025 + + minio: + image: quay.io/minio/minio + ports: + - 9000:9000 + - 9001:9001 + volumes: + - minio_data:/data + environment: + - MINIO_ROOT_USER=minio + - MINIO_ROOT_PASSWORD=12341234 + command: server /data --console-address ":9001" + + minio-mc: + image: quay.io/minio/mc + depends_on: + - minio + restart: on-failure + volumes: + - ./minio/readonly-policy.json:/tmp/readonly-policy.json + entrypoint: > + /bin/sh -c " + sleep 5; + /usr/bin/mc alias set dockerminio http://minio:9000 minio 12341234; + /usr/bin/mc mb --ignore-existing dockerminio/ocelot; + /usr/bin/mc anonymous set-json /tmp/readonly-policy.json dockerminio/ocelot; + " + +volumes: + minio_data: diff --git a/docker-compose.yml b/docker-compose.yml index 8397c4e47..4a1e9e951 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,8 +53,6 @@ services: - neo4j ports: - 4000:4000 - volumes: - - backend_uploads:/app/public/uploads environment: # Envs used in Dockerfile # - DOCKER_WORKDIR="/app" @@ -104,5 +102,4 @@ services: # command: ["tail", "-f", "/dev/null"] volumes: - backend_uploads: neo4j_data: diff --git a/minio/readonly-policy.json b/minio/readonly-policy.json new file mode 100644 index 000000000..6cdbdd667 --- /dev/null +++ b/minio/readonly-policy.json @@ -0,0 +1,19 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": [ + "*" + ] + }, + "Action": [ + "s3:GetObject" + ], + "Resource": [ + "arn:aws:s3:::ocelot/*" + ] + } + ] +}