diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 766717a97..a7eff0ef2 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -113,7 +113,7 @@ jobs: - name: backend | docker compose # doesn't work without the --build flag - this either means we should not load the cached images or cache the correct image - run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach --no-deps neo4j backend --build + run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach backend --build - name: backend | Initialize Database run: docker compose exec -T backend yarn db:migrate init diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index d232e2909..fc6a2ea65 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -135,7 +135,7 @@ jobs: docker load < /tmp/neo4j.tar docker load < /tmp/backend.tar docker load < /tmp/webapp.tar - docker compose -f docker-compose.yml -f docker-compose.test.yml up --build --detach --no-deps webapp neo4j backend mailserver + docker compose -f docker-compose.yml -f docker-compose.test.yml up --build --detach webapp mailserver sleep 90s - name: Full stack tests | run tests @@ -176,4 +176,4 @@ jobs: done echo "Done" env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/backend/.dockerignore b/backend/.dockerignore index a0883bf4d..af1b934a3 100644 --- a/backend/.dockerignore +++ b/backend/.dockerignore @@ -17,5 +17,4 @@ build/ maintenance-worker/ neo4j/ -public/uploads/* !.gitkeep diff --git a/backend/.env.template b/backend/.env.template index 5aa44036b..e8c1f4168 100644 --- a/backend/.env.template +++ b/backend/.env.template @@ -40,11 +40,12 @@ COMMIT= PUBLIC_REGISTRATION=false INVITE_REGISTRATION=true -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= -AWS_ENDPOINT= -AWS_REGION= -AWS_BUCKET= +AWS_ACCESS_KEY_ID=minio +AWS_SECRET_ACCESS_KEY=12341234 +AWS_ENDPOINT=http://localhost:9000 +AWS_REGION=local +AWS_BUCKET=ocelot +S3_PUBLIC_GATEWAY=http://localhost:8000 CATEGORIES_ACTIVE=false MAX_PINNED_POSTS=1 diff --git a/backend/.gitignore b/backend/.gitignore index 833f7e34e..a5d3db1fa 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -6,8 +6,7 @@ yarn-error.log build/* coverage.lcov .nyc_output/ -public/uploads/* !.gitkeep # Apple macOS folder attribute file -.DS_Store \ No newline at end of file +.DS_Store diff --git a/backend/src/config/index.ts b/backend/src/config/index.ts index 82f0f674e..0aee79626 100644 --- a/backend/src/config/index.ts +++ b/backend/src/config/index.ts @@ -102,12 +102,26 @@ const s3 = { AWS_ENDPOINT: env.AWS_ENDPOINT, AWS_REGION: env.AWS_REGION, AWS_BUCKET: env.AWS_BUCKET, - S3_CONFIGURED: - env.AWS_ACCESS_KEY_ID && - env.AWS_SECRET_ACCESS_KEY && - env.AWS_ENDPOINT && - env.AWS_REGION && - env.AWS_BUCKET, + S3_PUBLIC_GATEWAY: env.S3_PUBLIC_GATEWAY, +} + +export interface S3Configured { + AWS_ACCESS_KEY_ID: string + AWS_SECRET_ACCESS_KEY: string + AWS_ENDPOINT: string + AWS_REGION: string + AWS_BUCKET: string + S3_PUBLIC_GATEWAY: string | undefined +} + +export const isS3configured = (config: typeof s3): config is S3Configured => { + return !!( + config.AWS_ACCESS_KEY_ID && + config.AWS_SECRET_ACCESS_KEY && + config.AWS_ENDPOINT && + config.AWS_REGION && + config.AWS_BUCKET + ) } const options = { diff --git a/backend/src/db/migrations/20250502230521-migrate-to-s3.ts b/backend/src/db/migrations/20250502230521-migrate-to-s3.ts new file mode 100644 index 000000000..243b7f4e9 --- /dev/null +++ b/backend/src/db/migrations/20250502230521-migrate-to-s3.ts @@ -0,0 +1,91 @@ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ + +import { open } from 'node:fs/promises' +import path from 'node:path' + +import { S3Client, ObjectCannedACL } from '@aws-sdk/client-s3' +import { Upload } from '@aws-sdk/lib-storage' +import { lookup } from 'mime-types' + +import CONFIG from '@config/index' +import { getDriver } from '@db/neo4j' + +export const description = + 'Upload all image files to a S3 compatible object storage in order to reduce load on our backend.' + +export async function up(_next) { + const { AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_ENDPOINT, AWS_BUCKET } = CONFIG + if (!(AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY && AWS_ENDPOINT && AWS_BUCKET)) { + throw new Error('No S3 configuration given, cannot upload image files') + } + + const driver = getDriver() + const session = driver.session() + const transaction = session.beginTransaction() + + const s3 = new S3Client({ + credentials: { + accessKeyId: AWS_ACCESS_KEY_ID, + secretAccessKey: AWS_SECRET_ACCESS_KEY, + }, + endpoint: AWS_ENDPOINT, + forcePathStyle: true, + }) + try { + const { records } = await transaction.run('MATCH (image:Image) RETURN image.url as url') + let urls: string[] = records.map((r) => r.get('url') as string) + urls = urls.filter((url) => url.startsWith('/uploads')) + // eslint-disable-next-line no-console + console.log('URLS uploaded:') + await Promise.all( + urls + .map((url) => async () => { + const { pathname } = new URL(url, 'http://example.org') + // TODO: find a better way to do this - this is quite a hack + const fileLocation = + CONFIG.NODE_ENV === 'production' + ? path.join(__dirname, `../../../../public/${pathname}`) // we're in the /build folder + : path.join(__dirname, `../../../public/${pathname}`) + const s3Location = `original${pathname}` + const mimeType = lookup(fileLocation) + // eslint-disable-next-line security/detect-non-literal-fs-filename + const fileHandle = await open(fileLocation) + const params = { + Bucket: AWS_BUCKET, + Key: s3Location, + ACL: ObjectCannedACL.public_read, + ContentType: mimeType || 'image/jpeg', + Body: fileHandle.createReadStream(), + } + const command = new Upload({ client: s3, params }) + + const data = await command.done() + const { Location: spacesUrl } = data + + const updatedRecord = await transaction.run( + 'MATCH (image:Image {url: $url}) SET image.url = $spacesUrl RETURN image.url as url', + { url, spacesUrl }, + ) + const [updatedUrl] = updatedRecord.records.map((record) => record.get('url') as string) + // eslint-disable-next-line no-console + console.log(updatedUrl) + return updatedUrl + }) + .map((p) => p()), + ) + await transaction.commit() + } catch (error) { + // eslint-disable-next-line no-console + console.log(error) + await transaction.rollback() + // eslint-disable-next-line no-console + console.log('rolled back') + throw new Error(error) + } finally { + await session.close() + } +} + +export function down(_next) { + throw new Error('This migration is irreversible: The backend does not have disk access anymore.') +} diff --git a/backend/src/graphql/resolvers/groups.ts b/backend/src/graphql/resolvers/groups.ts index a3ce3285a..9e330bade 100644 --- a/backend/src/graphql/resolvers/groups.ts +++ b/backend/src/graphql/resolvers/groups.ts @@ -18,7 +18,7 @@ import Resolver, { removeUndefinedNullValuesFromObject, convertObjectToCypherMapLiteral, } from './helpers/Resolver' -import { mergeImage } from './images/images' +import { images } from './images/images' import { createOrUpdateLocations } from './users/location' export default { @@ -260,7 +260,7 @@ export default { }) const [group] = transactionResponse.records.map((record) => record.get('group')) if (avatarInput) { - await mergeImage(group, 'AVATAR_IMAGE', avatarInput, { transaction }) + await images.mergeImage(group, 'AVATAR_IMAGE', avatarInput, { transaction }) } return group }) diff --git a/backend/src/graphql/resolvers/images/images.ts b/backend/src/graphql/resolvers/images/images.ts index f437b3c85..6c2fa8b3a 100644 --- a/backend/src/graphql/resolvers/images/images.ts +++ b/backend/src/graphql/resolvers/images/images.ts @@ -1,32 +1,27 @@ -/* eslint-disable @typescript-eslint/require-await */ -/* eslint-disable @typescript-eslint/restrict-template-expressions */ -/* eslint-disable @typescript-eslint/no-unsafe-call */ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -/* eslint-disable @typescript-eslint/no-unsafe-argument */ -/* eslint-disable promise/avoid-new */ -/* eslint-disable security/detect-non-literal-fs-filename */ +import CONFIG, { isS3configured } from '@config/index' -import { existsSync, unlinkSync, createWriteStream } from 'node:fs' -import path from 'node:path' - -import { S3Client, DeleteObjectCommand, ObjectCannedACL } from '@aws-sdk/client-s3' -import { Upload } from '@aws-sdk/lib-storage' -import { UserInputError } from 'apollo-server' -import slug from 'slug' -import { v4 as uuid } from 'uuid' - -import CONFIG from '@config/index' -import { getDriver } from '@db/neo4j' +import { images as imagesLocal } from './imagesLocal' +import { images as imagesS3 } from './imagesS3' import type { FileUpload } from 'graphql-upload' import type { Transaction } from 'neo4j-driver' -type FileDeleteCallback = (url: string) => Promise -type FileUploadCallback = ( +export type FileDeleteCallback = (url: string) => Promise + +export type FileUploadCallback = ( upload: Pick & { uniqueFilename: string }, ) => Promise +export interface DeleteImageOpts { + transaction?: Transaction + deleteCallback?: FileDeleteCallback +} + +export interface MergeImageOpts { + transaction?: Transaction + uploadCallback?: FileUploadCallback + deleteCallback?: FileDeleteCallback +} + export interface ImageInput { upload?: Promise alt?: string @@ -35,191 +30,29 @@ export interface ImageInput { type?: string } -// const widths = [34, 160, 320, 640, 1024] -const { AWS_BUCKET: Bucket, S3_CONFIGURED } = CONFIG - -const createS3Client = () => { - const { AWS_ENDPOINT, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY } = CONFIG - if (!(AWS_ENDPOINT && AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY)) { - throw new Error('Missing AWS credentials.') - } - return new S3Client({ - credentials: { - accessKeyId: AWS_ACCESS_KEY_ID, - secretAccessKey: AWS_SECRET_ACCESS_KEY, - }, - endpoint: AWS_ENDPOINT, - forcePathStyle: true, - }) +export interface Image { + url: string + alt?: string + sensitive?: boolean + aspectRatio?: number + type?: string } -interface DeleteImageOpts { - transaction?: Transaction - deleteCallback?: FileDeleteCallback -} -export async function deleteImage(resource, relationshipType, opts: DeleteImageOpts = {}) { - sanitizeRelationshipType(relationshipType) - const { transaction, deleteCallback } = opts - if (!transaction) return wrapTransaction(deleteImage, [resource, relationshipType], opts) - const txResult = await transaction.run( - ` - MATCH (resource {id: $resource.id})-[rel:${relationshipType}]->(image:Image) - WITH image, image {.*} as imageProps - DETACH DELETE image - RETURN imageProps - `, - { resource }, - ) - const [image] = txResult.records.map((record) => record.get('imageProps')) - // This behaviour differs from `mergeImage`. If you call `mergeImage` - // with metadata for an image that does not exist, it's an indicator - // of an error (so throw an error). If we bulk delete an image, it - // could very well be that there is no image for the resource. - if (image) deleteImageFile(image, deleteCallback) - return image +export interface Images { + deleteImage: ( + resource: { id: string }, + relationshipType: 'HERO_IMAGE' | 'AVATAR_IMAGE', + opts?: DeleteImageOpts, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ) => Promise + + mergeImage: ( + resource: { id: string }, + relationshipType: 'HERO_IMAGE' | 'AVATAR_IMAGE', + imageInput: ImageInput | null | undefined, + opts?: MergeImageOpts, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ) => Promise } -interface MergeImageOpts { - transaction?: Transaction - uploadCallback?: FileUploadCallback - deleteCallback?: FileDeleteCallback -} - -export async function mergeImage( - resource, - relationshipType, - imageInput: ImageInput | null | undefined, - opts: MergeImageOpts = {}, -) { - if (typeof imageInput === 'undefined') return - if (imageInput === null) return deleteImage(resource, relationshipType, opts) - sanitizeRelationshipType(relationshipType) - const { transaction, uploadCallback, deleteCallback } = opts - if (!transaction) - return wrapTransaction(mergeImage, [resource, relationshipType, imageInput], opts) - - let txResult - txResult = await transaction.run( - ` - MATCH (resource {id: $resource.id})-[:${relationshipType}]->(image:Image) - RETURN image {.*} - `, - { resource }, - ) - const [existingImage] = txResult.records.map((record) => record.get('image')) - const { upload } = imageInput - if (!(existingImage || upload)) throw new UserInputError('Cannot find image for given resource') - if (existingImage && upload) deleteImageFile(existingImage, deleteCallback) - const url = await uploadImageFile(upload, uploadCallback) - const { alt, sensitive, aspectRatio, type } = imageInput - const image = { alt, sensitive, aspectRatio, url, type } - txResult = await transaction.run( - ` - MATCH (resource {id: $resource.id}) - MERGE (resource)-[:${relationshipType}]->(image:Image) - ON CREATE SET image.createdAt = toString(datetime()) - ON MATCH SET image.updatedAt = toString(datetime()) - SET image += $image - RETURN image {.*} - `, - { resource, image }, - ) - const [mergedImage] = txResult.records.map((record) => record.get('image')) - return mergedImage -} - -const wrapTransaction = async (wrappedCallback, args, opts) => { - const session = getDriver().session() - try { - const result = await session.writeTransaction(async (transaction) => { - return wrappedCallback(...args, { ...opts, transaction }) - }) - return result - } finally { - await session.close() - } -} - -const deleteImageFile = (image, deleteCallback: FileDeleteCallback | undefined) => { - if (!deleteCallback) { - deleteCallback = S3_CONFIGURED ? s3Delete : localFileDelete - } - const { url } = image - // eslint-disable-next-line @typescript-eslint/no-floating-promises - deleteCallback(url) - return url -} - -const uploadImageFile = async ( - upload: Promise | undefined, - uploadCallback: FileUploadCallback | undefined, -) => { - if (!upload) return undefined - if (!uploadCallback) { - uploadCallback = S3_CONFIGURED ? s3Upload : localFileUpload - } - // eslint-disable-next-line @typescript-eslint/unbound-method - const { createReadStream, filename, mimetype } = await upload - const { name, ext } = path.parse(filename) - const uniqueFilename = `${uuid()}-${slug(name)}${ext}` - return uploadCallback({ createReadStream, uniqueFilename, mimetype }) -} - -const sanitizeRelationshipType = (relationshipType) => { - // Cypher query language does not allow to parameterize relationship types - // See: https://github.com/neo4j/neo4j/issues/340 - if (!['HERO_IMAGE', 'AVATAR_IMAGE'].includes(relationshipType)) { - throw new Error(`Unknown relationship type ${relationshipType}`) - } -} - -const localFileUpload: FileUploadCallback = ({ createReadStream, uniqueFilename }) => { - const destination = `/uploads/${uniqueFilename}` - return new Promise((resolve, reject) => - createReadStream().pipe( - createWriteStream(`public${destination}`) - .on('finish', () => resolve(destination)) - .on('error', (error) => reject(error)), - ), - ) -} - -const s3Upload: FileUploadCallback = async ({ createReadStream, uniqueFilename, mimetype }) => { - const s3Location = `original/${uniqueFilename}` - const params = { - Bucket, - Key: s3Location, - ACL: ObjectCannedACL.public_read, - ContentType: mimetype, - Body: createReadStream(), - } - const s3 = createS3Client() - const command = new Upload({ client: s3, params }) - const data = await command.done() - const { Location } = data - if (!Location) { - throw new Error('File upload did not return `Location`') - } - return Location -} - -const localFileDelete: FileDeleteCallback = async (url) => { - const location = `public${url}` - // eslint-disable-next-line n/no-sync - if (existsSync(location)) unlinkSync(location) -} - -const s3Delete: FileDeleteCallback = async (url) => { - let { pathname } = new URL(url, 'http://example.org') // dummy domain to avoid invalid URL error - pathname = pathname.substring(1) // remove first character '/' - const prefix = `${Bucket}/` - if (pathname.startsWith(prefix)) { - pathname = pathname.slice(prefix.length) - } - const params = { - Bucket, - Key: pathname, - } - const s3 = createS3Client() - await s3.send(new DeleteObjectCommand(params)) -} +export const images = isS3configured(CONFIG) ? imagesS3(CONFIG) : imagesLocal diff --git a/backend/src/graphql/resolvers/images/images.spec.ts b/backend/src/graphql/resolvers/images/imagesLocal.spec.ts similarity index 93% rename from backend/src/graphql/resolvers/images/images.spec.ts rename to backend/src/graphql/resolvers/images/imagesLocal.spec.ts index 75f5f4feb..257a24e78 100644 --- a/backend/src/graphql/resolvers/images/images.spec.ts +++ b/backend/src/graphql/resolvers/images/imagesLocal.spec.ts @@ -11,7 +11,7 @@ import { UserInputError } from 'apollo-server' import Factory, { cleanDatabase } from '@db/factories' import { getNeode, getDriver } from '@db/neo4j' -import { deleteImage, mergeImage } from './images' +import { images } from './imagesLocal' import type { ImageInput } from './images' import type { FileUpload } from 'graphql-upload' @@ -42,10 +42,12 @@ afterEach(async () => { }) describe('deleteImage', () => { + const { deleteImage } = images + describe('given a resource with an image', () => { - let user + let user: { id: string } beforeEach(async () => { - user = await Factory.build( + const u = await Factory.build( 'user', {}, { @@ -55,7 +57,7 @@ describe('deleteImage', () => { }), }, ) - user = await user.toJson() + user = await u.toJson() }) it('deletes `Image` node', async () => { @@ -65,8 +67,8 @@ describe('deleteImage', () => { }) it('calls deleteCallback', async () => { - user = await Factory.build('user') - user = await user.toJson() + const u = await Factory.build('user') + user = await u.toJson() await deleteImage(user, 'AVATAR_IMAGE', { deleteCallback }) expect(deleteCallback).toHaveBeenCalled() }) @@ -117,8 +119,9 @@ describe('deleteImage', () => { }) describe('mergeImage', () => { + const { mergeImage } = images let imageInput: ImageInput - let post + let post: { id: string } beforeEach(() => { imageInput = { alt: 'A description of the new image', @@ -145,7 +148,7 @@ describe('mergeImage', () => { describe('on existing resource', () => { beforeEach(async () => { - post = await Factory.build( + const p = await Factory.build( 'post', { id: 'p99' }, { @@ -153,7 +156,7 @@ describe('mergeImage', () => { image: null, }, ) - post = await post.toJson() + post = await p.toJson() }) it('returns new image', async () => { @@ -196,7 +199,7 @@ describe('mergeImage', () => { `MATCH(p:Post {id: "p99"})-[:HERO_IMAGE]->(i:Image) RETURN i,p`, {}, ) - post = neode.hydrateFirst(result, 'p', neode.model('Post')) + post = neode.hydrateFirst<{ id: string }>(result, 'p', neode.model('Post')).properties() const image = neode.hydrateFirst(result, 'i', neode.model('Image')) expect(post).toBeTruthy() expect(image).toBeTruthy() @@ -204,7 +207,10 @@ describe('mergeImage', () => { it('whitelists relationship types', async () => { await expect( - mergeImage(post, 'WHATEVER', imageInput, { uploadCallback, deleteCallback }), + mergeImage(post, 'WHATEVER' as 'HERO_IMAGE', imageInput, { + uploadCallback, + deleteCallback, + }), ).rejects.toEqual(new Error('Unknown relationship type WHATEVER')) }) @@ -311,8 +317,8 @@ describe('mergeImage', () => { describe('without image.upload', () => { it('throws UserInputError', async () => { - post = await Factory.build('post', { id: 'p99' }, { image: null }) - post = await post.toJson() + const p = await Factory.build('post', { id: 'p99' }, { image: null }) + post = await p.toJson() await expect(mergeImage(post, 'HERO_IMAGE', imageInput)).rejects.toEqual( new UserInputError('Cannot find image for given resource'), ) @@ -320,7 +326,7 @@ describe('mergeImage', () => { describe('if resource has an image already', () => { beforeEach(async () => { - post = await Factory.build( + const p = await Factory.build( 'post', { id: 'p99', @@ -339,7 +345,7 @@ describe('mergeImage', () => { }), }, ) - post = await post.toJson() + post = await p.toJson() }) it('does not call deleteCallback', async () => { diff --git a/backend/src/graphql/resolvers/images/imagesLocal.ts b/backend/src/graphql/resolvers/images/imagesLocal.ts new file mode 100644 index 000000000..c9f575777 --- /dev/null +++ b/backend/src/graphql/resolvers/images/imagesLocal.ts @@ -0,0 +1,134 @@ +/* eslint-disable @typescript-eslint/require-await */ + +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ +/* eslint-disable promise/avoid-new */ +/* eslint-disable security/detect-non-literal-fs-filename */ + +import { existsSync, unlinkSync, createWriteStream } from 'node:fs' +import path from 'node:path' + +import { UserInputError } from 'apollo-server' +import slug from 'slug' +import { v4 as uuid } from 'uuid' + +import { sanitizeRelationshipType } from './sanitizeRelationshipTypes' +import { wrapTransaction } from './wrapTransaction' + +import type { Images, FileDeleteCallback, FileUploadCallback } from './images' +import type { FileUpload } from 'graphql-upload' + +const deleteImage: Images['deleteImage'] = async (resource, relationshipType, opts = {}) => { + sanitizeRelationshipType(relationshipType) + const { transaction, deleteCallback } = opts + if (!transaction) return wrapTransaction(deleteImage, [resource, relationshipType], opts) + const txResult = await transaction.run( + ` + MATCH (resource {id: $resource.id})-[rel:${relationshipType}]->(image:Image) + WITH image, image {.*} as imageProps + DETACH DELETE image + RETURN imageProps + `, + { resource }, + ) + const [image] = txResult.records.map((record) => record.get('imageProps')) + // This behaviour differs from `mergeImage`. If you call `mergeImage` + // with metadata for an image that does not exist, it's an indicator + // of an error (so throw an error). If we bulk delete an image, it + // could very well be that there is no image for the resource. + if (image) deleteImageFile(image, deleteCallback) + return image +} + +const mergeImage: Images['mergeImage'] = async ( + resource, + relationshipType, + imageInput, + opts = {}, +) => { + if (typeof imageInput === 'undefined') return + if (imageInput === null) return deleteImage(resource, relationshipType, opts) + sanitizeRelationshipType(relationshipType) + const { transaction, uploadCallback, deleteCallback } = opts + if (!transaction) + return wrapTransaction(mergeImage, [resource, relationshipType, imageInput], opts) + + let txResult + txResult = await transaction.run( + ` + MATCH (resource {id: $resource.id})-[:${relationshipType}]->(image:Image) + RETURN image {.*} + `, + { resource }, + ) + const [existingImage] = txResult.records.map((record) => record.get('image')) + const { upload } = imageInput + if (!(existingImage || upload)) throw new UserInputError('Cannot find image for given resource') + if (existingImage && upload) deleteImageFile(existingImage, deleteCallback) + const url = await uploadImageFile(upload, uploadCallback) + const { alt, sensitive, aspectRatio, type } = imageInput + const image = { alt, sensitive, aspectRatio, url, type } + txResult = await transaction.run( + ` + MATCH (resource {id: $resource.id}) + MERGE (resource)-[:${relationshipType}]->(image:Image) + ON CREATE SET image.createdAt = toString(datetime()) + ON MATCH SET image.updatedAt = toString(datetime()) + SET image += $image + RETURN image {.*} + `, + { resource, image }, + ) + const [mergedImage] = txResult.records.map((record) => record.get('image')) + return mergedImage +} + +const localFileDelete: FileDeleteCallback = async (url) => { + const location = `public${url}` + // eslint-disable-next-line n/no-sync + if (existsSync(location)) unlinkSync(location) +} + +const deleteImageFile = (image, deleteCallback: FileDeleteCallback | undefined) => { + if (!deleteCallback) { + deleteCallback = localFileDelete + } + const { url } = image + // eslint-disable-next-line @typescript-eslint/no-floating-promises + deleteCallback(url) + return url +} + +const uploadImageFile = async ( + upload: Promise | undefined, + uploadCallback: FileUploadCallback | undefined, +) => { + if (!upload) return undefined + if (!uploadCallback) { + uploadCallback = localFileUpload + } + // eslint-disable-next-line @typescript-eslint/unbound-method + const { createReadStream, filename, mimetype } = await upload + const { name, ext } = path.parse(filename) + const uniqueFilename = `${uuid()}-${slug(name)}${ext}` + return uploadCallback({ createReadStream, uniqueFilename, mimetype }) +} + +const localFileUpload: FileUploadCallback = ({ createReadStream, uniqueFilename }) => { + const destination = `/uploads/${uniqueFilename}` + return new Promise((resolve, reject) => + createReadStream().pipe( + createWriteStream(`public${destination}`) + .on('finish', () => resolve(destination)) + .on('error', (error) => reject(error)), + ), + ) +} + +export const images: Images = { + deleteImage, + mergeImage, +} diff --git a/backend/src/graphql/resolvers/images/imagesS3.spec.ts b/backend/src/graphql/resolvers/images/imagesS3.spec.ts new file mode 100644 index 000000000..2bedec3cd --- /dev/null +++ b/backend/src/graphql/resolvers/images/imagesS3.spec.ts @@ -0,0 +1,407 @@ +/* eslint-disable @typescript-eslint/require-await */ + +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable @typescript-eslint/restrict-template-expressions */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable promise/prefer-await-to-callbacks */ +import { UserInputError } from 'apollo-server' + +import Factory, { cleanDatabase } from '@db/factories' +import { getNeode, getDriver } from '@db/neo4j' +import type { S3Configured } from '@src/config' + +import { images } from './imagesS3' + +import type { ImageInput } from './images' +import type { FileUpload } from 'graphql-upload' + +const driver = getDriver() +const neode = getNeode() +const uuid = '[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}' +let uploadCallback +let deleteCallback + +const config: S3Configured = { + AWS_ACCESS_KEY_ID: 'AWS_ACCESS_KEY_ID', + AWS_SECRET_ACCESS_KEY: 'AWS_SECRET_ACCESS_KEY', + AWS_BUCKET: 'AWS_BUCKET', + AWS_ENDPOINT: 'AWS_ENDPOINT', + AWS_REGION: 'AWS_REGION', + S3_PUBLIC_GATEWAY: undefined, +} + +beforeAll(async () => { + await cleanDatabase() +}) + +afterAll(async () => { + await cleanDatabase() + await driver.close() +}) + +beforeEach(async () => { + uploadCallback = jest.fn( + ({ uniqueFilename }) => `http://your-objectstorage.com/bucket/${uniqueFilename}`, + ) + deleteCallback = jest.fn() +}) + +// TODO: avoid database clean after each test in the future if possible for performance and flakyness reasons by filling the database step by step, see issue https://github.com/Ocelot-Social-Community/Ocelot-Social/issues/4543 +afterEach(async () => { + await cleanDatabase() +}) + +describe('deleteImage', () => { + const { deleteImage } = images(config) + describe('given a resource with an image', () => { + let user: { id: string } + beforeEach(async () => { + const u = await Factory.build( + 'user', + {}, + { + avatar: Factory.build('image', { + url: 'http://localhost/some/avatar/url/', + alt: 'This is the avatar image of a user', + }), + }, + ) + user = await u.toJson() + }) + + it('deletes `Image` node', async () => { + await expect(neode.all('Image')).resolves.toHaveLength(1) + await deleteImage(user, 'AVATAR_IMAGE', { deleteCallback }) + await expect(neode.all('Image')).resolves.toHaveLength(0) + }) + + it('calls deleteCallback', async () => { + await deleteImage(user, 'AVATAR_IMAGE', { deleteCallback }) + expect(deleteCallback).toHaveBeenCalled() + }) + + describe('given a transaction parameter', () => { + it('executes cypher statements within the transaction', async () => { + const session = driver.session() + let someString: string + try { + someString = await session.writeTransaction(async (transaction) => { + await deleteImage(user, 'AVATAR_IMAGE', { + deleteCallback, + transaction, + }) + const txResult = await transaction.run('RETURN "Hello" as result') + const [result] = txResult.records.map((record) => record.get('result')) + return result + }) + } finally { + await session.close() + } + await expect(neode.all('Image')).resolves.toHaveLength(0) + expect(someString).toEqual('Hello') + }) + + it('rolls back the transaction in case of errors', async () => { + await expect(neode.all('Image')).resolves.toHaveLength(1) + const session = driver.session() + try { + await session.writeTransaction(async (transaction) => { + await deleteImage(user, 'AVATAR_IMAGE', { + deleteCallback, + transaction, + }) + throw new Error('Ouch!') + }) + // eslint-disable-next-line no-catch-all/no-catch-all + } catch (err) { + // nothing has been deleted + await expect(neode.all('Image')).resolves.toHaveLength(1) + // all good + } finally { + await session.close() + } + }) + }) + }) +}) + +describe('mergeImage', () => { + const { mergeImage } = images(config) + let imageInput: ImageInput + let post: { id: string } + beforeEach(() => { + imageInput = { + alt: 'A description of the new image', + } + }) + + describe('given image.upload', () => { + beforeEach(() => { + const createReadStream: FileUpload['createReadStream'] = (() => ({ + pipe: () => ({ + on: (_, callback) => callback(), + }), + })) as unknown as FileUpload['createReadStream'] + imageInput = { + ...imageInput, + upload: Promise.resolve({ + filename: 'image.jpg', + mimetype: 'image/jpeg', + encoding: '7bit', + createReadStream, + }), + } + }) + + describe('on existing resource', () => { + beforeEach(async () => { + const p = await Factory.build( + 'post', + { id: 'p99' }, + { + author: Factory.build('user', {}, { avatar: null }), + image: null, + }, + ) + post = await p.toJson() + }) + + it('returns new image', async () => { + await expect( + mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }), + ).resolves.toMatchObject({ + url: expect.any(String), + alt: 'A description of the new image', + }) + }) + + it('calls upload callback', async () => { + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + expect(uploadCallback).toHaveBeenCalled() + }) + + it('creates `:Image` node', async () => { + await expect(neode.all('Image')).resolves.toHaveLength(0) + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + await expect(neode.all('Image')).resolves.toHaveLength(1) + }) + + it('creates a url safe name', async () => { + if (!imageInput.upload) { + throw new Error('Test imageInput was not setup correctly.') + } + const upload = await imageInput.upload + upload.filename = '/path/to/awkward?/ file-location/?foo- bar-avatar.jpg' + imageInput.upload = Promise.resolve(upload) + await expect( + mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }), + ).resolves.toMatchObject({ + url: expect.stringMatching( + new RegExp(`^http://your-objectstorage.com/bucket/${uuid}-foo-bar-avatar.jpg`), + ), + }) + }) + + describe('given a `S3_PUBLIC_GATEWAY` configuration', () => { + const { mergeImage } = images({ + ...config, + S3_PUBLIC_GATEWAY: 'http://s3-public-gateway.com', + }) + + it('changes the domain of the URL to a server that could e.g. apply image transformations', async () => { + if (!imageInput.upload) { + throw new Error('Test imageInput was not setup correctly.') + } + const upload = await imageInput.upload + upload.filename = '/path/to/file-location/foo-bar-avatar.jpg' + imageInput.upload = Promise.resolve(upload) + await expect( + mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }), + ).resolves.toMatchObject({ + url: expect.stringMatching( + new RegExp(`^http://s3-public-gateway.com/bucket/${uuid}-foo-bar-avatar.jpg`), + ), + }) + }) + }) + + it('connects resource with image via given image type', async () => { + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + const result = await neode.cypher( + `MATCH(p:Post {id: "p99"})-[:HERO_IMAGE]->(i:Image) RETURN i,p`, + {}, + ) + post = neode.hydrateFirst<{ id: string }>(result, 'p', neode.model('Post')).properties() + const image = neode.hydrateFirst(result, 'i', neode.model('Image')) + expect(post).toBeTruthy() + expect(image).toBeTruthy() + }) + + it('whitelists relationship types', async () => { + await expect( + mergeImage(post, 'WHATEVER' as 'HERO_IMAGE', imageInput, { + uploadCallback, + deleteCallback, + }), + ).rejects.toEqual(new Error('Unknown relationship type WHATEVER')) + }) + + it('sets metadata', async () => { + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + const image = await neode.first('Image', {}, undefined) + await expect(image.toJson()).resolves.toMatchObject({ + alt: 'A description of the new image', + createdAt: expect.any(String), + url: expect.any(String), + }) + }) + + describe('given a transaction parameter', () => { + it('executes cypher statements within the transaction', async () => { + const session = driver.session() + try { + await session.writeTransaction(async (transaction) => { + const image = await mergeImage(post, 'HERO_IMAGE', imageInput, { + uploadCallback, + deleteCallback, + transaction, + }) + return transaction.run( + ` + MATCH(image:Image {url: $image.url}) + SET image.alt = 'This alt text gets overwritten' + RETURN image {.*} + `, + { image }, + ) + }) + } finally { + await session.close() + } + const image = await neode.first( + 'Image', + { alt: 'This alt text gets overwritten' }, + undefined, + ) + await expect(image.toJson()).resolves.toMatchObject({ + alt: 'This alt text gets overwritten', + }) + }) + + it('rolls back the transaction in case of errors', async () => { + const session = driver.session() + try { + await session.writeTransaction(async (transaction) => { + const image = await mergeImage(post, 'HERO_IMAGE', imageInput, { + uploadCallback, + deleteCallback, + transaction, + }) + return transaction.run('Ooops invalid cypher!', { image }) + }) + // eslint-disable-next-line no-catch-all/no-catch-all + } catch (err) { + // nothing has been created + await expect(neode.all('Image')).resolves.toHaveLength(0) + // all good + } finally { + await session.close() + } + }) + }) + + describe('if resource has an image already', () => { + beforeEach(async () => { + const [post, image] = await Promise.all([ + neode.find('Post', 'p99'), + Factory.build('image'), + ]) + await post.relateTo(image, 'image') + }) + + it('calls deleteCallback', async () => { + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + expect(deleteCallback).toHaveBeenCalled() + }) + + it('calls uploadCallback', async () => { + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + expect(uploadCallback).toHaveBeenCalled() + }) + + it('updates metadata of existing image node', async () => { + await expect(neode.all('Image')).resolves.toHaveLength(1) + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + await expect(neode.all('Image')).resolves.toHaveLength(1) + const image = await neode.first('Image', {}, undefined) + await expect(image.toJson()).resolves.toMatchObject({ + alt: 'A description of the new image', + createdAt: expect.any(String), + url: expect.any(String), + // TODO + // width: + // height: + }) + }) + }) + }) + }) + + describe('without image.upload', () => { + it('throws UserInputError', async () => { + const p = await Factory.build('post', { id: 'p99' }, { image: null }) + post = await p.toJson() + await expect(mergeImage(post, 'HERO_IMAGE', imageInput)).rejects.toEqual( + new UserInputError('Cannot find image for given resource'), + ) + }) + + describe('if resource has an image already', () => { + beforeEach(async () => { + const p = await Factory.build( + 'post', + { + id: 'p99', + }, + { + author: Factory.build( + 'user', + {}, + { + avatar: null, + }, + ), + image: Factory.build('image', { + alt: 'This is the previous, not updated image', + url: 'http://localhost/some/original/url', + }), + }, + ) + post = await p.toJson() + }) + + it('does not call deleteCallback', async () => { + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + expect(deleteCallback).not.toHaveBeenCalled() + }) + + it('does not call uploadCallback', async () => { + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + expect(uploadCallback).not.toHaveBeenCalled() + }) + + it('updates metadata', async () => { + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) + const images = await neode.all('Image') + expect(images).toHaveLength(1) + await expect(images.first().toJson()).resolves.toMatchObject({ + createdAt: expect.any(String), + url: expect.any(String), + alt: 'A description of the new image', + }) + }) + }) + }) +}) diff --git a/backend/src/graphql/resolvers/images/imagesS3.ts b/backend/src/graphql/resolvers/images/imagesS3.ts new file mode 100644 index 000000000..66c4a0a69 --- /dev/null +++ b/backend/src/graphql/resolvers/images/imagesS3.ts @@ -0,0 +1,153 @@ +import path from 'node:path' + +import { S3Client, DeleteObjectCommand, ObjectCannedACL } from '@aws-sdk/client-s3' +import { Upload } from '@aws-sdk/lib-storage' +import { UserInputError } from 'apollo-server' +import slug from 'slug' +import { v4 as uuid } from 'uuid' + +import { S3Configured } from '@config/index' + +import { sanitizeRelationshipType } from './sanitizeRelationshipTypes' +import { wrapTransaction } from './wrapTransaction' + +import type { Image, Images, FileDeleteCallback, FileUploadCallback } from './images' +import type { FileUpload } from 'graphql-upload' + +export const images = (config: S3Configured) => { + // const widths = [34, 160, 320, 640, 1024] + const { AWS_BUCKET: Bucket, S3_PUBLIC_GATEWAY } = config + + const { AWS_ENDPOINT, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY } = config + const s3 = new S3Client({ + credentials: { + accessKeyId: AWS_ACCESS_KEY_ID, + secretAccessKey: AWS_SECRET_ACCESS_KEY, + }, + endpoint: AWS_ENDPOINT, + forcePathStyle: true, + }) + + const deleteImage: Images['deleteImage'] = async (resource, relationshipType, opts = {}) => { + sanitizeRelationshipType(relationshipType) + const { transaction, deleteCallback = s3Delete } = opts + if (!transaction) return wrapTransaction(deleteImage, [resource, relationshipType], opts) + const txResult = await transaction.run( + ` + MATCH (resource {id: $resource.id})-[rel:${relationshipType}]->(image:Image) + WITH image, image {.*} as imageProps + DETACH DELETE image + RETURN imageProps + `, + { resource }, + ) + const [image] = txResult.records.map((record) => record.get('imageProps') as Image) + // This behaviour differs from `mergeImage`. If you call `mergeImage` + // with metadata for an image that does not exist, it's an indicator + // of an error (so throw an error). If we bulk delete an image, it + // could very well be that there is no image for the resource. + if (image) { + await deleteCallback(image.url) + } + return image + } + + const mergeImage: Images['mergeImage'] = async ( + resource, + relationshipType, + imageInput, + opts = {}, + ) => { + if (typeof imageInput === 'undefined') return + if (imageInput === null) return deleteImage(resource, relationshipType, opts) + sanitizeRelationshipType(relationshipType) + const { transaction, uploadCallback, deleteCallback = s3Delete } = opts + if (!transaction) + return wrapTransaction(mergeImage, [resource, relationshipType, imageInput], opts) + + let txResult = await transaction.run( + ` + MATCH (resource {id: $resource.id})-[:${relationshipType}]->(image:Image) + RETURN image {.*} + `, + { resource }, + ) + const [existingImage] = txResult.records.map((record) => record.get('image') as Image) + const { upload } = imageInput + if (!(existingImage || upload)) throw new UserInputError('Cannot find image for given resource') + if (existingImage && upload) { + await deleteCallback(existingImage.url) + } + const url = await uploadImageFile(upload, uploadCallback) + const { alt, sensitive, aspectRatio, type } = imageInput + const image = { alt, sensitive, aspectRatio, url, type } + txResult = await transaction.run( + ` + MATCH (resource {id: $resource.id}) + MERGE (resource)-[:${relationshipType}]->(image:Image) + ON CREATE SET image.createdAt = toString(datetime()) + ON MATCH SET image.updatedAt = toString(datetime()) + SET image += $image + RETURN image {.*} + `, + { resource, image }, + ) + const [mergedImage] = txResult.records.map((record) => record.get('image') as Image) + return mergedImage + } + + const uploadImageFile = async ( + uploadPromise: Promise | undefined, + uploadCallback: FileUploadCallback | undefined = s3Upload, + ) => { + if (!uploadPromise) return undefined + const upload = await uploadPromise + const { name, ext } = path.parse(upload.filename) + const uniqueFilename = `${uuid()}-${slug(name)}${ext}` + const Location = await uploadCallback({ ...upload, uniqueFilename }) + if (!S3_PUBLIC_GATEWAY) { + return Location + } + const publicLocation = new URL(S3_PUBLIC_GATEWAY) + publicLocation.pathname = new URL(Location).pathname + return publicLocation.href + } + + const s3Upload: FileUploadCallback = async ({ createReadStream, uniqueFilename, mimetype }) => { + const s3Location = `original/${uniqueFilename}` + const params = { + Bucket, + Key: s3Location, + ACL: ObjectCannedACL.public_read, + ContentType: mimetype, + Body: createReadStream(), + } + const command = new Upload({ client: s3, params }) + const data = await command.done() + const { Location } = data + if (!Location) { + throw new Error('File upload did not return `Location`') + } + return Location + } + + const s3Delete: FileDeleteCallback = async (url) => { + let { pathname } = new URL(url, 'http://example.org') // dummy domain to avoid invalid URL error + pathname = pathname.substring(1) // remove first character '/' + const prefix = `${Bucket}/` + if (pathname.startsWith(prefix)) { + pathname = pathname.slice(prefix.length) + } + const params = { + Bucket, + Key: pathname, + } + await s3.send(new DeleteObjectCommand(params)) + } + + const images: Images = { + deleteImage, + mergeImage, + } + return images +} diff --git a/backend/src/graphql/resolvers/images/sanitizeRelationshipTypes.ts b/backend/src/graphql/resolvers/images/sanitizeRelationshipTypes.ts new file mode 100644 index 000000000..a6b984a13 --- /dev/null +++ b/backend/src/graphql/resolvers/images/sanitizeRelationshipTypes.ts @@ -0,0 +1,9 @@ +export function sanitizeRelationshipType( + relationshipType: string, +): asserts relationshipType is 'HERO_IMAGE' | 'AVATAR_IMAGE' { + // Cypher query language does not allow to parameterize relationship types + // See: https://github.com/neo4j/neo4j/issues/340 + if (!['HERO_IMAGE', 'AVATAR_IMAGE'].includes(relationshipType)) { + throw new Error(`Unknown relationship type ${relationshipType}`) + } +} diff --git a/backend/src/graphql/resolvers/images/wrapTransaction.ts b/backend/src/graphql/resolvers/images/wrapTransaction.ts new file mode 100644 index 000000000..bcc17877d --- /dev/null +++ b/backend/src/graphql/resolvers/images/wrapTransaction.ts @@ -0,0 +1,23 @@ +import { getDriver } from '@db/neo4j' + +import type { DeleteImageOpts, MergeImageOpts } from './images' + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type AsyncFunc = (...args: any[]) => Promise +export const wrapTransaction = async ( + wrappedCallback: F, + args: unknown[], + opts: DeleteImageOpts | MergeImageOpts, +) => { + const session = getDriver().session() + try { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const result = await session.writeTransaction((transaction) => { + return wrappedCallback(...args, { ...opts, transaction }) + }) + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return result + } finally { + await session.close() + } +} diff --git a/backend/src/graphql/resolvers/posts.ts b/backend/src/graphql/resolvers/posts.ts index 5190cabf8..34d190ba5 100644 --- a/backend/src/graphql/resolvers/posts.ts +++ b/backend/src/graphql/resolvers/posts.ts @@ -17,7 +17,7 @@ import { filterForMutedUsers } from './helpers/filterForMutedUsers' import { filterInvisiblePosts } from './helpers/filterInvisiblePosts' import { filterPostsOfMyGroups } from './helpers/filterPostsOfMyGroups' import Resolver from './helpers/Resolver' -import { mergeImage, deleteImage } from './images/images' +import { images } from './images/images' import { createOrUpdateLocations } from './users/location' const maintainPinnedPosts = (params) => { @@ -176,7 +176,7 @@ export default { ) const [post] = createPostTransactionResponse.records.map((record) => record.get('post')) if (imageInput) { - await mergeImage(post, 'HERO_IMAGE', imageInput, { transaction }) + await images.mergeImage(post, 'HERO_IMAGE', imageInput, { transaction }) } return post }) @@ -247,7 +247,7 @@ export default { updatePostVariables, ) const [post] = updatePostTransactionResponse.records.map((record) => record.get('post')) - await mergeImage(post, 'HERO_IMAGE', imageInput, { transaction }) + await images.mergeImage(post, 'HERO_IMAGE', imageInput, { transaction }) return post }) const post = await writeTxResultPromise @@ -277,7 +277,7 @@ export default { { postId: args.id }, ) const [post] = deletePostTransactionResponse.records.map((record) => record.get('post')) - await deleteImage(post, 'HERO_IMAGE', { transaction }) + await images.deleteImage(post, 'HERO_IMAGE', { transaction }) return post }) try { diff --git a/backend/src/graphql/resolvers/users.ts b/backend/src/graphql/resolvers/users.ts index a2f4b9dcd..9418ef3e6 100644 --- a/backend/src/graphql/resolvers/users.ts +++ b/backend/src/graphql/resolvers/users.ts @@ -15,7 +15,7 @@ import { Context } from '@src/server' import { defaultTrophyBadge, defaultVerificationBadge } from './badges' import normalizeEmail from './helpers/normalizeEmail' import Resolver from './helpers/Resolver' -import { mergeImage, deleteImage } from './images/images' +import { images } from './images/images' import { createOrUpdateLocations } from './users/location' const neode = getNeode() @@ -210,7 +210,7 @@ export default { ) const [user] = updateUserTransactionResponse.records.map((record) => record.get('user')) if (avatarInput) { - await mergeImage(user, 'AVATAR_IMAGE', avatarInput, { transaction }) + await images.mergeImage(user, 'AVATAR_IMAGE', avatarInput, { transaction }) } return user }) @@ -253,7 +253,7 @@ export default { return Promise.all( txResult.records .map((record) => record.get('resource')) - .map((resource) => deleteImage(resource, 'HERO_IMAGE', { transaction })), + .map((resource) => images.deleteImage(resource, 'HERO_IMAGE', { transaction })), ) }), ) @@ -281,7 +281,7 @@ export default { { userId }, ) const [user] = deleteUserTransactionResponse.records.map((record) => record.get('user')) - await deleteImage(user, 'AVATAR_IMAGE', { transaction }) + await images.deleteImage(user, 'AVATAR_IMAGE', { transaction }) return user }) try { diff --git a/deployment/helm/charts/ocelot-social/templates/backend/stateful-set.yaml b/deployment/helm/charts/ocelot-social/templates/backend/stateful-set.yaml index 618a99f7f..604b79826 100644 --- a/deployment/helm/charts/ocelot-social/templates/backend/stateful-set.yaml +++ b/deployment/helm/charts/ocelot-social/templates/backend/stateful-set.yaml @@ -25,6 +25,9 @@ spec: name: {{ .Release.Name }}-backend-env - secretRef: name: {{ .Release.Name }}-backend-secret-env + volumeMounts: + - mountPath: /app/public/uploads + name: uploads containers: - name: {{ .Release.Name }}-backend image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag | default (include "defaultTag" .) }}" diff --git a/deployment/helm/helmfile/secrets/ocelot.yaml b/deployment/helm/helmfile/secrets/ocelot.yaml index 3965bc09e..41eff134c 100644 --- a/deployment/helm/helmfile/secrets/ocelot.yaml +++ b/deployment/helm/helmfile/secrets/ocelot.yaml @@ -23,15 +23,16 @@ secrets: NEO4J_USERNAME: null NEO4J_PASSWORD: null REDIS_PASSWORD: null + AWS_ACCESS_KEY_ID: ENC[AES256_GCM,data:iiN5ueqyo60VHb9e2bnhc19iGTg=,iv:zawYpKrFafgsu1+YRet1hzZf1G3a6BIlZgsh7xNADaE=,tag:rTsmm8cqei34b6cT6vn08w==,type:str] + AWS_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:Zl4LRXdDh/6Q8F9RVp+3L7NXGZ0F2cgFMKPhl/TVeuD5Bhy68W5ekg==,iv:AmPoinGISrSOZdoBKdeFFXfr2hwOK4nWMnniz8K5qgU=,tag:K8Q7M7e+6G9T0Oh3Sp4OzA==,type:str] + AWS_ENDPOINT: ENC[AES256_GCM,data:/waEqUgcOmldZ+peFTNVsDQf2KrpWY8ZZMt1nT5117SkbY4=,iv:n+Kvidjb/TM4bQYKqTaFxt8GkHo02PuxEGpzgOcywr4=,tag:lrGPgCWWy3GMIcTv75IYTg==,type:str] + AWS_REGION: ENC[AES256_GCM,data:kBPpHZ8zw4PMpg==,iv:R+QZe303do37Hd/97NpS1pt9VaBE/gqZDY2/qlIvvps=,tag:0WduW8wfJXtBqlh4qfRGNA==,type:str] + AWS_BUCKET: ENC[AES256_GCM,data:0fAspN/PoRVPlSbz+qDBRUOieeC4,iv:JGJ/LyLpMymN0tpZmW6DjPT3xqXzK/KhYQsy9sgPd60=,tag:Y6PBs0916JkHRHSe7hqSMA==,type:str] neo4j: env: NEO4J_USERNAME: "" NEO4J_PASSWORD: "" sops: - kms: [] - gcp_kms: [] - azure_kv: [] - hc_vault: [] age: - recipient: age1llp6k66265q3rzqemxpnq0x3562u20989vcjf65fl9s3hjhgcscq6mhnjw enc: | @@ -69,8 +70,7 @@ sops: aGNFeXZZRmlJM041OHdTM0pmM3BBdGMKGvFgYY1jhKwciAOZKyw0hlFVNbOk7CM7 041g17JXNV1Wk6WgMZ4w8p54RKQVaWCT4wxChy6wNNdQ3IeKgqEU2w== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-10-29T14:26:49Z" - mac: ENC[AES256_GCM,data:YXX7MEAK0wmuxLTmdr7q5uVd6DG6FhGUeE+EzbhWe/OovH6n+CjKZGklnEX+5ztDO0IgZh/T9Hx1CgFYuVbcOkvDoFBDwNpRA/QOQrM0p/+tRlMNCypC/Wh2xL0DhA4A/Qum2oyE/BDkt1Yy8N5wZDZn575+ZAjXEgAzlhpT5qk=,iv:ire3gkHTY6+0lgbV1Es6Lf8bcKTg4WKnq46M+b/VRcU=,tag:MkZULKcwROvIw/C0YtcUbA==,type:str] - pgp: [] + lastmodified: "2025-05-29T06:57:01Z" + mac: ENC[AES256_GCM,data:0eWUqVsJrolrVYFsG4aigAYSjBW35Z+64y/ZE8Az15GmI0E4p/1kZPIY6hv9SUiCD1R2Ro0nm1QsV9A/hvNeWla0EdARNnARxIphxMJWKRGwcVkFVk28Jh4g4jE/rTggWx/wLbR6bza1RLHA1wRMb1PYuLfZybsb/whN4+gTMfg=,iv:irQkLpj1+3egSsiV9HqZP4tgG1fotCOizubL42gRjSQ=,tag:DC0xzG/VqcL4ib6ijxQZnA==,type:str] unencrypted_suffix: _unencrypted - version: 3.9.0 + version: 3.10.2 diff --git a/docker-compose.override.yml b/docker-compose.override.yml index ae77abd5e..a9e957dca 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -25,6 +25,9 @@ services: backend: image: ghcr.io/ocelot-social-community/ocelot-social/backend:local-development + depends_on: + - minio + - minio-mc build: target: development environment: @@ -32,6 +35,12 @@ services: - DEBUG=true - SMTP_PORT=1025 - SMTP_HOST=mailserver + - AWS_ACCESS_KEY_ID=minio + - AWS_SECRET_ACCESS_KEY=12341234 + - AWS_ENDPOINT=http:/minio:9000 + - AWS_REGION=local + - AWS_BUCKET=ocelot + - S3_PUBLIC_GATEWAY=http:/localhost:9000 volumes: - ./backend:/app @@ -46,3 +55,33 @@ services: ports: - 1080:1080 - 1025:1025 + + minio: + image: quay.io/minio/minio + ports: + - 9000:9000 + - 9001:9001 + volumes: + - minio_data:/data + environment: + - MINIO_ROOT_USER=minio + - MINIO_ROOT_PASSWORD=12341234 + command: server /data --console-address ":9001" + + minio-mc: + image: quay.io/minio/mc + depends_on: + - minio + restart: on-failure + volumes: + - ./minio/readonly-policy.json:/tmp/readonly-policy.json + entrypoint: > + /bin/sh -c " + sleep 5; + /usr/bin/mc alias set dockerminio http://minio:9000 minio 12341234; + /usr/bin/mc mb --ignore-existing dockerminio/ocelot; + /usr/bin/mc anonymous set-json /tmp/readonly-policy.json dockerminio/ocelot; + " + +volumes: + minio_data: diff --git a/docker-compose.test.yml b/docker-compose.test.yml index cc7490106..edd2f30d2 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -13,10 +13,19 @@ services: backend: # name the image so that it cannot be found in a DockerHub repository, otherwise it will not be built locally from the 'dockerfile' but pulled from there image: ghcr.io/ocelot-social-community/ocelot-social/backend:test + depends_on: + - minio + - minio-mc + - mailserver build: target: test environment: - NODE_ENV="test" + - AWS_ACCESS_KEY_ID=minio + - AWS_SECRET_ACCESS_KEY=12341234 + - AWS_ENDPOINT=http:/minio:9000 + - AWS_REGION=local + - AWS_BUCKET=ocelot volumes: - ./coverage:/app/coverage @@ -41,3 +50,33 @@ services: ports: - 1080:1080 - 1025:1025 + + minio: + image: quay.io/minio/minio + ports: + - 9000:9000 + - 9001:9001 + volumes: + - minio_data:/data + environment: + - MINIO_ROOT_USER=minio + - MINIO_ROOT_PASSWORD=12341234 + command: server /data --console-address ":9001" + + minio-mc: + image: quay.io/minio/mc + depends_on: + - minio + restart: on-failure + volumes: + - ./minio/readonly-policy.json:/tmp/readonly-policy.json + entrypoint: > + /bin/sh -c " + sleep 5; + /usr/bin/mc alias set dockerminio http://minio:9000 minio 12341234; + /usr/bin/mc mb --ignore-existing dockerminio/ocelot; + /usr/bin/mc anonymous set-json /tmp/readonly-policy.json dockerminio/ocelot; + " + +volumes: + minio_data: diff --git a/docker-compose.yml b/docker-compose.yml index 8397c4e47..4a1e9e951 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,8 +53,6 @@ services: - neo4j ports: - 4000:4000 - volumes: - - backend_uploads:/app/public/uploads environment: # Envs used in Dockerfile # - DOCKER_WORKDIR="/app" @@ -104,5 +102,4 @@ services: # command: ["tail", "-f", "/dev/null"] volumes: - backend_uploads: neo4j_data: diff --git a/minio/readonly-policy.json b/minio/readonly-policy.json new file mode 100644 index 000000000..6cdbdd667 --- /dev/null +++ b/minio/readonly-policy.json @@ -0,0 +1,19 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": [ + "*" + ] + }, + "Action": [ + "s3:GetObject" + ], + "Resource": [ + "arn:aws:s3:::ocelot/*" + ] + } + ] +}