diff --git a/backend/.env.template b/backend/.env.template index 07e07f887..8cbeb83ce 100644 --- a/backend/.env.template +++ b/backend/.env.template @@ -45,7 +45,8 @@ AWS_SECRET_ACCESS_KEY=12341234 AWS_ENDPOINT=http://localhost:9000 AWS_REGION=local AWS_BUCKET=ocelot -S3_PUBLIC_GATEWAY=http://localhost:8000 +IMAGOR_PUBLIC_URL=http://localhost:8000 +IMAGOR_SECRET=mysecret CATEGORIES_ACTIVE=false MAX_PINNED_POSTS=1 diff --git a/backend/.env.test_e2e b/backend/.env.test_e2e index 833f45b8b..689e6786a 100644 --- a/backend/.env.test_e2e +++ b/backend/.env.test_e2e @@ -37,7 +37,8 @@ AWS_SECRET_ACCESS_KEY=12341234 AWS_ENDPOINT=http://localhost:9000 AWS_REGION=local AWS_BUCKET=ocelot -S3_PUBLIC_GATEWAY=http://localhost:8000 +IMAGOR_PUBLIC_URL=http://localhost:8000 +IMAGOR_SECRET=mysecret CATEGORIES_ACTIVE=false MAX_PINNED_POSTS=1 diff --git a/backend/src/config/index.ts b/backend/src/config/index.ts index c625456f3..e04373bcf 100644 --- a/backend/src/config/index.ts +++ b/backend/src/config/index.ts @@ -30,6 +30,7 @@ const environment = { : [], SEND_MAIL: env.NODE_ENV !== 'test', LOG_LEVEL: 'DEBUG', + PROXY_S3: env.PROXY_S3, } const server = { @@ -98,13 +99,14 @@ const required = { AWS_REGION: env.AWS_REGION, AWS_BUCKET: env.AWS_BUCKET, + IMAGOR_PUBLIC_URL: env.IMAGOR_PUBLIC_URL, + IMAGOR_SECRET: env.IMAGOR_SECRET, + MAPBOX_TOKEN: env.MAPBOX_TOKEN, JWT_SECRET: env.JWT_SECRET, PRIVATE_KEY_PASSPHRASE: env.PRIVATE_KEY_PASSPHRASE, } -const S3_PUBLIC_GATEWAY = env.S3_PUBLIC_GATEWAY - // https://stackoverflow.com/a/53050575 type NoUndefinedField = { [P in keyof T]-?: NoUndefinedField> } @@ -151,7 +153,6 @@ const CONFIG = { ...redis, ...options, ...language, - S3_PUBLIC_GATEWAY, } export type Config = typeof CONFIG @@ -162,7 +163,8 @@ export type S3Config = Pick< | 'AWS_ENDPOINT' | 'AWS_REGION' | 'AWS_BUCKET' - | 'S3_PUBLIC_GATEWAY' + | 'IMAGOR_SECRET' + | 'IMAGOR_PUBLIC_URL' > export default CONFIG diff --git a/backend/src/graphql/resolvers/attachments/attachments.spec.ts b/backend/src/graphql/resolvers/attachments/attachments.spec.ts index f5ac5ddff..2b4a06a83 100644 --- a/backend/src/graphql/resolvers/attachments/attachments.spec.ts +++ b/backend/src/graphql/resolvers/attachments/attachments.spec.ts @@ -43,7 +43,8 @@ const config: S3Config = { AWS_BUCKET: 'AWS_BUCKET', AWS_ENDPOINT: 'AWS_ENDPOINT', AWS_REGION: 'AWS_REGION', - S3_PUBLIC_GATEWAY: undefined, + IMAGOR_SECRET: 'IMAGOR_SECRET', + IMAGOR_PUBLIC_URL: 'IMAGOR_PUBLIC_URL', } let authenticatedUser @@ -233,25 +234,6 @@ describe('add Attachment', () => { await expect(database.neode.all('File')).resolves.toHaveLength(1) }) - describe('given a `S3_PUBLIC_GATEWAY` configuration', () => { - const { add: addAttachment } = attachments({ - ...config, - S3_PUBLIC_GATEWAY: 'http://s3-public-gateway.com', - }) - - it('changes the domain of the URL to a server that could e.g. apply image transformations', async () => { - if (!fileInput.upload) { - throw new Error('Test imageInput was not setup correctly.') - } - const upload = await fileInput.upload - upload.filename = '/path/to/file-location/foo-bar-avatar.jpg' - fileInput.upload = Promise.resolve(upload) - await expect(addAttachment(post, 'ATTACHMENT', fileInput)).resolves.toMatchObject({ - url: 'http://s3-public-gateway.com/bucket/', - }) - }) - }) - it('connects resource with image via given image type', async () => { await addAttachment(post, 'ATTACHMENT', fileInput) const result = await database.neode.cypher( diff --git a/backend/src/graphql/resolvers/attachments/attachments.ts b/backend/src/graphql/resolvers/attachments/attachments.ts index 348f7a1a0..3b97697f7 100644 --- a/backend/src/graphql/resolvers/attachments/attachments.ts +++ b/backend/src/graphql/resolvers/attachments/attachments.ts @@ -5,7 +5,7 @@ import slug from 'slugify' import { v4 as uuid } from 'uuid' import type { S3Config } from '@config/index' -import { wrapTransaction } from '@graphql/resolvers/images/wrapTransaction' +import { getDriver } from '@db/neo4j' import { s3Service } from '@src/uploads/s3Service' import type { FileUpload } from 'graphql-upload' @@ -41,8 +41,7 @@ export interface Attachments { resource: { id: string }, relationshipType: 'ATTACHMENT', opts?: DeleteAttachmentsOpts, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - ) => Promise + ) => Promise add: ( resource: { id: string }, @@ -50,8 +49,44 @@ export interface Attachments { file: FileInput, fileAttributes?: object, opts?: AddAttachmentOpts, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - ) => Promise + ) => Promise +} + +const wrapTransactionDeleteAttachment = async ( + wrappedCallback: Attachments['del'], + args: [resource: { id: string }, relationshipType: 'ATTACHMENT'], + opts: DeleteAttachmentsOpts, +): ReturnType => { + const session = getDriver().session() + try { + const result = await session.writeTransaction((transaction) => { + return wrappedCallback(...args, { ...opts, transaction }) + }) + return result + } finally { + await session.close() + } +} + +const wrapTransactionMergeAttachment = async ( + wrappedCallback: Attachments['add'], + args: [ + resource: { id: string }, + relationshipType: 'ATTACHMENT', + file: FileInput, + fileAttributes?: object, + ], + opts: AddAttachmentOpts, +): ReturnType => { + const session = getDriver().session() + try { + const result = await session.writeTransaction((transaction) => { + return wrappedCallback(...args, { ...opts, transaction }) + }) + return result + } finally { + await session.close() + } } export const attachments = (config: S3Config) => { @@ -59,7 +94,8 @@ export const attachments = (config: S3Config) => { const del: Attachments['del'] = async (resource, relationshipType, opts = {}) => { const { transaction } = opts - if (!transaction) return wrapTransaction(del, [resource, relationshipType], opts) + if (!transaction) + return wrapTransactionDeleteAttachment(del, [resource, relationshipType], opts) const txResult = await transaction.run( ` MATCH (resource {id: $resource.id})-[rel:${relationshipType}]->(file:File) @@ -85,7 +121,11 @@ export const attachments = (config: S3Config) => { ) => { const { transaction } = opts if (!transaction) - return wrapTransaction(add, [resource, relationshipType, fileInput, fileAttributes], opts) + return wrapTransactionMergeAttachment( + add, + [resource, relationshipType, fileInput, fileAttributes], + opts, + ) const { upload } = fileInput if (!upload) throw new UserInputError('Cannot find attachment for given resource') @@ -121,9 +161,9 @@ export const attachments = (config: S3Config) => { return uploadedFile } - const attachments: Attachments = { + const attachments = { del, add, - } + } satisfies Attachments return attachments } diff --git a/backend/src/graphql/resolvers/images.spec.ts b/backend/src/graphql/resolvers/images.spec.ts new file mode 100644 index 000000000..4a6f8903d --- /dev/null +++ b/backend/src/graphql/resolvers/images.spec.ts @@ -0,0 +1,52 @@ +import { TEST_CONFIG } from '@root/test/helpers' + +import ImageResolver from './images' + +describe('Image', () => { + const { Image } = ImageResolver + const Location = + 'https://fsn1.your-objectstorage.com/ocelot-social-staging/original/f965ea15-1f6b-43aa-a535-927410e2585e-dsc02586.jpg' + const defaultConfig = { + ...TEST_CONFIG, + AWS_ENDPOINT: 'https://fsn1.your-objectstorage.com', + IMAGOR_PUBLIC_URL: 'https://imagor-public-url.com', + IMAGOR_SECRET: 'IMAGOR_SECRET', + } + + describe('.transform', () => { + describe('no transformations', () => { + const config = { ...defaultConfig } + const args = {} + + it('just points the original url to imagor and adds a signature', () => { + const expectedUrl = + 'https://imagor-public-url.com/f_qz7PlAWIQx-IrMOZfikzDFM6I=/ocelot-social-staging/original/f965ea15-1f6b-43aa-a535-927410e2585e-dsc02586.jpg' + expect(Image.transform({ url: Location }, args, { config })).toEqual(expectedUrl) + }) + + describe('if `IMAGOR_PUBLIC_URL` has a path segment', () => { + const config = { + ...defaultConfig, + IMAGOR_PUBLIC_URL: 'https://imagor-public-url.com/path-segment', + } + + it('keeps the path segment', () => { + const expectedUrl = + 'https://imagor-public-url.com/path-segment/f_qz7PlAWIQx-IrMOZfikzDFM6I=/ocelot-social-staging/original/f965ea15-1f6b-43aa-a535-927410e2585e-dsc02586.jpg' + expect(Image.transform({ url: Location }, args, { config })).toEqual(expectedUrl) + }) + }) + }) + + describe('resize transformations', () => { + const config = { ...defaultConfig } + const args = { width: 320 } + + it('encodes `fit-in` imagor transformations in the URL', () => { + const expectedUrl = + 'https://imagor-public-url.com/1OEqC7g0YFxuvnRCX2hOukYMJEY=/fit-in/320x5000/ocelot-social-staging/original/f965ea15-1f6b-43aa-a535-927410e2585e-dsc02586.jpg' + expect(Image.transform({ url: Location }, args, { config })).toEqual(expectedUrl) + }) + }) + }) +}) diff --git a/backend/src/graphql/resolvers/images.ts b/backend/src/graphql/resolvers/images.ts index ea596a183..9e000d34f 100644 --- a/backend/src/graphql/resolvers/images.ts +++ b/backend/src/graphql/resolvers/images.ts @@ -1,9 +1,83 @@ +import crypto from 'node:crypto' +import { join as joinPath } from 'node:path/posix' + +import type { Context } from '@src/context' + import Resolver from './helpers/Resolver' +type UrlResolver = ( + parent: { url: string }, + args: { width?: number; height?: number }, + { + config: { IMAGOR_PUBLIC_URL }, + }: Pick, +) => string + +const pointUrlToImagor: (opts: { transformations: UrlResolver[] }) => UrlResolver = + ({ transformations }) => + ({ url }, _args, context) => { + const { config } = context + const { IMAGOR_PUBLIC_URL, AWS_ENDPOINT } = config + if (!IMAGOR_PUBLIC_URL) { + return url + } + const originalUrl = new URL(url, AWS_ENDPOINT) + if (originalUrl.host !== new URL(AWS_ENDPOINT).host) { + // In this case it's an external upload - maybe seeded? + // Let's not change the URL in this case + return url + } + + const transformedUrl = new URL( + chain(...transformations)({ url: originalUrl.href }, _args, context), + ) + const imagorUrl = new URL(IMAGOR_PUBLIC_URL) + imagorUrl.pathname = joinPath(imagorUrl.pathname, transformedUrl.pathname) + return imagorUrl.href + } + +const sign: UrlResolver = ({ url }, _args, { config: { IMAGOR_SECRET } }) => { + if (!IMAGOR_SECRET) { + throw new Error('IMAGOR_SECRET is not set') + } + const newUrl = new URL(url) + const path = newUrl.pathname.replace('/', '') + const hash = crypto + .createHmac('sha1', IMAGOR_SECRET) + .update(path) + .digest('base64') + .replace(/\+/g, '-') + .replace(/\//g, '_') + newUrl.pathname = hash + newUrl.pathname + return newUrl.href +} + +const FALLBACK_MAXIMUM_LENGTH = 5000 +const resize: UrlResolver = ({ url }, { height, width }) => { + if (!(height || width)) { + return url + } + const window = `/fit-in/${width ?? FALLBACK_MAXIMUM_LENGTH}x${height ?? FALLBACK_MAXIMUM_LENGTH}` + const newUrl = new URL(url) + newUrl.pathname = window + newUrl.pathname + return newUrl.href +} + +const chain: (...methods: UrlResolver[]) => UrlResolver = (...methods) => { + return (parent, args, context) => { + let { url } = parent + for (const method of methods) { + url = method({ url }, args, context) + } + return url + } +} + export default { Image: { ...Resolver('Image', { undefinedToNull: ['sensitive', 'alt', 'aspectRatio', 'type'], }), + transform: pointUrlToImagor({ transformations: [resize, sign] }), }, } diff --git a/backend/src/graphql/resolvers/images/images.ts b/backend/src/graphql/resolvers/images/images.ts index 0130f2436..7d4da562a 100644 --- a/backend/src/graphql/resolvers/images/images.ts +++ b/backend/src/graphql/resolvers/images/images.ts @@ -38,16 +38,14 @@ export interface Images { resource: { id: string }, relationshipType: 'HERO_IMAGE' | 'AVATAR_IMAGE', opts?: DeleteImageOpts, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - ) => Promise + ) => Promise mergeImage: ( resource: { id: string }, relationshipType: 'HERO_IMAGE' | 'AVATAR_IMAGE', imageInput: ImageInput | null | undefined, opts?: MergeImageOpts, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - ) => Promise + ) => Promise } export const images = (config: Context['config']) => imagesS3(config) diff --git a/backend/src/graphql/resolvers/images/imagesS3.spec.ts b/backend/src/graphql/resolvers/images/imagesS3.spec.ts index 0fe885f7c..0617c50ba 100644 --- a/backend/src/graphql/resolvers/images/imagesS3.spec.ts +++ b/backend/src/graphql/resolvers/images/imagesS3.spec.ts @@ -2,7 +2,6 @@ /* eslint-disable @typescript-eslint/no-unsafe-call */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ -/* eslint-disable promise/prefer-await-to-callbacks */ import { DeleteObjectCommand } from '@aws-sdk/client-s3' import { Upload } from '@aws-sdk/lib-storage' import { UserInputError } from 'apollo-server' @@ -47,7 +46,8 @@ const config: S3Config = { AWS_BUCKET: 'AWS_BUCKET', AWS_ENDPOINT: 'AWS_ENDPOINT', AWS_REGION: 'AWS_REGION', - S3_PUBLIC_GATEWAY: undefined, + IMAGOR_SECRET: 'IMAGOR_SECRET', + IMAGOR_PUBLIC_URL: 'IMAGOR_PUBLIC_URL', } beforeAll(async () => { @@ -151,7 +151,7 @@ describe('mergeImage', () => { beforeEach(() => { const createReadStream: FileUpload['createReadStream'] = (() => ({ pipe: () => ({ - on: (_, callback) => callback(), + on: (_: unknown, callback: () => void) => callback(), // eslint-disable-line promise/prefer-await-to-callbacks }), })) as unknown as FileUpload['createReadStream'] imageInput = { @@ -210,29 +210,6 @@ describe('mergeImage', () => { }) }) - describe('given a `S3_PUBLIC_GATEWAY` configuration', () => { - const { mergeImage } = images({ - ...config, - S3_PUBLIC_GATEWAY: 'http://s3-public-gateway.com', - }) - - it('changes the domain of the URL to a server that could e.g. apply image transformations', async () => { - if (!imageInput.upload) { - throw new Error('Test imageInput was not setup correctly.') - } - const upload = await imageInput.upload - upload.filename = '/path/to/file-location/foo-bar-avatar.jpg' - imageInput.upload = Promise.resolve(upload) - await expect(mergeImage(post, 'HERO_IMAGE', imageInput)).resolves.toMatchObject({ - url: expect.stringMatching( - new RegExp( - `^http://s3-public-gateway.com/bucket/original/${uuid}-foo-bar-avatar.jpg`, - ), - ), - }) - }) - }) - it('connects resource with image via given image type', async () => { await mergeImage(post, 'HERO_IMAGE', imageInput) const result = await neode.cypher( diff --git a/backend/src/graphql/resolvers/images/imagesS3.ts b/backend/src/graphql/resolvers/images/imagesS3.ts index 00dbc0d41..7549e50b6 100644 --- a/backend/src/graphql/resolvers/images/imagesS3.ts +++ b/backend/src/graphql/resolvers/images/imagesS3.ts @@ -8,7 +8,7 @@ import { v4 as uuid } from 'uuid' import type { S3Config } from '@config/index' import { s3Service } from '@src/uploads/s3Service' -import { wrapTransaction } from './wrapTransaction' +import { wrapTransactionDeleteImage, wrapTransactionMergeImage } from './wrapTransaction' import type { Image, Images } from './images' @@ -17,7 +17,8 @@ export const images = (config: S3Config) => { const deleteImage: Images['deleteImage'] = async (resource, relationshipType, opts = {}) => { const { transaction } = opts - if (!transaction) return wrapTransaction(deleteImage, [resource, relationshipType], opts) + if (!transaction) + return wrapTransactionDeleteImage(deleteImage, [resource, relationshipType], opts) const txResult = await transaction.run( ` MATCH (resource {id: $resource.id})-[rel:${relationshipType}]->(image:Image) @@ -48,7 +49,7 @@ export const images = (config: S3Config) => { if (imageInput === null) return deleteImage(resource, relationshipType, opts) const { transaction } = opts if (!transaction) - return wrapTransaction(mergeImage, [resource, relationshipType, imageInput], opts) + return wrapTransactionMergeImage(mergeImage, [resource, relationshipType, imageInput], opts) let txResult = await transaction.run( ` diff --git a/backend/src/graphql/resolvers/images/wrapTransaction.ts b/backend/src/graphql/resolvers/images/wrapTransaction.ts index bcc17877d..a91a3f4c0 100644 --- a/backend/src/graphql/resolvers/images/wrapTransaction.ts +++ b/backend/src/graphql/resolvers/images/wrapTransaction.ts @@ -1,21 +1,37 @@ import { getDriver } from '@db/neo4j' -import type { DeleteImageOpts, MergeImageOpts } from './images' +import type { DeleteImageOpts, MergeImageOpts, Images, ImageInput } from './images' -// eslint-disable-next-line @typescript-eslint/no-explicit-any -type AsyncFunc = (...args: any[]) => Promise -export const wrapTransaction = async ( - wrappedCallback: F, - args: unknown[], - opts: DeleteImageOpts | MergeImageOpts, -) => { +export const wrapTransactionDeleteImage = async ( + wrappedCallback: Images['deleteImage'], + args: [resource: { id: string }, relationshipType: 'HERO_IMAGE' | 'AVATAR_IMAGE'], + opts: DeleteImageOpts, +): ReturnType => { + const session = getDriver().session() + try { + const result = await session.writeTransaction((transaction) => { + return wrappedCallback(...args, { ...opts, transaction }) + }) + return result + } finally { + await session.close() + } +} + +export const wrapTransactionMergeImage = async ( + wrappedCallback: Images['mergeImage'], + args: [ + resource: { id: string }, + relationshipType: 'HERO_IMAGE' | 'AVATAR_IMAGE', + imageInput: ImageInput | null | undefined, + ], + opts: MergeImageOpts, +): ReturnType => { const session = getDriver().session() try { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment const result = await session.writeTransaction((transaction) => { return wrappedCallback(...args, { ...opts, transaction }) }) - // eslint-disable-next-line @typescript-eslint/no-unsafe-return return result } finally { await session.close() diff --git a/backend/src/graphql/resolvers/messages.ts b/backend/src/graphql/resolvers/messages.ts index ad75dd181..0898f7016 100644 --- a/backend/src/graphql/resolvers/messages.ts +++ b/backend/src/graphql/resolvers/messages.ts @@ -13,6 +13,8 @@ import { CHAT_MESSAGE_ADDED } from '@constants/subscriptions' import { attachments } from './attachments/attachments' import Resolver from './helpers/Resolver' +import type { File } from './attachments/attachments' + const setMessagesAsDistributed = async (undistributedMessagesIds, session) => { return session.writeTransaction(async (transaction) => { const setDistributedCypher = ` diff --git a/backend/src/graphql/types/type/Image.gql b/backend/src/graphql/types/type/Image.gql index f171a4b77..1cbe9c78c 100644 --- a/backend/src/graphql/types/type/Image.gql +++ b/backend/src/graphql/types/type/Image.gql @@ -1,5 +1,6 @@ type Image { url: ID!, + transform(width: Int, height: Int): String # urlW34: String, # urlW160: String, # urlW320: String, diff --git a/backend/src/index.ts b/backend/src/index.ts index 481d95183..f1b3827fe 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -2,6 +2,7 @@ import CONFIG from './config' import { loggerPlugin } from './plugins/apolloLogger' +import createProxy from './proxy' import createServer from './server' const { server, httpServer } = createServer({ @@ -14,3 +15,20 @@ httpServer.listen({ port: url.port }, () => { /* eslint-disable-next-line no-console */ console.log(`🚀 Subscriptions ready at ws://localhost:${url.port}${server.subscriptionsPath}`) }) + +if (CONFIG.PROXY_S3) { + /* + In a Docker environment, the `AWS_ENDPOINT` of the backend container would be `http://minio:9000` but this domain is not reachable from the Docker host. + Therefore, we forward the local port 9000 to "http://minio:9000." The backend can upload files to its own proxy `http://localhost:9000` and the returned file location is going to be accessible from the web frontend. + This behavior is only required in local development, not in production. Therefore, we put it behind a `CONFIG.PROXY_S3` feature flag. + */ + const target = new URL(CONFIG.PROXY_S3) + const proxy = createProxy(target) + const forwardedPort = target.port // target port and forwarded port must be the same + proxy.listen(forwardedPort, () => { + /* eslint-disable-next-line no-console */ + console.log(`Simple HTTP proxy listening on port ${forwardedPort}`) + /* eslint-disable-next-line no-console */ + console.log(`Proxying requests to ${target}`) + }) +} diff --git a/backend/src/proxy.ts b/backend/src/proxy.ts new file mode 100644 index 000000000..4c9b3b271 --- /dev/null +++ b/backend/src/proxy.ts @@ -0,0 +1,30 @@ +import http from 'node:http' + +const createProxy = (target: URL) => { + const proxy = http.createServer((req, res) => { + const options = { + hostname: target.hostname, + port: target.port, + path: req.url, + method: req.method, + headers: req.headers, + } + + const proxyReq = http.request(options, (proxyRes) => { + res.writeHead(proxyRes.statusCode ?? 500, proxyRes.headers) + proxyRes.pipe(res) // Pipe the response from the target server back to the client + }) + + req.pipe(proxyReq) // Pipe the client's request body to the target server + + proxyReq.on('error', (err) => { + /* eslint-disable-next-line no-console */ + console.error('Proxy request error:', err) + res.writeHead(500, { 'Content-Type': 'text/plain' }) + res.end('Proxy error') + }) + }) + return proxy +} + +export default createProxy diff --git a/backend/src/uploads/s3Service.spec.ts b/backend/src/uploads/s3Service.spec.ts new file mode 100644 index 000000000..4cca92389 --- /dev/null +++ b/backend/src/uploads/s3Service.spec.ts @@ -0,0 +1,80 @@ +import { Upload } from '@aws-sdk/lib-storage' + +import type { S3Config } from '@config/index' + +import { s3Service } from './s3Service' + +import type { FileUpload } from 'graphql-upload' + +jest.mock('@aws-sdk/client-s3', () => { + return { + S3Client: jest.fn().mockImplementation(() => ({ + send: jest.fn(), + })), + ObjectCannedACL: { public_read: 'public_read' }, + DeleteObjectCommand: jest.fn().mockImplementation(() => ({})), + } +}) + +jest.mock('@aws-sdk/lib-storage', () => { + return { + Upload: jest.fn(), + } +}) + +const uploadMock = Upload as unknown as jest.Mock + +const createReadStream: FileUpload['createReadStream'] = (() => ({ + pipe: () => ({ + on: (_: unknown, callback: () => void) => callback(), // eslint-disable-line promise/prefer-await-to-callbacks + }), +})) as unknown as FileUpload['createReadStream'] +const input = { + uniqueFilename: 'unique-filename.jpg', + mimetype: 'image/jpeg', + createReadStream, +} + +const config: S3Config = { + AWS_ACCESS_KEY_ID: 'AWS_ACCESS_KEY_ID', + AWS_SECRET_ACCESS_KEY: 'AWS_SECRET_ACCESS_KEY', + AWS_BUCKET: 'AWS_BUCKET', + AWS_ENDPOINT: 'AWS_ENDPOINT', + AWS_REGION: 'AWS_REGION', + IMAGOR_SECRET: 'IMAGOR_SECRET', + IMAGOR_PUBLIC_URL: 'IMAGOR_PUBLIC_URL', +} + +describe('s3Service', () => { + describe('upload', () => { + describe('if the S3 service returns a valid URL as a `Location`', () => { + beforeEach(() => { + uploadMock.mockImplementation(({ params: { Key } }: { params: { Key: string } }) => ({ + done: () => Promise.resolve({ Location: `http://your-objectstorage.com/bucket/${Key}` }), + })) + }) + + it('returns the `Location` that was returned by the s3 client library', async () => { + const service = s3Service(config, 'ocelot-social') + await expect(service.uploadFile(input)).resolves.toEqual( + 'http://your-objectstorage.com/bucket/ocelot-social/unique-filename.jpg', + ) + }) + }) + + describe('but if for some reason, the S3 service returns a `Location` wich is not a valid URL and misses the protocol part', () => { + beforeEach(() => { + uploadMock.mockImplementation(({ params: { Key } }: { params: { Key: string } }) => ({ + done: () => Promise.resolve({ Location: `your-objectstorage.com/bucket/${Key}` }), + })) + }) + + it('adds `https:` as protocol', async () => { + const service = s3Service(config, 'ocelot-social') + await expect(service.uploadFile(input)).resolves.toEqual( + 'https://your-objectstorage.com/bucket/ocelot-social/unique-filename.jpg', + ) + }) + }) + }) +}) diff --git a/backend/src/uploads/s3Service.ts b/backend/src/uploads/s3Service.ts index 816d9fe4a..76baf2cad 100644 --- a/backend/src/uploads/s3Service.ts +++ b/backend/src/uploads/s3Service.ts @@ -8,7 +8,7 @@ import { FileUploadCallback, FileDeleteCallback } from './types' export const s3Service = (config: S3Config, prefix: string) => { const { AWS_BUCKET: Bucket } = config - const { AWS_ENDPOINT, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, S3_PUBLIC_GATEWAY } = config + const { AWS_ENDPOINT, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY } = config const s3 = new S3Client({ credentials: { accessKeyId: AWS_ACCESS_KEY_ID, @@ -40,13 +40,7 @@ export const s3Service = (config: S3Config, prefix: string) => { location = `https://${location}` } - if (!S3_PUBLIC_GATEWAY) { - return location - } - - const publicLocation = new URL(S3_PUBLIC_GATEWAY) - publicLocation.pathname = new URL(location).pathname - return publicLocation.href + return location } const deleteFile: FileDeleteCallback = async (url) => { diff --git a/backend/test/helpers.ts b/backend/test/helpers.ts index c75dd47d5..132f47d05 100644 --- a/backend/test/helpers.ts +++ b/backend/test/helpers.ts @@ -16,6 +16,7 @@ export const TEST_CONFIG = { PRODUCTION_DB_CLEAN_ALLOW: false, DISABLED_MIDDLEWARES: [], SEND_MAIL: false, + PROXY_S3: 'http://minio:9000', CLIENT_URI: 'http://webapp:3000', GRAPHQL_URI: 'http://localhost:4000', @@ -43,7 +44,8 @@ export const TEST_CONFIG = { AWS_REGION: 'local', AWS_BUCKET: 'ocelot', - S3_PUBLIC_GATEWAY: undefined, + IMAGOR_SECRET: 'IMAGOR_SECRET', + IMAGOR_PUBLIC_URL: 'IMAGOR_PUBLIC_URL', EMAIL_DEFAULT_SENDER: '', SUPPORT_EMAIL: '', diff --git a/deployment/helm/charts/ocelot-social/templates/backend/stateful-set.yaml b/deployment/helm/charts/ocelot-social/templates/backend/stateful-set.yaml index 604b79826..2bf30f935 100644 --- a/deployment/helm/charts/ocelot-social/templates/backend/stateful-set.yaml +++ b/deployment/helm/charts/ocelot-social/templates/backend/stateful-set.yaml @@ -20,6 +20,9 @@ spec: imagePullPolicy: {{ quote .Values.global.image.pullPolicy }} command: ["/bin/sh", "-c", "yarn prod:migrate init && yarn prod:migrate up"] {{- include "resources" .Values.backend.resources | indent 10 }} + env: + - name: IMAGOR_PUBLIC_URL + value: "https://{{ .Values.domain }}/imagor" envFrom: - configMapRef: name: {{ .Release.Name }}-backend-env @@ -38,6 +41,8 @@ spec: value: "http://{{ .Release.Name }}-backend:4000" - name: CLIENT_URI value: "https://{{ .Values.domain }}" + - name: IMAGOR_PUBLIC_URL + value: "https://{{ .Values.domain }}/imagor" envFrom: - configMapRef: name: {{ .Release.Name }}-backend-env diff --git a/deployment/helm/charts/ocelot-social/templates/imagor/deployment.yaml b/deployment/helm/charts/ocelot-social/templates/imagor/deployment.yaml new file mode 100644 index 000000000..80bac5e8b --- /dev/null +++ b/deployment/helm/charts/ocelot-social/templates/imagor/deployment.yaml @@ -0,0 +1,28 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Release.Name }}-imagor +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-imagor + template: + metadata: + labels: + app: {{ .Release.Name }}-imagor + spec: + restartPolicy: Always + containers: + - name: {{ .Release.Name }}-imagor + image: "{{ .Values.imagor.image.repository }}:{{ .Values.imagor.image.tag | default (include "defaultTag" .) }}" + imagePullPolicy: {{ quote .Values.global.image.pullPolicy }} + {{- include "resources" .Values.imagor.resources | indent 8 }} + ports: + - containerPort: 8000 + env: + - name: S3_FORCE_PATH_STYLE + value: "1" + envFrom: + - secretRef: + name: {{ .Release.Name }}-imagor-secret-env diff --git a/deployment/helm/charts/ocelot-social/templates/imagor/secret.yaml b/deployment/helm/charts/ocelot-social/templates/imagor/secret.yaml new file mode 100644 index 000000000..7e11e933f --- /dev/null +++ b/deployment/helm/charts/ocelot-social/templates/imagor/secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-imagor-secret-env +type: Opaque +stringData: +{{ .Values.secrets.imagor.env | toYaml | indent 2 }} diff --git a/deployment/helm/charts/ocelot-social/templates/imagor/service.yaml b/deployment/helm/charts/ocelot-social/templates/imagor/service.yaml new file mode 100644 index 000000000..229a039b4 --- /dev/null +++ b/deployment/helm/charts/ocelot-social/templates/imagor/service.yaml @@ -0,0 +1,11 @@ +kind: Service +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-imagor +spec: + ports: + - name: {{ .Release.Name }}-http + port: 8000 + targetPort: 8000 + selector: + app: {{ .Release.Name }}-imagor diff --git a/deployment/helm/charts/ocelot-social/templates/ingress.yaml b/deployment/helm/charts/ocelot-social/templates/ingress.yaml index 56142f650..bf5e202df 100644 --- a/deployment/helm/charts/ocelot-social/templates/ingress.yaml +++ b/deployment/helm/charts/ocelot-social/templates/ingress.yaml @@ -62,4 +62,34 @@ spec: regex: ^https://{{ . }}(.*) replacement: https://{{ $.Values.domain }}${1} permanent: true -{{- end }} \ No newline at end of file +{{- end }} + +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: {{ .Release.Name }}-stripprefix +spec: + stripPrefix: + prefixes: + - /imagor + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ .Release.Name }}-path-prefixes + annotations: + traefik.ingress.kubernetes.io/router.middlewares: "{{ .Release.Namespace }}-{{ .Release.Name }}-stripprefix@kubernetescrd" +spec: + rules: + - host: {{ quote .Values.domain }} + http: + paths: + - path: /imagor + pathType: Prefix + backend: + service: + name: {{ .Release.Name }}-imagor + port: + number: 8000 diff --git a/deployment/helm/charts/ocelot-social/values.yaml b/deployment/helm/charts/ocelot-social/values.yaml index 2213c5007..64867edb6 100644 --- a/deployment/helm/charts/ocelot-social/values.yaml +++ b/deployment/helm/charts/ocelot-social/values.yaml @@ -25,3 +25,9 @@ webapp: maintenance: image: repository: ghcr.io/ocelot-social-community/ocelot-social/maintenance + +imagor: + image: + repository: shumc/imagor + tag: 1.5.4 + diff --git a/deployment/helm/helmfile/secrets/ocelot.yaml b/deployment/helm/helmfile/secrets/ocelot.yaml index 41eff134c..5b08931c7 100644 --- a/deployment/helm/helmfile/secrets/ocelot.yaml +++ b/deployment/helm/helmfile/secrets/ocelot.yaml @@ -23,15 +23,25 @@ secrets: NEO4J_USERNAME: null NEO4J_PASSWORD: null REDIS_PASSWORD: null - AWS_ACCESS_KEY_ID: ENC[AES256_GCM,data:iiN5ueqyo60VHb9e2bnhc19iGTg=,iv:zawYpKrFafgsu1+YRet1hzZf1G3a6BIlZgsh7xNADaE=,tag:rTsmm8cqei34b6cT6vn08w==,type:str] - AWS_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:Zl4LRXdDh/6Q8F9RVp+3L7NXGZ0F2cgFMKPhl/TVeuD5Bhy68W5ekg==,iv:AmPoinGISrSOZdoBKdeFFXfr2hwOK4nWMnniz8K5qgU=,tag:K8Q7M7e+6G9T0Oh3Sp4OzA==,type:str] - AWS_ENDPOINT: ENC[AES256_GCM,data:/waEqUgcOmldZ+peFTNVsDQf2KrpWY8ZZMt1nT5117SkbY4=,iv:n+Kvidjb/TM4bQYKqTaFxt8GkHo02PuxEGpzgOcywr4=,tag:lrGPgCWWy3GMIcTv75IYTg==,type:str] - AWS_REGION: ENC[AES256_GCM,data:kBPpHZ8zw4PMpg==,iv:R+QZe303do37Hd/97NpS1pt9VaBE/gqZDY2/qlIvvps=,tag:0WduW8wfJXtBqlh4qfRGNA==,type:str] - AWS_BUCKET: ENC[AES256_GCM,data:0fAspN/PoRVPlSbz+qDBRUOieeC4,iv:JGJ/LyLpMymN0tpZmW6DjPT3xqXzK/KhYQsy9sgPd60=,tag:Y6PBs0916JkHRHSe7hqSMA==,type:str] + IMAGOR_SECRET: null + AWS_ACCESS_KEY_ID: null + AWS_SECRET_ACCESS_KEY: null + AWS_ENDPOINT: null + AWS_REGION: null + AWS_BUCKET: null neo4j: env: NEO4J_USERNAME: "" NEO4J_PASSWORD: "" + imagor: + env: + HTTP_LOADER_BASE_URL: null + IMAGOR_SECRET: null + AWS_ACCESS_KEY_ID: null + AWS_SECRET_ACCESS_KEY: null + AWS_ENDPOINT: null + AWS_REGION: null + AWS_BUCKET: null sops: age: - recipient: age1llp6k66265q3rzqemxpnq0x3562u20989vcjf65fl9s3hjhgcscq6mhnjw @@ -70,7 +80,7 @@ sops: aGNFeXZZRmlJM041OHdTM0pmM3BBdGMKGvFgYY1jhKwciAOZKyw0hlFVNbOk7CM7 041g17JXNV1Wk6WgMZ4w8p54RKQVaWCT4wxChy6wNNdQ3IeKgqEU2w== -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-05-29T06:57:01Z" - mac: ENC[AES256_GCM,data:0eWUqVsJrolrVYFsG4aigAYSjBW35Z+64y/ZE8Az15GmI0E4p/1kZPIY6hv9SUiCD1R2Ro0nm1QsV9A/hvNeWla0EdARNnARxIphxMJWKRGwcVkFVk28Jh4g4jE/rTggWx/wLbR6bza1RLHA1wRMb1PYuLfZybsb/whN4+gTMfg=,iv:irQkLpj1+3egSsiV9HqZP4tgG1fotCOizubL42gRjSQ=,tag:DC0xzG/VqcL4ib6ijxQZnA==,type:str] + lastmodified: "2025-05-30T12:50:05Z" + mac: ENC[AES256_GCM,data:b9GHzTW9yQ2Fd+EI+bhe6D+f72ToWDwvaJfJEoIIWUC1oExU7W1uRE9tftM8iPjD9CjM/bOSH8otQYGSXcN/SM3N9DW0UnGo5yIqcz/abpLSAgXK4a5MHMFtbJ7uPlsmgEixkPo9Kc82if4qJ1lPK8LL9+W2rZC5FLTHD/a9GKU=,iv:kBUvBsxxjWlXVIzVTLvl+zGKuCeefeNWAxo7OtAoyTg=,tag:6THq7miNLRbwhqg/xt6hXw==,type:str] unencrypted_suffix: _unencrypted version: 3.10.2 diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 4e91c9f01..9b310702b 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -25,9 +25,12 @@ services: backend: image: ghcr.io/ocelot-social-community/ocelot-social/backend:local-development + ports: + - 9000:9000 depends_on: - minio - minio-mc + - imagor build: target: development environment: @@ -36,10 +39,12 @@ services: - SMTP_HOST=mailserver - AWS_ACCESS_KEY_ID=minio - AWS_SECRET_ACCESS_KEY=12341234 - - AWS_ENDPOINT=http:/minio:9000 + - AWS_ENDPOINT=http:/localhost:9000 - AWS_REGION=local - AWS_BUCKET=ocelot - - S3_PUBLIC_GATEWAY=http:/localhost:9000 + - IMAGOR_PUBLIC_URL=http://localhost:8000 + - IMAGOR_SECRET=mysecret + - PROXY_S3=http://minio:9000 volumes: - ./backend:/app @@ -58,7 +63,6 @@ services: minio: image: quay.io/minio/minio ports: - - 9000:9000 - 9001:9001 volumes: - minio_data:/data @@ -82,5 +86,22 @@ services: /usr/bin/mc anonymous set-json /tmp/readonly-policy.json dockerminio/ocelot; " + imagor: + image: shumc/imagor:latest + ports: + - 8000:8000 + environment: + PORT: 8000 + IMAGOR_SECRET: mysecret # secret key for URL signature + # IMAGOR_UNSAFE: 1 # unsafe URL for testing + AWS_ACCESS_KEY_ID: minio + AWS_SECRET_ACCESS_KEY: 12341234 + AWS_ENDPOINT: http:/minio:9000 + S3_FORCE_PATH_STYLE: 1 + S3_LOADER_BUCKET: ocelot # enable S3 loader by specifying bucket + S3_STORAGE_BUCKET: ocelot # enable S3 storage by specifying bucket + S3_RESULT_STORAGE_BUCKET: ocelot # enable S3 result storage by specifying bucket + HTTP_LOADER_BASE_URL: http://minio:9000 + volumes: minio_data: diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 161962cdb..1bea14b22 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -27,6 +27,9 @@ services: - AWS_REGION=local - AWS_BUCKET=ocelot - DEBUG= + - IMAGOR_PUBLIC_URL=http://localhost:8000 + - IMAGOR_SECRET=mysecret + - PROXY_S3=http://minio:9000 volumes: - ./coverage:/app/coverage diff --git a/webapp/components/BadgeSelection.vue b/webapp/components/BadgeSelection.vue index a6554d779..a24c06fac 100644 --- a/webapp/components/BadgeSelection.vue +++ b/webapp/components/BadgeSelection.vue @@ -7,7 +7,7 @@ @click="handleBadgeClick(badge, index)" >
- +
{{ badge.description }}
@@ -17,6 +17,7 @@ diff --git a/webapp/components/UserTeaser/UserTeaser.spec.js b/webapp/components/UserTeaser/UserTeaser.spec.js index 2926e31da..3e6e6d127 100644 --- a/webapp/components/UserTeaser/UserTeaser.spec.js +++ b/webapp/components/UserTeaser/UserTeaser.spec.js @@ -21,7 +21,12 @@ const userTilda = { name: 'Tilda Swinton', slug: 'tilda-swinton', id: 'user1', - avatar: '/avatars/tilda-swinton', + avatar: { + url: '/avatars/tilda-swinton', + w320: '/avatars/tilda-swinton-w320', + w640: '/avatars/tilda-swinton-w640', + w1024: '/avatars/tilda-swinton-w1024', + }, badgeVerification: { id: 'bv1', icon: '/icons/verified', diff --git a/webapp/components/UserTeaser/__snapshots__/UserTeaser.spec.js.snap b/webapp/components/UserTeaser/__snapshots__/UserTeaser.spec.js.snap index bd4a09327..942200d2a 100644 --- a/webapp/components/UserTeaser/__snapshots__/UserTeaser.spec.js.snap +++ b/webapp/components/UserTeaser/__snapshots__/UserTeaser.spec.js.snap @@ -173,7 +173,9 @@ exports[`UserTeaser given an user user is disabled current user is a moderator r Tilda Swinton
@@ -293,7 +295,9 @@ exports[`UserTeaser given an user with linkToProfile, on desktop renders 1`] = ` Tilda Swinton @@ -381,7 +385,9 @@ exports[`UserTeaser given an user with linkToProfile, on desktop when hovering t Tilda Swinton @@ -469,7 +475,9 @@ exports[`UserTeaser given an user with linkToProfile, on touch screen renders 1` Tilda Swinton @@ -558,7 +566,9 @@ exports[`UserTeaser given an user with linkToProfile, on touch screen when click Tilda Swinton @@ -651,7 +661,9 @@ exports[`UserTeaser given an user without linkToProfile, on desktop renders 1`] Tilda Swinton @@ -739,7 +751,9 @@ exports[`UserTeaser given an user without linkToProfile, on desktop when hoverin Tilda Swinton @@ -827,7 +841,9 @@ exports[`UserTeaser given an user without linkToProfile, on desktop when hoverin Tilda Swinton @@ -915,7 +931,9 @@ exports[`UserTeaser given an user without linkToProfile, on touch screen renders Tilda Swinton @@ -1004,7 +1022,9 @@ exports[`UserTeaser given an user without linkToProfile, on touch screen when cl Tilda Swinton @@ -1097,7 +1117,9 @@ exports[`UserTeaser given an user without linkToProfile, on touch screen when cl Tilda Swinton diff --git a/webapp/components/_new/features/Admin/Badges/BadgesSection.vue b/webapp/components/_new/features/Admin/Badges/BadgesSection.vue index fc89d2a50..016cf6432 100644 --- a/webapp/components/_new/features/Admin/Badges/BadgesSection.vue +++ b/webapp/components/_new/features/Admin/Badges/BadgesSection.vue @@ -8,7 +8,7 @@ @click="toggleBadge(badge)" :class="{ badge, inactive: !badge.isActive }" > - +
@@ -18,6 +18,7 @@