feat(backend): migrate to s3 (#8545)

## 🍰 Pullrequest
This will migrate our assets to an objectstorage via S3.

Before this PR is rolled out, the S3 credentials need to be configured in the respective infrastructure repository. The migration is implemented in a backend migration, i.e. I expect the `initContainer` to take a little longer but I hope then it's going to be fine. If any errors occcur, the migration should be repeatable, since the disk volume is still there.

### Issues
The backend having direct access on disk.

### Todo
- [ ] Configure backend environment variables in every infrastructure repo
- [ ] Remove kubernetes uploads volume in a future PR

Commits:

* refactor: follow @ulfgebhardt
  Here: https://github.com/Ocelot-Social-Community/Ocelot-Social/pull/8545#pullrequestreview-2846163417
  I don't know why the PR didn't include these changes already, I believe I made a mistake during rebase and lost the relevant commits.
* refactor: use typescript assertions
  I found it a better way to react to this comment: https://github.com/Ocelot-Social-Community/Ocelot-Social/pull/8545/files#r2092766596
* add S3 credentials
* refactor: easier to remember credentials
  It's for local development only
* give init container necessary file access
* fix: wrong upload location on production
* refactor: follow @ulfgebhardt's review
  See: https://github.com/Ocelot-Social-Community/Ocelot-Social/pull/8545#pullrequestreview-2881626504
This commit is contained in:
Robert Schäfer 2025-06-01 17:53:31 +08:00 committed by GitHub
parent 16803e45e6
commit d6a8de478b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
23 changed files with 1025 additions and 259 deletions

View File

@ -113,7 +113,7 @@ jobs:
- name: backend | docker compose - name: backend | docker compose
# doesn't work without the --build flag - this either means we should not load the cached images or cache the correct image # doesn't work without the --build flag - this either means we should not load the cached images or cache the correct image
run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach --no-deps neo4j backend --build run: docker compose -f docker-compose.yml -f docker-compose.test.yml up --detach backend --build
- name: backend | Initialize Database - name: backend | Initialize Database
run: docker compose exec -T backend yarn db:migrate init run: docker compose exec -T backend yarn db:migrate init

View File

@ -135,7 +135,7 @@ jobs:
docker load < /tmp/neo4j.tar docker load < /tmp/neo4j.tar
docker load < /tmp/backend.tar docker load < /tmp/backend.tar
docker load < /tmp/webapp.tar docker load < /tmp/webapp.tar
docker compose -f docker-compose.yml -f docker-compose.test.yml up --build --detach --no-deps webapp neo4j backend mailserver docker compose -f docker-compose.yml -f docker-compose.test.yml up --build --detach webapp mailserver
sleep 90s sleep 90s
- name: Full stack tests | run tests - name: Full stack tests | run tests
@ -176,4 +176,4 @@ jobs:
done done
echo "Done" echo "Done"
env: env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

View File

@ -17,5 +17,4 @@ build/
maintenance-worker/ maintenance-worker/
neo4j/ neo4j/
public/uploads/*
!.gitkeep !.gitkeep

View File

@ -40,11 +40,12 @@ COMMIT=
PUBLIC_REGISTRATION=false PUBLIC_REGISTRATION=false
INVITE_REGISTRATION=true INVITE_REGISTRATION=true
AWS_ACCESS_KEY_ID= AWS_ACCESS_KEY_ID=minio
AWS_SECRET_ACCESS_KEY= AWS_SECRET_ACCESS_KEY=12341234
AWS_ENDPOINT= AWS_ENDPOINT=http://localhost:9000
AWS_REGION= AWS_REGION=local
AWS_BUCKET= AWS_BUCKET=ocelot
S3_PUBLIC_GATEWAY=http://localhost:8000
CATEGORIES_ACTIVE=false CATEGORIES_ACTIVE=false
MAX_PINNED_POSTS=1 MAX_PINNED_POSTS=1

3
backend/.gitignore vendored
View File

@ -6,8 +6,7 @@ yarn-error.log
build/* build/*
coverage.lcov coverage.lcov
.nyc_output/ .nyc_output/
public/uploads/*
!.gitkeep !.gitkeep
# Apple macOS folder attribute file # Apple macOS folder attribute file
.DS_Store .DS_Store

View File

@ -102,12 +102,26 @@ const s3 = {
AWS_ENDPOINT: env.AWS_ENDPOINT, AWS_ENDPOINT: env.AWS_ENDPOINT,
AWS_REGION: env.AWS_REGION, AWS_REGION: env.AWS_REGION,
AWS_BUCKET: env.AWS_BUCKET, AWS_BUCKET: env.AWS_BUCKET,
S3_CONFIGURED: S3_PUBLIC_GATEWAY: env.S3_PUBLIC_GATEWAY,
env.AWS_ACCESS_KEY_ID && }
env.AWS_SECRET_ACCESS_KEY &&
env.AWS_ENDPOINT && export interface S3Configured {
env.AWS_REGION && AWS_ACCESS_KEY_ID: string
env.AWS_BUCKET, AWS_SECRET_ACCESS_KEY: string
AWS_ENDPOINT: string
AWS_REGION: string
AWS_BUCKET: string
S3_PUBLIC_GATEWAY: string | undefined
}
export const isS3configured = (config: typeof s3): config is S3Configured => {
return !!(
config.AWS_ACCESS_KEY_ID &&
config.AWS_SECRET_ACCESS_KEY &&
config.AWS_ENDPOINT &&
config.AWS_REGION &&
config.AWS_BUCKET
)
} }
const options = { const options = {

View File

@ -0,0 +1,91 @@
/* eslint-disable @typescript-eslint/no-unsafe-argument */
import { open } from 'node:fs/promises'
import path from 'node:path'
import { S3Client, ObjectCannedACL } from '@aws-sdk/client-s3'
import { Upload } from '@aws-sdk/lib-storage'
import { lookup } from 'mime-types'
import CONFIG from '@config/index'
import { getDriver } from '@db/neo4j'
export const description =
'Upload all image files to a S3 compatible object storage in order to reduce load on our backend.'
export async function up(_next) {
const { AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_ENDPOINT, AWS_BUCKET } = CONFIG
if (!(AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY && AWS_ENDPOINT && AWS_BUCKET)) {
throw new Error('No S3 configuration given, cannot upload image files')
}
const driver = getDriver()
const session = driver.session()
const transaction = session.beginTransaction()
const s3 = new S3Client({
credentials: {
accessKeyId: AWS_ACCESS_KEY_ID,
secretAccessKey: AWS_SECRET_ACCESS_KEY,
},
endpoint: AWS_ENDPOINT,
forcePathStyle: true,
})
try {
const { records } = await transaction.run('MATCH (image:Image) RETURN image.url as url')
let urls: string[] = records.map((r) => r.get('url') as string)
urls = urls.filter((url) => url.startsWith('/uploads'))
// eslint-disable-next-line no-console
console.log('URLS uploaded:')
await Promise.all(
urls
.map((url) => async () => {
const { pathname } = new URL(url, 'http://example.org')
// TODO: find a better way to do this - this is quite a hack
const fileLocation =
CONFIG.NODE_ENV === 'production'
? path.join(__dirname, `../../../../public/${pathname}`) // we're in the /build folder
: path.join(__dirname, `../../../public/${pathname}`)
const s3Location = `original${pathname}`
const mimeType = lookup(fileLocation)
// eslint-disable-next-line security/detect-non-literal-fs-filename
const fileHandle = await open(fileLocation)
const params = {
Bucket: AWS_BUCKET,
Key: s3Location,
ACL: ObjectCannedACL.public_read,
ContentType: mimeType || 'image/jpeg',
Body: fileHandle.createReadStream(),
}
const command = new Upload({ client: s3, params })
const data = await command.done()
const { Location: spacesUrl } = data
const updatedRecord = await transaction.run(
'MATCH (image:Image {url: $url}) SET image.url = $spacesUrl RETURN image.url as url',
{ url, spacesUrl },
)
const [updatedUrl] = updatedRecord.records.map((record) => record.get('url') as string)
// eslint-disable-next-line no-console
console.log(updatedUrl)
return updatedUrl
})
.map((p) => p()),
)
await transaction.commit()
} catch (error) {
// eslint-disable-next-line no-console
console.log(error)
await transaction.rollback()
// eslint-disable-next-line no-console
console.log('rolled back')
throw new Error(error)
} finally {
await session.close()
}
}
export function down(_next) {
throw new Error('This migration is irreversible: The backend does not have disk access anymore.')
}

View File

@ -18,7 +18,7 @@ import Resolver, {
removeUndefinedNullValuesFromObject, removeUndefinedNullValuesFromObject,
convertObjectToCypherMapLiteral, convertObjectToCypherMapLiteral,
} from './helpers/Resolver' } from './helpers/Resolver'
import { mergeImage } from './images/images' import { images } from './images/images'
import { createOrUpdateLocations } from './users/location' import { createOrUpdateLocations } from './users/location'
export default { export default {
@ -260,7 +260,7 @@ export default {
}) })
const [group] = transactionResponse.records.map((record) => record.get('group')) const [group] = transactionResponse.records.map((record) => record.get('group'))
if (avatarInput) { if (avatarInput) {
await mergeImage(group, 'AVATAR_IMAGE', avatarInput, { transaction }) await images.mergeImage(group, 'AVATAR_IMAGE', avatarInput, { transaction })
} }
return group return group
}) })

View File

@ -1,32 +1,27 @@
/* eslint-disable @typescript-eslint/require-await */ import CONFIG, { isS3configured } from '@config/index'
/* eslint-disable @typescript-eslint/restrict-template-expressions */
/* eslint-disable @typescript-eslint/no-unsafe-call */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-return */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable @typescript-eslint/no-unsafe-argument */
/* eslint-disable promise/avoid-new */
/* eslint-disable security/detect-non-literal-fs-filename */
import { existsSync, unlinkSync, createWriteStream } from 'node:fs' import { images as imagesLocal } from './imagesLocal'
import path from 'node:path' import { images as imagesS3 } from './imagesS3'
import { S3Client, DeleteObjectCommand, ObjectCannedACL } from '@aws-sdk/client-s3'
import { Upload } from '@aws-sdk/lib-storage'
import { UserInputError } from 'apollo-server'
import slug from 'slug'
import { v4 as uuid } from 'uuid'
import CONFIG from '@config/index'
import { getDriver } from '@db/neo4j'
import type { FileUpload } from 'graphql-upload' import type { FileUpload } from 'graphql-upload'
import type { Transaction } from 'neo4j-driver' import type { Transaction } from 'neo4j-driver'
type FileDeleteCallback = (url: string) => Promise<void> export type FileDeleteCallback = (url: string) => Promise<void>
type FileUploadCallback = (
export type FileUploadCallback = (
upload: Pick<FileUpload, 'createReadStream' | 'mimetype'> & { uniqueFilename: string }, upload: Pick<FileUpload, 'createReadStream' | 'mimetype'> & { uniqueFilename: string },
) => Promise<string> ) => Promise<string>
export interface DeleteImageOpts {
transaction?: Transaction
deleteCallback?: FileDeleteCallback
}
export interface MergeImageOpts {
transaction?: Transaction
uploadCallback?: FileUploadCallback
deleteCallback?: FileDeleteCallback
}
export interface ImageInput { export interface ImageInput {
upload?: Promise<FileUpload> upload?: Promise<FileUpload>
alt?: string alt?: string
@ -35,191 +30,29 @@ export interface ImageInput {
type?: string type?: string
} }
// const widths = [34, 160, 320, 640, 1024] export interface Image {
const { AWS_BUCKET: Bucket, S3_CONFIGURED } = CONFIG url: string
alt?: string
const createS3Client = () => { sensitive?: boolean
const { AWS_ENDPOINT, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY } = CONFIG aspectRatio?: number
if (!(AWS_ENDPOINT && AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY)) { type?: string
throw new Error('Missing AWS credentials.')
}
return new S3Client({
credentials: {
accessKeyId: AWS_ACCESS_KEY_ID,
secretAccessKey: AWS_SECRET_ACCESS_KEY,
},
endpoint: AWS_ENDPOINT,
forcePathStyle: true,
})
} }
interface DeleteImageOpts { export interface Images {
transaction?: Transaction deleteImage: (
deleteCallback?: FileDeleteCallback resource: { id: string },
} relationshipType: 'HERO_IMAGE' | 'AVATAR_IMAGE',
export async function deleteImage(resource, relationshipType, opts: DeleteImageOpts = {}) { opts?: DeleteImageOpts,
sanitizeRelationshipType(relationshipType) // eslint-disable-next-line @typescript-eslint/no-explicit-any
const { transaction, deleteCallback } = opts ) => Promise<any>
if (!transaction) return wrapTransaction(deleteImage, [resource, relationshipType], opts)
const txResult = await transaction.run( mergeImage: (
` resource: { id: string },
MATCH (resource {id: $resource.id})-[rel:${relationshipType}]->(image:Image) relationshipType: 'HERO_IMAGE' | 'AVATAR_IMAGE',
WITH image, image {.*} as imageProps imageInput: ImageInput | null | undefined,
DETACH DELETE image opts?: MergeImageOpts,
RETURN imageProps // eslint-disable-next-line @typescript-eslint/no-explicit-any
`, ) => Promise<any>
{ resource },
)
const [image] = txResult.records.map((record) => record.get('imageProps'))
// This behaviour differs from `mergeImage`. If you call `mergeImage`
// with metadata for an image that does not exist, it's an indicator
// of an error (so throw an error). If we bulk delete an image, it
// could very well be that there is no image for the resource.
if (image) deleteImageFile(image, deleteCallback)
return image
} }
interface MergeImageOpts { export const images = isS3configured(CONFIG) ? imagesS3(CONFIG) : imagesLocal
transaction?: Transaction
uploadCallback?: FileUploadCallback
deleteCallback?: FileDeleteCallback
}
export async function mergeImage(
resource,
relationshipType,
imageInput: ImageInput | null | undefined,
opts: MergeImageOpts = {},
) {
if (typeof imageInput === 'undefined') return
if (imageInput === null) return deleteImage(resource, relationshipType, opts)
sanitizeRelationshipType(relationshipType)
const { transaction, uploadCallback, deleteCallback } = opts
if (!transaction)
return wrapTransaction(mergeImage, [resource, relationshipType, imageInput], opts)
let txResult
txResult = await transaction.run(
`
MATCH (resource {id: $resource.id})-[:${relationshipType}]->(image:Image)
RETURN image {.*}
`,
{ resource },
)
const [existingImage] = txResult.records.map((record) => record.get('image'))
const { upload } = imageInput
if (!(existingImage || upload)) throw new UserInputError('Cannot find image for given resource')
if (existingImage && upload) deleteImageFile(existingImage, deleteCallback)
const url = await uploadImageFile(upload, uploadCallback)
const { alt, sensitive, aspectRatio, type } = imageInput
const image = { alt, sensitive, aspectRatio, url, type }
txResult = await transaction.run(
`
MATCH (resource {id: $resource.id})
MERGE (resource)-[:${relationshipType}]->(image:Image)
ON CREATE SET image.createdAt = toString(datetime())
ON MATCH SET image.updatedAt = toString(datetime())
SET image += $image
RETURN image {.*}
`,
{ resource, image },
)
const [mergedImage] = txResult.records.map((record) => record.get('image'))
return mergedImage
}
const wrapTransaction = async (wrappedCallback, args, opts) => {
const session = getDriver().session()
try {
const result = await session.writeTransaction(async (transaction) => {
return wrappedCallback(...args, { ...opts, transaction })
})
return result
} finally {
await session.close()
}
}
const deleteImageFile = (image, deleteCallback: FileDeleteCallback | undefined) => {
if (!deleteCallback) {
deleteCallback = S3_CONFIGURED ? s3Delete : localFileDelete
}
const { url } = image
// eslint-disable-next-line @typescript-eslint/no-floating-promises
deleteCallback(url)
return url
}
const uploadImageFile = async (
upload: Promise<FileUpload> | undefined,
uploadCallback: FileUploadCallback | undefined,
) => {
if (!upload) return undefined
if (!uploadCallback) {
uploadCallback = S3_CONFIGURED ? s3Upload : localFileUpload
}
// eslint-disable-next-line @typescript-eslint/unbound-method
const { createReadStream, filename, mimetype } = await upload
const { name, ext } = path.parse(filename)
const uniqueFilename = `${uuid()}-${slug(name)}${ext}`
return uploadCallback({ createReadStream, uniqueFilename, mimetype })
}
const sanitizeRelationshipType = (relationshipType) => {
// Cypher query language does not allow to parameterize relationship types
// See: https://github.com/neo4j/neo4j/issues/340
if (!['HERO_IMAGE', 'AVATAR_IMAGE'].includes(relationshipType)) {
throw new Error(`Unknown relationship type ${relationshipType}`)
}
}
const localFileUpload: FileUploadCallback = ({ createReadStream, uniqueFilename }) => {
const destination = `/uploads/${uniqueFilename}`
return new Promise((resolve, reject) =>
createReadStream().pipe(
createWriteStream(`public${destination}`)
.on('finish', () => resolve(destination))
.on('error', (error) => reject(error)),
),
)
}
const s3Upload: FileUploadCallback = async ({ createReadStream, uniqueFilename, mimetype }) => {
const s3Location = `original/${uniqueFilename}`
const params = {
Bucket,
Key: s3Location,
ACL: ObjectCannedACL.public_read,
ContentType: mimetype,
Body: createReadStream(),
}
const s3 = createS3Client()
const command = new Upload({ client: s3, params })
const data = await command.done()
const { Location } = data
if (!Location) {
throw new Error('File upload did not return `Location`')
}
return Location
}
const localFileDelete: FileDeleteCallback = async (url) => {
const location = `public${url}`
// eslint-disable-next-line n/no-sync
if (existsSync(location)) unlinkSync(location)
}
const s3Delete: FileDeleteCallback = async (url) => {
let { pathname } = new URL(url, 'http://example.org') // dummy domain to avoid invalid URL error
pathname = pathname.substring(1) // remove first character '/'
const prefix = `${Bucket}/`
if (pathname.startsWith(prefix)) {
pathname = pathname.slice(prefix.length)
}
const params = {
Bucket,
Key: pathname,
}
const s3 = createS3Client()
await s3.send(new DeleteObjectCommand(params))
}

View File

@ -11,7 +11,7 @@ import { UserInputError } from 'apollo-server'
import Factory, { cleanDatabase } from '@db/factories' import Factory, { cleanDatabase } from '@db/factories'
import { getNeode, getDriver } from '@db/neo4j' import { getNeode, getDriver } from '@db/neo4j'
import { deleteImage, mergeImage } from './images' import { images } from './imagesLocal'
import type { ImageInput } from './images' import type { ImageInput } from './images'
import type { FileUpload } from 'graphql-upload' import type { FileUpload } from 'graphql-upload'
@ -42,10 +42,12 @@ afterEach(async () => {
}) })
describe('deleteImage', () => { describe('deleteImage', () => {
const { deleteImage } = images
describe('given a resource with an image', () => { describe('given a resource with an image', () => {
let user let user: { id: string }
beforeEach(async () => { beforeEach(async () => {
user = await Factory.build( const u = await Factory.build(
'user', 'user',
{}, {},
{ {
@ -55,7 +57,7 @@ describe('deleteImage', () => {
}), }),
}, },
) )
user = await user.toJson() user = await u.toJson()
}) })
it('deletes `Image` node', async () => { it('deletes `Image` node', async () => {
@ -65,8 +67,8 @@ describe('deleteImage', () => {
}) })
it('calls deleteCallback', async () => { it('calls deleteCallback', async () => {
user = await Factory.build('user') const u = await Factory.build('user')
user = await user.toJson() user = await u.toJson()
await deleteImage(user, 'AVATAR_IMAGE', { deleteCallback }) await deleteImage(user, 'AVATAR_IMAGE', { deleteCallback })
expect(deleteCallback).toHaveBeenCalled() expect(deleteCallback).toHaveBeenCalled()
}) })
@ -117,8 +119,9 @@ describe('deleteImage', () => {
}) })
describe('mergeImage', () => { describe('mergeImage', () => {
const { mergeImage } = images
let imageInput: ImageInput let imageInput: ImageInput
let post let post: { id: string }
beforeEach(() => { beforeEach(() => {
imageInput = { imageInput = {
alt: 'A description of the new image', alt: 'A description of the new image',
@ -145,7 +148,7 @@ describe('mergeImage', () => {
describe('on existing resource', () => { describe('on existing resource', () => {
beforeEach(async () => { beforeEach(async () => {
post = await Factory.build( const p = await Factory.build(
'post', 'post',
{ id: 'p99' }, { id: 'p99' },
{ {
@ -153,7 +156,7 @@ describe('mergeImage', () => {
image: null, image: null,
}, },
) )
post = await post.toJson() post = await p.toJson()
}) })
it('returns new image', async () => { it('returns new image', async () => {
@ -196,7 +199,7 @@ describe('mergeImage', () => {
`MATCH(p:Post {id: "p99"})-[:HERO_IMAGE]->(i:Image) RETURN i,p`, `MATCH(p:Post {id: "p99"})-[:HERO_IMAGE]->(i:Image) RETURN i,p`,
{}, {},
) )
post = neode.hydrateFirst(result, 'p', neode.model('Post')) post = neode.hydrateFirst<{ id: string }>(result, 'p', neode.model('Post')).properties()
const image = neode.hydrateFirst(result, 'i', neode.model('Image')) const image = neode.hydrateFirst(result, 'i', neode.model('Image'))
expect(post).toBeTruthy() expect(post).toBeTruthy()
expect(image).toBeTruthy() expect(image).toBeTruthy()
@ -204,7 +207,10 @@ describe('mergeImage', () => {
it('whitelists relationship types', async () => { it('whitelists relationship types', async () => {
await expect( await expect(
mergeImage(post, 'WHATEVER', imageInput, { uploadCallback, deleteCallback }), mergeImage(post, 'WHATEVER' as 'HERO_IMAGE', imageInput, {
uploadCallback,
deleteCallback,
}),
).rejects.toEqual(new Error('Unknown relationship type WHATEVER')) ).rejects.toEqual(new Error('Unknown relationship type WHATEVER'))
}) })
@ -311,8 +317,8 @@ describe('mergeImage', () => {
describe('without image.upload', () => { describe('without image.upload', () => {
it('throws UserInputError', async () => { it('throws UserInputError', async () => {
post = await Factory.build('post', { id: 'p99' }, { image: null }) const p = await Factory.build('post', { id: 'p99' }, { image: null })
post = await post.toJson() post = await p.toJson()
await expect(mergeImage(post, 'HERO_IMAGE', imageInput)).rejects.toEqual( await expect(mergeImage(post, 'HERO_IMAGE', imageInput)).rejects.toEqual(
new UserInputError('Cannot find image for given resource'), new UserInputError('Cannot find image for given resource'),
) )
@ -320,7 +326,7 @@ describe('mergeImage', () => {
describe('if resource has an image already', () => { describe('if resource has an image already', () => {
beforeEach(async () => { beforeEach(async () => {
post = await Factory.build( const p = await Factory.build(
'post', 'post',
{ {
id: 'p99', id: 'p99',
@ -339,7 +345,7 @@ describe('mergeImage', () => {
}), }),
}, },
) )
post = await post.toJson() post = await p.toJson()
}) })
it('does not call deleteCallback', async () => { it('does not call deleteCallback', async () => {

View File

@ -0,0 +1,134 @@
/* eslint-disable @typescript-eslint/require-await */
/* eslint-disable @typescript-eslint/no-unsafe-call */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-return */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable @typescript-eslint/no-unsafe-argument */
/* eslint-disable promise/avoid-new */
/* eslint-disable security/detect-non-literal-fs-filename */
import { existsSync, unlinkSync, createWriteStream } from 'node:fs'
import path from 'node:path'
import { UserInputError } from 'apollo-server'
import slug from 'slug'
import { v4 as uuid } from 'uuid'
import { sanitizeRelationshipType } from './sanitizeRelationshipTypes'
import { wrapTransaction } from './wrapTransaction'
import type { Images, FileDeleteCallback, FileUploadCallback } from './images'
import type { FileUpload } from 'graphql-upload'
const deleteImage: Images['deleteImage'] = async (resource, relationshipType, opts = {}) => {
sanitizeRelationshipType(relationshipType)
const { transaction, deleteCallback } = opts
if (!transaction) return wrapTransaction(deleteImage, [resource, relationshipType], opts)
const txResult = await transaction.run(
`
MATCH (resource {id: $resource.id})-[rel:${relationshipType}]->(image:Image)
WITH image, image {.*} as imageProps
DETACH DELETE image
RETURN imageProps
`,
{ resource },
)
const [image] = txResult.records.map((record) => record.get('imageProps'))
// This behaviour differs from `mergeImage`. If you call `mergeImage`
// with metadata for an image that does not exist, it's an indicator
// of an error (so throw an error). If we bulk delete an image, it
// could very well be that there is no image for the resource.
if (image) deleteImageFile(image, deleteCallback)
return image
}
const mergeImage: Images['mergeImage'] = async (
resource,
relationshipType,
imageInput,
opts = {},
) => {
if (typeof imageInput === 'undefined') return
if (imageInput === null) return deleteImage(resource, relationshipType, opts)
sanitizeRelationshipType(relationshipType)
const { transaction, uploadCallback, deleteCallback } = opts
if (!transaction)
return wrapTransaction(mergeImage, [resource, relationshipType, imageInput], opts)
let txResult
txResult = await transaction.run(
`
MATCH (resource {id: $resource.id})-[:${relationshipType}]->(image:Image)
RETURN image {.*}
`,
{ resource },
)
const [existingImage] = txResult.records.map((record) => record.get('image'))
const { upload } = imageInput
if (!(existingImage || upload)) throw new UserInputError('Cannot find image for given resource')
if (existingImage && upload) deleteImageFile(existingImage, deleteCallback)
const url = await uploadImageFile(upload, uploadCallback)
const { alt, sensitive, aspectRatio, type } = imageInput
const image = { alt, sensitive, aspectRatio, url, type }
txResult = await transaction.run(
`
MATCH (resource {id: $resource.id})
MERGE (resource)-[:${relationshipType}]->(image:Image)
ON CREATE SET image.createdAt = toString(datetime())
ON MATCH SET image.updatedAt = toString(datetime())
SET image += $image
RETURN image {.*}
`,
{ resource, image },
)
const [mergedImage] = txResult.records.map((record) => record.get('image'))
return mergedImage
}
const localFileDelete: FileDeleteCallback = async (url) => {
const location = `public${url}`
// eslint-disable-next-line n/no-sync
if (existsSync(location)) unlinkSync(location)
}
const deleteImageFile = (image, deleteCallback: FileDeleteCallback | undefined) => {
if (!deleteCallback) {
deleteCallback = localFileDelete
}
const { url } = image
// eslint-disable-next-line @typescript-eslint/no-floating-promises
deleteCallback(url)
return url
}
const uploadImageFile = async (
upload: Promise<FileUpload> | undefined,
uploadCallback: FileUploadCallback | undefined,
) => {
if (!upload) return undefined
if (!uploadCallback) {
uploadCallback = localFileUpload
}
// eslint-disable-next-line @typescript-eslint/unbound-method
const { createReadStream, filename, mimetype } = await upload
const { name, ext } = path.parse(filename)
const uniqueFilename = `${uuid()}-${slug(name)}${ext}`
return uploadCallback({ createReadStream, uniqueFilename, mimetype })
}
const localFileUpload: FileUploadCallback = ({ createReadStream, uniqueFilename }) => {
const destination = `/uploads/${uniqueFilename}`
return new Promise((resolve, reject) =>
createReadStream().pipe(
createWriteStream(`public${destination}`)
.on('finish', () => resolve(destination))
.on('error', (error) => reject(error)),
),
)
}
export const images: Images = {
deleteImage,
mergeImage,
}

View File

@ -0,0 +1,407 @@
/* eslint-disable @typescript-eslint/require-await */
/* eslint-disable @typescript-eslint/no-unsafe-return */
/* eslint-disable @typescript-eslint/restrict-template-expressions */
/* eslint-disable @typescript-eslint/no-unsafe-call */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable promise/prefer-await-to-callbacks */
import { UserInputError } from 'apollo-server'
import Factory, { cleanDatabase } from '@db/factories'
import { getNeode, getDriver } from '@db/neo4j'
import type { S3Configured } from '@src/config'
import { images } from './imagesS3'
import type { ImageInput } from './images'
import type { FileUpload } from 'graphql-upload'
const driver = getDriver()
const neode = getNeode()
const uuid = '[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}'
let uploadCallback
let deleteCallback
const config: S3Configured = {
AWS_ACCESS_KEY_ID: 'AWS_ACCESS_KEY_ID',
AWS_SECRET_ACCESS_KEY: 'AWS_SECRET_ACCESS_KEY',
AWS_BUCKET: 'AWS_BUCKET',
AWS_ENDPOINT: 'AWS_ENDPOINT',
AWS_REGION: 'AWS_REGION',
S3_PUBLIC_GATEWAY: undefined,
}
beforeAll(async () => {
await cleanDatabase()
})
afterAll(async () => {
await cleanDatabase()
await driver.close()
})
beforeEach(async () => {
uploadCallback = jest.fn(
({ uniqueFilename }) => `http://your-objectstorage.com/bucket/${uniqueFilename}`,
)
deleteCallback = jest.fn()
})
// TODO: avoid database clean after each test in the future if possible for performance and flakyness reasons by filling the database step by step, see issue https://github.com/Ocelot-Social-Community/Ocelot-Social/issues/4543
afterEach(async () => {
await cleanDatabase()
})
describe('deleteImage', () => {
const { deleteImage } = images(config)
describe('given a resource with an image', () => {
let user: { id: string }
beforeEach(async () => {
const u = await Factory.build(
'user',
{},
{
avatar: Factory.build('image', {
url: 'http://localhost/some/avatar/url/',
alt: 'This is the avatar image of a user',
}),
},
)
user = await u.toJson()
})
it('deletes `Image` node', async () => {
await expect(neode.all('Image')).resolves.toHaveLength(1)
await deleteImage(user, 'AVATAR_IMAGE', { deleteCallback })
await expect(neode.all('Image')).resolves.toHaveLength(0)
})
it('calls deleteCallback', async () => {
await deleteImage(user, 'AVATAR_IMAGE', { deleteCallback })
expect(deleteCallback).toHaveBeenCalled()
})
describe('given a transaction parameter', () => {
it('executes cypher statements within the transaction', async () => {
const session = driver.session()
let someString: string
try {
someString = await session.writeTransaction(async (transaction) => {
await deleteImage(user, 'AVATAR_IMAGE', {
deleteCallback,
transaction,
})
const txResult = await transaction.run('RETURN "Hello" as result')
const [result] = txResult.records.map((record) => record.get('result'))
return result
})
} finally {
await session.close()
}
await expect(neode.all('Image')).resolves.toHaveLength(0)
expect(someString).toEqual('Hello')
})
it('rolls back the transaction in case of errors', async () => {
await expect(neode.all('Image')).resolves.toHaveLength(1)
const session = driver.session()
try {
await session.writeTransaction(async (transaction) => {
await deleteImage(user, 'AVATAR_IMAGE', {
deleteCallback,
transaction,
})
throw new Error('Ouch!')
})
// eslint-disable-next-line no-catch-all/no-catch-all
} catch (err) {
// nothing has been deleted
await expect(neode.all('Image')).resolves.toHaveLength(1)
// all good
} finally {
await session.close()
}
})
})
})
})
describe('mergeImage', () => {
const { mergeImage } = images(config)
let imageInput: ImageInput
let post: { id: string }
beforeEach(() => {
imageInput = {
alt: 'A description of the new image',
}
})
describe('given image.upload', () => {
beforeEach(() => {
const createReadStream: FileUpload['createReadStream'] = (() => ({
pipe: () => ({
on: (_, callback) => callback(),
}),
})) as unknown as FileUpload['createReadStream']
imageInput = {
...imageInput,
upload: Promise.resolve({
filename: 'image.jpg',
mimetype: 'image/jpeg',
encoding: '7bit',
createReadStream,
}),
}
})
describe('on existing resource', () => {
beforeEach(async () => {
const p = await Factory.build(
'post',
{ id: 'p99' },
{
author: Factory.build('user', {}, { avatar: null }),
image: null,
},
)
post = await p.toJson()
})
it('returns new image', async () => {
await expect(
mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }),
).resolves.toMatchObject({
url: expect.any(String),
alt: 'A description of the new image',
})
})
it('calls upload callback', async () => {
await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback })
expect(uploadCallback).toHaveBeenCalled()
})
it('creates `:Image` node', async () => {
await expect(neode.all('Image')).resolves.toHaveLength(0)
await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback })
await expect(neode.all('Image')).resolves.toHaveLength(1)
})
it('creates a url safe name', async () => {
if (!imageInput.upload) {
throw new Error('Test imageInput was not setup correctly.')
}
const upload = await imageInput.upload
upload.filename = '/path/to/awkward?/ file-location/?foo- bar-avatar.jpg'
imageInput.upload = Promise.resolve(upload)
await expect(
mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }),
).resolves.toMatchObject({
url: expect.stringMatching(
new RegExp(`^http://your-objectstorage.com/bucket/${uuid}-foo-bar-avatar.jpg`),
),
})
})
describe('given a `S3_PUBLIC_GATEWAY` configuration', () => {
const { mergeImage } = images({
...config,
S3_PUBLIC_GATEWAY: 'http://s3-public-gateway.com',
})
it('changes the domain of the URL to a server that could e.g. apply image transformations', async () => {
if (!imageInput.upload) {
throw new Error('Test imageInput was not setup correctly.')
}
const upload = await imageInput.upload
upload.filename = '/path/to/file-location/foo-bar-avatar.jpg'
imageInput.upload = Promise.resolve(upload)
await expect(
mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }),
).resolves.toMatchObject({
url: expect.stringMatching(
new RegExp(`^http://s3-public-gateway.com/bucket/${uuid}-foo-bar-avatar.jpg`),
),
})
})
})
it('connects resource with image via given image type', async () => {
await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback })
const result = await neode.cypher(
`MATCH(p:Post {id: "p99"})-[:HERO_IMAGE]->(i:Image) RETURN i,p`,
{},
)
post = neode.hydrateFirst<{ id: string }>(result, 'p', neode.model('Post')).properties()
const image = neode.hydrateFirst(result, 'i', neode.model('Image'))
expect(post).toBeTruthy()
expect(image).toBeTruthy()
})
it('whitelists relationship types', async () => {
await expect(
mergeImage(post, 'WHATEVER' as 'HERO_IMAGE', imageInput, {
uploadCallback,
deleteCallback,
}),
).rejects.toEqual(new Error('Unknown relationship type WHATEVER'))
})
it('sets metadata', async () => {
await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback })
const image = await neode.first<typeof Image>('Image', {}, undefined)
await expect(image.toJson()).resolves.toMatchObject({
alt: 'A description of the new image',
createdAt: expect.any(String),
url: expect.any(String),
})
})
describe('given a transaction parameter', () => {
it('executes cypher statements within the transaction', async () => {
const session = driver.session()
try {
await session.writeTransaction(async (transaction) => {
const image = await mergeImage(post, 'HERO_IMAGE', imageInput, {
uploadCallback,
deleteCallback,
transaction,
})
return transaction.run(
`
MATCH(image:Image {url: $image.url})
SET image.alt = 'This alt text gets overwritten'
RETURN image {.*}
`,
{ image },
)
})
} finally {
await session.close()
}
const image = await neode.first<typeof Image>(
'Image',
{ alt: 'This alt text gets overwritten' },
undefined,
)
await expect(image.toJson()).resolves.toMatchObject({
alt: 'This alt text gets overwritten',
})
})
it('rolls back the transaction in case of errors', async () => {
const session = driver.session()
try {
await session.writeTransaction(async (transaction) => {
const image = await mergeImage(post, 'HERO_IMAGE', imageInput, {
uploadCallback,
deleteCallback,
transaction,
})
return transaction.run('Ooops invalid cypher!', { image })
})
// eslint-disable-next-line no-catch-all/no-catch-all
} catch (err) {
// nothing has been created
await expect(neode.all('Image')).resolves.toHaveLength(0)
// all good
} finally {
await session.close()
}
})
})
describe('if resource has an image already', () => {
beforeEach(async () => {
const [post, image] = await Promise.all([
neode.find('Post', 'p99'),
Factory.build('image'),
])
await post.relateTo(image, 'image')
})
it('calls deleteCallback', async () => {
await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback })
expect(deleteCallback).toHaveBeenCalled()
})
it('calls uploadCallback', async () => {
await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback })
expect(uploadCallback).toHaveBeenCalled()
})
it('updates metadata of existing image node', async () => {
await expect(neode.all('Image')).resolves.toHaveLength(1)
await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback })
await expect(neode.all('Image')).resolves.toHaveLength(1)
const image = await neode.first<typeof Image>('Image', {}, undefined)
await expect(image.toJson()).resolves.toMatchObject({
alt: 'A description of the new image',
createdAt: expect.any(String),
url: expect.any(String),
// TODO
// width:
// height:
})
})
})
})
})
describe('without image.upload', () => {
it('throws UserInputError', async () => {
const p = await Factory.build('post', { id: 'p99' }, { image: null })
post = await p.toJson()
await expect(mergeImage(post, 'HERO_IMAGE', imageInput)).rejects.toEqual(
new UserInputError('Cannot find image for given resource'),
)
})
describe('if resource has an image already', () => {
beforeEach(async () => {
const p = await Factory.build(
'post',
{
id: 'p99',
},
{
author: Factory.build(
'user',
{},
{
avatar: null,
},
),
image: Factory.build('image', {
alt: 'This is the previous, not updated image',
url: 'http://localhost/some/original/url',
}),
},
)
post = await p.toJson()
})
it('does not call deleteCallback', async () => {
await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback })
expect(deleteCallback).not.toHaveBeenCalled()
})
it('does not call uploadCallback', async () => {
await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback })
expect(uploadCallback).not.toHaveBeenCalled()
})
it('updates metadata', async () => {
await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback })
const images = await neode.all('Image')
expect(images).toHaveLength(1)
await expect(images.first().toJson()).resolves.toMatchObject({
createdAt: expect.any(String),
url: expect.any(String),
alt: 'A description of the new image',
})
})
})
})
})

View File

@ -0,0 +1,153 @@
import path from 'node:path'
import { S3Client, DeleteObjectCommand, ObjectCannedACL } from '@aws-sdk/client-s3'
import { Upload } from '@aws-sdk/lib-storage'
import { UserInputError } from 'apollo-server'
import slug from 'slug'
import { v4 as uuid } from 'uuid'
import { S3Configured } from '@config/index'
import { sanitizeRelationshipType } from './sanitizeRelationshipTypes'
import { wrapTransaction } from './wrapTransaction'
import type { Image, Images, FileDeleteCallback, FileUploadCallback } from './images'
import type { FileUpload } from 'graphql-upload'
export const images = (config: S3Configured) => {
// const widths = [34, 160, 320, 640, 1024]
const { AWS_BUCKET: Bucket, S3_PUBLIC_GATEWAY } = config
const { AWS_ENDPOINT, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY } = config
const s3 = new S3Client({
credentials: {
accessKeyId: AWS_ACCESS_KEY_ID,
secretAccessKey: AWS_SECRET_ACCESS_KEY,
},
endpoint: AWS_ENDPOINT,
forcePathStyle: true,
})
const deleteImage: Images['deleteImage'] = async (resource, relationshipType, opts = {}) => {
sanitizeRelationshipType(relationshipType)
const { transaction, deleteCallback = s3Delete } = opts
if (!transaction) return wrapTransaction(deleteImage, [resource, relationshipType], opts)
const txResult = await transaction.run(
`
MATCH (resource {id: $resource.id})-[rel:${relationshipType}]->(image:Image)
WITH image, image {.*} as imageProps
DETACH DELETE image
RETURN imageProps
`,
{ resource },
)
const [image] = txResult.records.map((record) => record.get('imageProps') as Image)
// This behaviour differs from `mergeImage`. If you call `mergeImage`
// with metadata for an image that does not exist, it's an indicator
// of an error (so throw an error). If we bulk delete an image, it
// could very well be that there is no image for the resource.
if (image) {
await deleteCallback(image.url)
}
return image
}
const mergeImage: Images['mergeImage'] = async (
resource,
relationshipType,
imageInput,
opts = {},
) => {
if (typeof imageInput === 'undefined') return
if (imageInput === null) return deleteImage(resource, relationshipType, opts)
sanitizeRelationshipType(relationshipType)
const { transaction, uploadCallback, deleteCallback = s3Delete } = opts
if (!transaction)
return wrapTransaction(mergeImage, [resource, relationshipType, imageInput], opts)
let txResult = await transaction.run(
`
MATCH (resource {id: $resource.id})-[:${relationshipType}]->(image:Image)
RETURN image {.*}
`,
{ resource },
)
const [existingImage] = txResult.records.map((record) => record.get('image') as Image)
const { upload } = imageInput
if (!(existingImage || upload)) throw new UserInputError('Cannot find image for given resource')
if (existingImage && upload) {
await deleteCallback(existingImage.url)
}
const url = await uploadImageFile(upload, uploadCallback)
const { alt, sensitive, aspectRatio, type } = imageInput
const image = { alt, sensitive, aspectRatio, url, type }
txResult = await transaction.run(
`
MATCH (resource {id: $resource.id})
MERGE (resource)-[:${relationshipType}]->(image:Image)
ON CREATE SET image.createdAt = toString(datetime())
ON MATCH SET image.updatedAt = toString(datetime())
SET image += $image
RETURN image {.*}
`,
{ resource, image },
)
const [mergedImage] = txResult.records.map((record) => record.get('image') as Image)
return mergedImage
}
const uploadImageFile = async (
uploadPromise: Promise<FileUpload> | undefined,
uploadCallback: FileUploadCallback | undefined = s3Upload,
) => {
if (!uploadPromise) return undefined
const upload = await uploadPromise
const { name, ext } = path.parse(upload.filename)
const uniqueFilename = `${uuid()}-${slug(name)}${ext}`
const Location = await uploadCallback({ ...upload, uniqueFilename })
if (!S3_PUBLIC_GATEWAY) {
return Location
}
const publicLocation = new URL(S3_PUBLIC_GATEWAY)
publicLocation.pathname = new URL(Location).pathname
return publicLocation.href
}
const s3Upload: FileUploadCallback = async ({ createReadStream, uniqueFilename, mimetype }) => {
const s3Location = `original/${uniqueFilename}`
const params = {
Bucket,
Key: s3Location,
ACL: ObjectCannedACL.public_read,
ContentType: mimetype,
Body: createReadStream(),
}
const command = new Upload({ client: s3, params })
const data = await command.done()
const { Location } = data
if (!Location) {
throw new Error('File upload did not return `Location`')
}
return Location
}
const s3Delete: FileDeleteCallback = async (url) => {
let { pathname } = new URL(url, 'http://example.org') // dummy domain to avoid invalid URL error
pathname = pathname.substring(1) // remove first character '/'
const prefix = `${Bucket}/`
if (pathname.startsWith(prefix)) {
pathname = pathname.slice(prefix.length)
}
const params = {
Bucket,
Key: pathname,
}
await s3.send(new DeleteObjectCommand(params))
}
const images: Images = {
deleteImage,
mergeImage,
}
return images
}

View File

@ -0,0 +1,9 @@
export function sanitizeRelationshipType(
relationshipType: string,
): asserts relationshipType is 'HERO_IMAGE' | 'AVATAR_IMAGE' {
// Cypher query language does not allow to parameterize relationship types
// See: https://github.com/neo4j/neo4j/issues/340
if (!['HERO_IMAGE', 'AVATAR_IMAGE'].includes(relationshipType)) {
throw new Error(`Unknown relationship type ${relationshipType}`)
}
}

View File

@ -0,0 +1,23 @@
import { getDriver } from '@db/neo4j'
import type { DeleteImageOpts, MergeImageOpts } from './images'
// eslint-disable-next-line @typescript-eslint/no-explicit-any
type AsyncFunc = (...args: any[]) => Promise<any>
export const wrapTransaction = async <F extends AsyncFunc>(
wrappedCallback: F,
args: unknown[],
opts: DeleteImageOpts | MergeImageOpts,
) => {
const session = getDriver().session()
try {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
const result = await session.writeTransaction((transaction) => {
return wrappedCallback(...args, { ...opts, transaction })
})
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return result
} finally {
await session.close()
}
}

View File

@ -17,7 +17,7 @@ import { filterForMutedUsers } from './helpers/filterForMutedUsers'
import { filterInvisiblePosts } from './helpers/filterInvisiblePosts' import { filterInvisiblePosts } from './helpers/filterInvisiblePosts'
import { filterPostsOfMyGroups } from './helpers/filterPostsOfMyGroups' import { filterPostsOfMyGroups } from './helpers/filterPostsOfMyGroups'
import Resolver from './helpers/Resolver' import Resolver from './helpers/Resolver'
import { mergeImage, deleteImage } from './images/images' import { images } from './images/images'
import { createOrUpdateLocations } from './users/location' import { createOrUpdateLocations } from './users/location'
const maintainPinnedPosts = (params) => { const maintainPinnedPosts = (params) => {
@ -176,7 +176,7 @@ export default {
) )
const [post] = createPostTransactionResponse.records.map((record) => record.get('post')) const [post] = createPostTransactionResponse.records.map((record) => record.get('post'))
if (imageInput) { if (imageInput) {
await mergeImage(post, 'HERO_IMAGE', imageInput, { transaction }) await images.mergeImage(post, 'HERO_IMAGE', imageInput, { transaction })
} }
return post return post
}) })
@ -247,7 +247,7 @@ export default {
updatePostVariables, updatePostVariables,
) )
const [post] = updatePostTransactionResponse.records.map((record) => record.get('post')) const [post] = updatePostTransactionResponse.records.map((record) => record.get('post'))
await mergeImage(post, 'HERO_IMAGE', imageInput, { transaction }) await images.mergeImage(post, 'HERO_IMAGE', imageInput, { transaction })
return post return post
}) })
const post = await writeTxResultPromise const post = await writeTxResultPromise
@ -277,7 +277,7 @@ export default {
{ postId: args.id }, { postId: args.id },
) )
const [post] = deletePostTransactionResponse.records.map((record) => record.get('post')) const [post] = deletePostTransactionResponse.records.map((record) => record.get('post'))
await deleteImage(post, 'HERO_IMAGE', { transaction }) await images.deleteImage(post, 'HERO_IMAGE', { transaction })
return post return post
}) })
try { try {

View File

@ -15,7 +15,7 @@ import { Context } from '@src/server'
import { defaultTrophyBadge, defaultVerificationBadge } from './badges' import { defaultTrophyBadge, defaultVerificationBadge } from './badges'
import normalizeEmail from './helpers/normalizeEmail' import normalizeEmail from './helpers/normalizeEmail'
import Resolver from './helpers/Resolver' import Resolver from './helpers/Resolver'
import { mergeImage, deleteImage } from './images/images' import { images } from './images/images'
import { createOrUpdateLocations } from './users/location' import { createOrUpdateLocations } from './users/location'
const neode = getNeode() const neode = getNeode()
@ -210,7 +210,7 @@ export default {
) )
const [user] = updateUserTransactionResponse.records.map((record) => record.get('user')) const [user] = updateUserTransactionResponse.records.map((record) => record.get('user'))
if (avatarInput) { if (avatarInput) {
await mergeImage(user, 'AVATAR_IMAGE', avatarInput, { transaction }) await images.mergeImage(user, 'AVATAR_IMAGE', avatarInput, { transaction })
} }
return user return user
}) })
@ -253,7 +253,7 @@ export default {
return Promise.all( return Promise.all(
txResult.records txResult.records
.map((record) => record.get('resource')) .map((record) => record.get('resource'))
.map((resource) => deleteImage(resource, 'HERO_IMAGE', { transaction })), .map((resource) => images.deleteImage(resource, 'HERO_IMAGE', { transaction })),
) )
}), }),
) )
@ -281,7 +281,7 @@ export default {
{ userId }, { userId },
) )
const [user] = deleteUserTransactionResponse.records.map((record) => record.get('user')) const [user] = deleteUserTransactionResponse.records.map((record) => record.get('user'))
await deleteImage(user, 'AVATAR_IMAGE', { transaction }) await images.deleteImage(user, 'AVATAR_IMAGE', { transaction })
return user return user
}) })
try { try {

View File

@ -25,6 +25,9 @@ spec:
name: {{ .Release.Name }}-backend-env name: {{ .Release.Name }}-backend-env
- secretRef: - secretRef:
name: {{ .Release.Name }}-backend-secret-env name: {{ .Release.Name }}-backend-secret-env
volumeMounts:
- mountPath: /app/public/uploads
name: uploads
containers: containers:
- name: {{ .Release.Name }}-backend - name: {{ .Release.Name }}-backend
image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag | default (include "defaultTag" .) }}" image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag | default (include "defaultTag" .) }}"

View File

@ -23,15 +23,16 @@ secrets:
NEO4J_USERNAME: null NEO4J_USERNAME: null
NEO4J_PASSWORD: null NEO4J_PASSWORD: null
REDIS_PASSWORD: null REDIS_PASSWORD: null
AWS_ACCESS_KEY_ID: ENC[AES256_GCM,data:iiN5ueqyo60VHb9e2bnhc19iGTg=,iv:zawYpKrFafgsu1+YRet1hzZf1G3a6BIlZgsh7xNADaE=,tag:rTsmm8cqei34b6cT6vn08w==,type:str]
AWS_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:Zl4LRXdDh/6Q8F9RVp+3L7NXGZ0F2cgFMKPhl/TVeuD5Bhy68W5ekg==,iv:AmPoinGISrSOZdoBKdeFFXfr2hwOK4nWMnniz8K5qgU=,tag:K8Q7M7e+6G9T0Oh3Sp4OzA==,type:str]
AWS_ENDPOINT: ENC[AES256_GCM,data:/waEqUgcOmldZ+peFTNVsDQf2KrpWY8ZZMt1nT5117SkbY4=,iv:n+Kvidjb/TM4bQYKqTaFxt8GkHo02PuxEGpzgOcywr4=,tag:lrGPgCWWy3GMIcTv75IYTg==,type:str]
AWS_REGION: ENC[AES256_GCM,data:kBPpHZ8zw4PMpg==,iv:R+QZe303do37Hd/97NpS1pt9VaBE/gqZDY2/qlIvvps=,tag:0WduW8wfJXtBqlh4qfRGNA==,type:str]
AWS_BUCKET: ENC[AES256_GCM,data:0fAspN/PoRVPlSbz+qDBRUOieeC4,iv:JGJ/LyLpMymN0tpZmW6DjPT3xqXzK/KhYQsy9sgPd60=,tag:Y6PBs0916JkHRHSe7hqSMA==,type:str]
neo4j: neo4j:
env: env:
NEO4J_USERNAME: "" NEO4J_USERNAME: ""
NEO4J_PASSWORD: "" NEO4J_PASSWORD: ""
sops: sops:
kms: []
gcp_kms: []
azure_kv: []
hc_vault: []
age: age:
- recipient: age1llp6k66265q3rzqemxpnq0x3562u20989vcjf65fl9s3hjhgcscq6mhnjw - recipient: age1llp6k66265q3rzqemxpnq0x3562u20989vcjf65fl9s3hjhgcscq6mhnjw
enc: | enc: |
@ -69,8 +70,7 @@ sops:
aGNFeXZZRmlJM041OHdTM0pmM3BBdGMKGvFgYY1jhKwciAOZKyw0hlFVNbOk7CM7 aGNFeXZZRmlJM041OHdTM0pmM3BBdGMKGvFgYY1jhKwciAOZKyw0hlFVNbOk7CM7
041g17JXNV1Wk6WgMZ4w8p54RKQVaWCT4wxChy6wNNdQ3IeKgqEU2w== 041g17JXNV1Wk6WgMZ4w8p54RKQVaWCT4wxChy6wNNdQ3IeKgqEU2w==
-----END AGE ENCRYPTED FILE----- -----END AGE ENCRYPTED FILE-----
lastmodified: "2024-10-29T14:26:49Z" lastmodified: "2025-05-29T06:57:01Z"
mac: ENC[AES256_GCM,data:YXX7MEAK0wmuxLTmdr7q5uVd6DG6FhGUeE+EzbhWe/OovH6n+CjKZGklnEX+5ztDO0IgZh/T9Hx1CgFYuVbcOkvDoFBDwNpRA/QOQrM0p/+tRlMNCypC/Wh2xL0DhA4A/Qum2oyE/BDkt1Yy8N5wZDZn575+ZAjXEgAzlhpT5qk=,iv:ire3gkHTY6+0lgbV1Es6Lf8bcKTg4WKnq46M+b/VRcU=,tag:MkZULKcwROvIw/C0YtcUbA==,type:str] mac: ENC[AES256_GCM,data:0eWUqVsJrolrVYFsG4aigAYSjBW35Z+64y/ZE8Az15GmI0E4p/1kZPIY6hv9SUiCD1R2Ro0nm1QsV9A/hvNeWla0EdARNnARxIphxMJWKRGwcVkFVk28Jh4g4jE/rTggWx/wLbR6bza1RLHA1wRMb1PYuLfZybsb/whN4+gTMfg=,iv:irQkLpj1+3egSsiV9HqZP4tgG1fotCOizubL42gRjSQ=,tag:DC0xzG/VqcL4ib6ijxQZnA==,type:str]
pgp: []
unencrypted_suffix: _unencrypted unencrypted_suffix: _unencrypted
version: 3.9.0 version: 3.10.2

View File

@ -25,6 +25,9 @@ services:
backend: backend:
image: ghcr.io/ocelot-social-community/ocelot-social/backend:local-development image: ghcr.io/ocelot-social-community/ocelot-social/backend:local-development
depends_on:
- minio
- minio-mc
build: build:
target: development target: development
environment: environment:
@ -32,6 +35,12 @@ services:
- DEBUG=true - DEBUG=true
- SMTP_PORT=1025 - SMTP_PORT=1025
- SMTP_HOST=mailserver - SMTP_HOST=mailserver
- AWS_ACCESS_KEY_ID=minio
- AWS_SECRET_ACCESS_KEY=12341234
- AWS_ENDPOINT=http:/minio:9000
- AWS_REGION=local
- AWS_BUCKET=ocelot
- S3_PUBLIC_GATEWAY=http:/localhost:9000
volumes: volumes:
- ./backend:/app - ./backend:/app
@ -46,3 +55,33 @@ services:
ports: ports:
- 1080:1080 - 1080:1080
- 1025:1025 - 1025:1025
minio:
image: quay.io/minio/minio
ports:
- 9000:9000
- 9001:9001
volumes:
- minio_data:/data
environment:
- MINIO_ROOT_USER=minio
- MINIO_ROOT_PASSWORD=12341234
command: server /data --console-address ":9001"
minio-mc:
image: quay.io/minio/mc
depends_on:
- minio
restart: on-failure
volumes:
- ./minio/readonly-policy.json:/tmp/readonly-policy.json
entrypoint: >
/bin/sh -c "
sleep 5;
/usr/bin/mc alias set dockerminio http://minio:9000 minio 12341234;
/usr/bin/mc mb --ignore-existing dockerminio/ocelot;
/usr/bin/mc anonymous set-json /tmp/readonly-policy.json dockerminio/ocelot;
"
volumes:
minio_data:

View File

@ -13,10 +13,19 @@ services:
backend: backend:
# name the image so that it cannot be found in a DockerHub repository, otherwise it will not be built locally from the 'dockerfile' but pulled from there # name the image so that it cannot be found in a DockerHub repository, otherwise it will not be built locally from the 'dockerfile' but pulled from there
image: ghcr.io/ocelot-social-community/ocelot-social/backend:test image: ghcr.io/ocelot-social-community/ocelot-social/backend:test
depends_on:
- minio
- minio-mc
- mailserver
build: build:
target: test target: test
environment: environment:
- NODE_ENV="test" - NODE_ENV="test"
- AWS_ACCESS_KEY_ID=minio
- AWS_SECRET_ACCESS_KEY=12341234
- AWS_ENDPOINT=http:/minio:9000
- AWS_REGION=local
- AWS_BUCKET=ocelot
volumes: volumes:
- ./coverage:/app/coverage - ./coverage:/app/coverage
@ -41,3 +50,33 @@ services:
ports: ports:
- 1080:1080 - 1080:1080
- 1025:1025 - 1025:1025
minio:
image: quay.io/minio/minio
ports:
- 9000:9000
- 9001:9001
volumes:
- minio_data:/data
environment:
- MINIO_ROOT_USER=minio
- MINIO_ROOT_PASSWORD=12341234
command: server /data --console-address ":9001"
minio-mc:
image: quay.io/minio/mc
depends_on:
- minio
restart: on-failure
volumes:
- ./minio/readonly-policy.json:/tmp/readonly-policy.json
entrypoint: >
/bin/sh -c "
sleep 5;
/usr/bin/mc alias set dockerminio http://minio:9000 minio 12341234;
/usr/bin/mc mb --ignore-existing dockerminio/ocelot;
/usr/bin/mc anonymous set-json /tmp/readonly-policy.json dockerminio/ocelot;
"
volumes:
minio_data:

View File

@ -53,8 +53,6 @@ services:
- neo4j - neo4j
ports: ports:
- 4000:4000 - 4000:4000
volumes:
- backend_uploads:/app/public/uploads
environment: environment:
# Envs used in Dockerfile # Envs used in Dockerfile
# - DOCKER_WORKDIR="/app" # - DOCKER_WORKDIR="/app"
@ -104,5 +102,4 @@ services:
# command: ["tail", "-f", "/dev/null"] # command: ["tail", "-f", "/dev/null"]
volumes: volumes:
backend_uploads:
neo4j_data: neo4j_data:

View File

@ -0,0 +1,19 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"AWS": [
"*"
]
},
"Action": [
"s3:GetObject"
],
"Resource": [
"arn:aws:s3:::ocelot/*"
]
}
]
}