diff --git a/backend/.env.template b/backend/.env.template index b4c91da9a..8531e6cd7 100644 --- a/backend/.env.template +++ b/backend/.env.template @@ -17,3 +17,9 @@ PRIVATE_KEY_PASSPHRASE="a7dsf78sadg87ad87sfagsadg78" SENTRY_DSN_BACKEND= COMMIT= PUBLIC_REGISTRATION=false + +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_ENDPOINT= +AWS_REGION= +AWS_BUCKET= diff --git a/backend/package.json b/backend/package.json index f6aeb8fcb..8ff921155 100644 --- a/backend/package.json +++ b/backend/package.json @@ -45,6 +45,7 @@ "apollo-link-http": "~1.5.16", "apollo-server": "~2.11.0", "apollo-server-express": "^2.11.0", + "aws-sdk": "^2.638.0", "babel-plugin-transform-runtime": "^6.23.0", "bcryptjs": "~2.4.3", "cheerio": "~1.0.0-rc.3", @@ -86,6 +87,7 @@ "metascraper-video": "^5.11.6", "metascraper-youtube": "^5.11.6", "migrate": "^1.6.2", + "mime-types": "^2.1.26", "minimatch": "^3.0.4", "mustache": "^4.0.1", "neo4j-driver": "^4.0.2", diff --git a/backend/src/config/index.js b/backend/src/config/index.js index b7ea99e9f..4c81bb181 100644 --- a/backend/src/config/index.js +++ b/backend/src/config/index.js @@ -18,6 +18,11 @@ const { SMTP_PASSWORD, SENTRY_DSN_BACKEND, COMMIT, + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + AWS_ENDPOINT, + AWS_REGION, + AWS_BUCKET, NEO4J_URI = 'bolt://localhost:7687', NEO4J_USERNAME = 'neo4j', NEO4J_PASSWORD = 'neo4j', @@ -64,7 +69,20 @@ export const developmentConfigs = { } export const sentryConfigs = { SENTRY_DSN_BACKEND, COMMIT } -export const redisConfiig = { REDIS_DOMAIN, REDIS_PORT, REDIS_PASSWORD } +export const redisConfigs = { REDIS_DOMAIN, REDIS_PORT, REDIS_PASSWORD } + +const S3_CONFIGURED = + AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY && AWS_ENDPOINT && AWS_REGION && AWS_BUCKET + +export const s3Configs = { + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + AWS_ENDPOINT, + AWS_REGION, + AWS_BUCKET, + S3_CONFIGURED, +} + export default { ...requiredConfigs, ...smtpConfigs, @@ -72,5 +90,6 @@ export default { ...serverConfigs, ...developmentConfigs, ...sentryConfigs, - ...redisConfiig, + ...redisConfigs, + ...s3Configs, } diff --git a/backend/src/db/migrations/20200312140328-bulk_upload_to_s3.js b/backend/src/db/migrations/20200312140328-bulk_upload_to_s3.js new file mode 100644 index 000000000..908c97b42 --- /dev/null +++ b/backend/src/db/migrations/20200312140328-bulk_upload_to_s3.js @@ -0,0 +1,103 @@ +import { getDriver } from '../../db/neo4j' +import { existsSync, createReadStream } from 'fs' +import path from 'path' +import { S3 } from 'aws-sdk' +import mime from 'mime-types' +import { s3Configs } from '../../config' + +export const description = ` +Upload all image files to a S3 compatible object storage in order to reduce +load on our backend. +` + +export async function up(next) { + const driver = getDriver() + const session = driver.session() + const transaction = session.beginTransaction() + + const { + AWS_ENDPOINT: endpoint, + AWS_REGION: region, + AWS_BUCKET: Bucket, + S3_CONFIGURED, + } = s3Configs + + if (!S3_CONFIGURED) { + // eslint-disable-next-line no-console + console.log('No S3 given, cannot upload image files') + return + } + + const s3 = new S3({ region, endpoint }) + try { + // Implement your migration here. + const { records } = await transaction.run('MATCH (image:Image) RETURN image.url as url') + let urls = records.map((r) => r.get('url')) + urls = urls.filter((url) => url.startsWith('/uploads')) + const locations = await Promise.all( + urls + .map((url) => { + return async () => { + const { pathname } = new URL(url, 'http://example.org') + const fileLocation = path.join(__dirname, `../../../public/${pathname}`) + const s3Location = `original${pathname}` + if (existsSync(fileLocation)) { + const mimeType = mime.lookup(fileLocation) + const params = { + Bucket, + Key: s3Location, + ACL: 'public-read', + ContentType: mimeType || 'image/jpeg', + Body: createReadStream(fileLocation), + } + + const data = await s3.upload(params).promise() + const { Location: spacesUrl } = data + + const updatedRecord = await transaction.run( + 'MATCH (image:Image {url: $url}) SET image.url = $spacesUrl RETURN image.url as url', + { url, spacesUrl }, + ) + const [updatedUrl] = updatedRecord.records.map((record) => record.get('url')) + return updatedUrl + } + } + }) + .map((p) => p()), + ) + // eslint-disable-next-line no-console + console.log('this is locations', locations) + await transaction.commit() + next() + } catch (error) { + // eslint-disable-next-line no-console + console.log(error) + await transaction.rollback() + // eslint-disable-next-line no-console + console.log('rolled back') + throw new Error(error) + } finally { + session.close() + } +} + +export async function down(next) { + const driver = getDriver() + const session = driver.session() + const transaction = session.beginTransaction() + + try { + // Implement your migration here. + await transaction.run(``) + await transaction.commit() + next() + } catch (error) { + // eslint-disable-next-line no-console + console.log(error) + await transaction.rollback() + // eslint-disable-next-line no-console + console.log('rolled back') + } finally { + session.close() + } +} diff --git a/backend/src/schema/resolvers/images/images.js b/backend/src/schema/resolvers/images/images.js index ad14c4833..18a3569b6 100644 --- a/backend/src/schema/resolvers/images/images.js +++ b/backend/src/schema/resolvers/images/images.js @@ -1,11 +1,14 @@ import path from 'path' import { v4 as uuid } from 'uuid' +import { S3 } from 'aws-sdk' import slug from 'slug' import { existsSync, unlinkSync, createWriteStream } from 'fs' -import { getDriver } from '../../../db/neo4j' import { UserInputError } from 'apollo-server' +import { getDriver } from '../../../db/neo4j' +import { s3Configs } from '../../../config' // const widths = [34, 160, 320, 640, 1024] +const { AWS_ENDPOINT: endpoint, AWS_REGION: region, AWS_BUCKET: Bucket, S3_CONFIGURED } = s3Configs export async function deleteImage(resource, relationshipType, opts = {}) { sanitizeRelationshipType(relationshipType) @@ -79,23 +82,24 @@ const wrapTransaction = async (wrappedCallback, args, opts) => { } } -const deleteImageFile = (image, deleteCallback = localFileDelete) => { +const deleteImageFile = (image, deleteCallback) => { + if (!deleteCallback) { + deleteCallback = S3_CONFIGURED ? s3Delete : localFileDelete + } const { url } = image deleteCallback(url) return url } -const uploadImageFile = async (upload, uploadCallback = localFileUpload) => { +const uploadImageFile = async (upload, uploadCallback) => { if (!upload) return undefined + if (!uploadCallback) { + uploadCallback = S3_CONFIGURED ? s3Upload : localFileUpload + } const { createReadStream, filename, mimetype } = await upload const { name, ext } = path.parse(filename) const uniqueFilename = `${uuid()}-${slug(name)}${ext}` - - return uploadCallback({ - createReadStream, - destination: `/uploads/${uniqueFilename}`, - mimetype, - }) + return uploadCallback({ createReadStream, uniqueFilename, mimetype }) } const sanitizeRelationshipType = (relationshipType) => { @@ -106,7 +110,8 @@ const sanitizeRelationshipType = (relationshipType) => { } } -const localFileUpload = ({ createReadStream, destination }) => { +const localFileUpload = ({ createReadStream, uniqueFilename }) => { + const destination = `/uploads/${uniqueFilename}` return new Promise((resolve, reject) => createReadStream() .pipe(createWriteStream(`public${destination}`)) @@ -115,7 +120,34 @@ const localFileUpload = ({ createReadStream, destination }) => { ) } +const s3Upload = async ({ createReadStream, uniqueFilename, mimetype }) => { + const s3 = new S3({ region, endpoint }) + const s3Location = `original/${uniqueFilename}` + + const params = { + Bucket, + Key: s3Location, + ACL: 'public-read', + ContentType: mimetype, + Body: createReadStream(), + } + const data = await s3.upload(params).promise() + const { Location } = data + return Location +} + const localFileDelete = async (url) => { const location = `public${url}` if (existsSync(location)) unlinkSync(location) } + +const s3Delete = async (url) => { + const s3 = new S3({ region, endpoint }) + let { pathname } = new URL(url, 'http://example.org') // dummy domain to avoid invalid URL error + pathname = pathname.substring(1) // remove first character '/' + const params = { + Bucket, + Key: pathname, + } + await s3.deleteObject(params).promise() +} diff --git a/backend/src/schema/resolvers/images/images.spec.js b/backend/src/schema/resolvers/images/images.spec.js index 4dc8449b7..42064621c 100644 --- a/backend/src/schema/resolvers/images/images.spec.js +++ b/backend/src/schema/resolvers/images/images.spec.js @@ -11,7 +11,7 @@ let deleteCallback beforeEach(async () => { await cleanDatabase() - uploadCallback = jest.fn(({ destination }) => destination) + uploadCallback = jest.fn(({ uniqueFilename }) => `/uploads/${uniqueFilename}`) deleteCallback = jest.fn() }) @@ -99,34 +99,34 @@ describe('mergeImage', () => { } }) - describe('on existing resource', () => { - beforeEach(async () => { - post = await Factory.build( - 'post', - { id: 'p99' }, - { - author: Factory.build('user', {}, { avatar: null }), - image: null, + describe('given image.upload', () => { + beforeEach(() => { + imageInput = { + ...imageInput, + upload: { + filename: 'image.jpg', + mimetype: 'image/jpeg', + encoding: '7bit', + createReadStream: () => ({ + pipe: () => ({ + on: (_, callback) => callback(), + }), + }), }, - ) - post = await post.toJson() + } }) - describe('given image.upload', () => { - beforeEach(() => { - imageInput = { - ...imageInput, - upload: { - filename: 'image.jpg', - mimetype: 'image/jpeg', - encoding: '7bit', - createReadStream: () => ({ - pipe: () => ({ - on: (_, callback) => callback(), - }), - }), + describe('on existing resource', () => { + beforeEach(async () => { + post = await Factory.build( + 'post', + { id: 'p99' }, + { + author: Factory.build('user', {}, { avatar: null }), + image: null, }, - } + ) + post = await post.toJson() }) it('returns new image', async () => { @@ -330,7 +330,7 @@ describe('mergeImage', () => { }) it('updates metadata', async () => { - await mergeImage(post, 'HERO_IMAGE', imageInput) + await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }) const images = await neode.all('Image') expect(images).toHaveLength(1) await expect(images.first().toJson()).resolves.toMatchObject({ diff --git a/backend/yarn.lock b/backend/yarn.lock index 0123145d3..326000992 100644 --- a/backend/yarn.lock +++ b/backend/yarn.lock @@ -2220,6 +2220,21 @@ audio-extensions@0.0.0: resolved "https://registry.yarnpkg.com/audio-extensions/-/audio-extensions-0.0.0.tgz#d0eefe077fb9eb625898eed9985890548cf1f8d2" integrity sha1-0O7+B3+562JYmO7ZmFiQVIzx+NI= +aws-sdk@^2.638.0: + version "2.638.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.638.0.tgz#43df5a956696177c577b841ea21b4007a81dbdaa" + integrity sha512-DOSwedH2YkPVs3c2AQezK6FHuGRIDffgULGvmpY9ZmZ/x45Sw+p7WHCYPgWfw/Z1fJWzMjaIpu531xG7pyJV4A== + dependencies: + buffer "4.9.1" + events "1.1.1" + ieee754 "1.1.13" + jmespath "0.15.0" + querystring "0.2.0" + sax "1.2.1" + url "0.10.3" + uuid "3.3.2" + xml2js "0.4.19" + aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" @@ -2319,6 +2334,11 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= +base64-js@^1.0.2: + version "1.3.1" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1" + integrity sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g== + base@^0.11.1: version "0.11.2" resolved "https://registry.yarnpkg.com/base/-/base-0.11.2.tgz#7bde5ced145b6d551a90db87f83c558b4eb48a8f" @@ -2472,6 +2492,15 @@ buffer-from@^1.0.0: resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== +buffer@4.9.1: + version "4.9.1" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298" + integrity sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg= + dependencies: + base64-js "^1.0.2" + ieee754 "^1.1.4" + isarray "^1.0.0" + busboy@^0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/busboy/-/busboy-0.3.1.tgz#170899274c5bf38aae27d5c62b71268cd585fd1b" @@ -3821,6 +3850,11 @@ eventemitter3@^3.1.0: resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-3.1.2.tgz#2d3d48f9c346698fce83a85d7d664e98535df6e7" integrity sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q== +events@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" + integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ= + exec-sh@^0.3.2: version "0.3.2" resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.2.tgz#6738de2eb7c8e671d0366aea0b0db8c6f7d7391b" @@ -4832,6 +4866,11 @@ iconv-lite@0.4.24, iconv-lite@^0.4.24, iconv-lite@^0.4.4: dependencies: safer-buffer ">= 2.1.2 < 3" +ieee754@1.1.13, ieee754@^1.1.4: + version "1.1.13" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" + integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== + ienoopen@1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/ienoopen/-/ienoopen-1.1.0.tgz#411e5d530c982287dbdc3bb31e7a9c9e32630974" @@ -5741,6 +5780,11 @@ jest@~25.2.0: import-local "^3.0.2" jest-cli "^25.2.0" +jmespath@0.15.0: + version "0.15.0" + resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" + integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc= + jquery@^3.3.1: version "3.4.1" resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.4.1.tgz#714f1f8d9dde4bdfa55764ba37ef214630d80ef2" @@ -6406,7 +6450,7 @@ mime-db@1.43.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.43.0.tgz#0a12e0502650e473d735535050e7c8f4eb4fae58" integrity sha512-+5dsGEEovYbT8UY9yD7eE4XTc4UwJ1jBYlgaQQF38ENsKR3wj/8q8RFZrF9WIZpB2V1ArTVFUva8sAul1NzRzQ== -mime-types@^2.1.12, mime-types@~2.1.19, mime-types@~2.1.22, mime-types@~2.1.24, mime-types@~2.1.26: +mime-types@^2.1.12, mime-types@^2.1.26, mime-types@~2.1.19, mime-types@~2.1.22, mime-types@~2.1.24, mime-types@~2.1.26: version "2.1.26" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.26.tgz#9c921fc09b7e149a65dfdc0da4d20997200b0a06" integrity sha512-01paPWYgLrkqAyrlDorC1uDwl2p3qZT7yl806vW7DvDoxwXi46jsjFbg+WdwotBIk6/MbEhO/dh5aZ5sNj/dWQ== @@ -7420,6 +7464,11 @@ punycode2@~1.0.0: resolved "https://registry.yarnpkg.com/punycode2/-/punycode2-1.0.0.tgz#e2b4b9a9a8ff157d0b84438e203181ee7892dfd8" integrity sha1-4rS5qaj/FX0LhEOOIDGB7niS39g= +punycode@1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" + integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0= + punycode@^2.1.0, punycode@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" @@ -7440,6 +7489,11 @@ qs@~6.5.2: resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA== +querystring@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" + integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= + querystringify@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.1.1.tgz#60e5a5fd64a7f8bfa4d2ab2ed6fdf4c85bad154e" @@ -7928,6 +7982,11 @@ sanitize-html@~1.22.0: srcset "^2.0.1" xtend "^4.0.1" +sax@1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" + integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o= + sax@>=0.6.0, sax@^1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -9048,6 +9107,14 @@ url-regex@~5.0.0: ip-regex "^4.1.0" tlds "^1.203.0" +url@0.10.3: + version "0.10.3" + resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" + integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ= + dependencies: + punycode "1.3.2" + querystring "0.2.0" + use@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f" @@ -9083,6 +9150,11 @@ utils-merge@1.0.1: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= +uuid@3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" + integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== + uuid@^3.1.0, uuid@^3.3.2: version "3.4.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" @@ -9327,7 +9399,7 @@ xml-name-validator@^3.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== -xml2js@^0.4.17: +xml2js@0.4.19, xml2js@^0.4.17: version "0.4.19" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==