diff --git a/backend/.env.template b/backend/.env.template index b4c91da9a..dab8bee72 100644 --- a/backend/.env.template +++ b/backend/.env.template @@ -17,3 +17,8 @@ PRIVATE_KEY_PASSPHRASE="a7dsf78sadg87ad87sfagsadg78" SENTRY_DSN_BACKEND= COMMIT= PUBLIC_REGISTRATION=false + +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_ENDPOINT= +AWS_REGION= diff --git a/backend/package.json b/backend/package.json index baf03a56c..d63705903 100644 --- a/backend/package.json +++ b/backend/package.json @@ -45,6 +45,7 @@ "apollo-link-http": "~1.5.16", "apollo-server": "~2.11.0", "apollo-server-express": "^2.11.0", + "aws-sdk": "^2.638.0", "babel-plugin-transform-runtime": "^6.23.0", "bcryptjs": "~2.4.3", "cheerio": "~1.0.0-rc.3", diff --git a/backend/src/config/index.js b/backend/src/config/index.js index b7ea99e9f..10332e798 100644 --- a/backend/src/config/index.js +++ b/backend/src/config/index.js @@ -18,6 +18,11 @@ const { SMTP_PASSWORD, SENTRY_DSN_BACKEND, COMMIT, + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + AWS_ENDPOINT, + AWS_REGION, + AWS_BUCKET = 'image-upload', NEO4J_URI = 'bolt://localhost:7687', NEO4J_USERNAME = 'neo4j', NEO4J_PASSWORD = 'neo4j', @@ -64,7 +69,16 @@ export const developmentConfigs = { } export const sentryConfigs = { SENTRY_DSN_BACKEND, COMMIT } -export const redisConfiig = { REDIS_DOMAIN, REDIS_PORT, REDIS_PASSWORD } +export const redisConfigs = { REDIS_DOMAIN, REDIS_PORT, REDIS_PASSWORD } + +export const s3Configs = { + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + AWS_ENDPOINT, + AWS_REGION, + AWS_BUCKET, +} + export default { ...requiredConfigs, ...smtpConfigs, @@ -72,5 +86,6 @@ export default { ...serverConfigs, ...developmentConfigs, ...sentryConfigs, - ...redisConfiig, + ...redisConfigs, + ...s3Configs, } diff --git a/backend/src/db/migrations/20200312140328-bulk_upload_to_s3.js b/backend/src/db/migrations/20200312140328-bulk_upload_to_s3.js new file mode 100644 index 000000000..97a9c01d0 --- /dev/null +++ b/backend/src/db/migrations/20200312140328-bulk_upload_to_s3.js @@ -0,0 +1,97 @@ +import { getDriver } from '../../db/neo4j' +import { createReadStream } from 'fs' +import path from 'path' +import { S3 } from 'aws-sdk' +import { s3Configs } from '../../config' + +export const description = ` +Upload all image files to a S3 compatible object storage in order to reduce +load on our backend. +` + +export async function up(next) { + const driver = getDriver() + const session = driver.session() + const transaction = session.beginTransaction() + + const { + AWS_ACCESS_KEY_ID: accessKeyId, + AWS_SECRET_ACCESS_KEY: secretAccessKey, + AWS_ENDPOINT: endpoint, + AWS_REGION: region, + AWS_BUCKET: Bucket, + } = s3Configs + + if (!(accessKeyId || secretAccessKey)) { + // eslint-disable-next-line no-console + console.log('No S3 given, cannot upload image files') + return + } + + const s3 = new S3({ region, endpoint }) + + try { + // Implement your migration here. + const { records } = await transaction.run('MATCH (image:Image) RETURN image.url as url') + let urls = records.map(r => r.get('url')) + urls = urls.filter(url => url.startsWith('/uploads')) + const locations = await Promise.all( + urls + .map(url => { + return async () => { + const { pathname } = new URL(url, 'http://example.org') + const fileLocation = path.join(__dirname, `../../../public/${pathname}`) + const s3Location = `/original/${pathname}` + + const params = { + Bucket, + Key: s3Location, + ACL: 'public-read', + // TODO: check the actual mime type + ContentType: 'image/jpg', + Body: createReadStream(fileLocation), + } + const data = await s3.upload(params).promise() + const { Location } = data + return Location + } + }) + .map(p => p()), + ) + // TODO: update the urls in the database + // eslint-disable-next-line no-console + console.log(locations) + await transaction.commit() + next() + } catch (error) { + // eslint-disable-next-line no-console + console.log(error) + await transaction.rollback() + // eslint-disable-next-line no-console + console.log('rolled back') + throw new Error(error) + } finally { + session.close() + } +} + +export async function down(next) { + const driver = getDriver() + const session = driver.session() + const transaction = session.beginTransaction() + + try { + // Implement your migration here. + await transaction.run(``) + await transaction.commit() + next() + } catch (error) { + // eslint-disable-next-line no-console + console.log(error) + await transaction.rollback() + // eslint-disable-next-line no-console + console.log('rolled back') + } finally { + session.close() + } +} diff --git a/backend/yarn.lock b/backend/yarn.lock index e3d2c0da1..9e60cd776 100644 --- a/backend/yarn.lock +++ b/backend/yarn.lock @@ -2210,6 +2210,21 @@ audio-extensions@0.0.0: resolved "https://registry.yarnpkg.com/audio-extensions/-/audio-extensions-0.0.0.tgz#d0eefe077fb9eb625898eed9985890548cf1f8d2" integrity sha1-0O7+B3+562JYmO7ZmFiQVIzx+NI= +aws-sdk@^2.638.0: + version "2.638.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.638.0.tgz#43df5a956696177c577b841ea21b4007a81dbdaa" + integrity sha512-DOSwedH2YkPVs3c2AQezK6FHuGRIDffgULGvmpY9ZmZ/x45Sw+p7WHCYPgWfw/Z1fJWzMjaIpu531xG7pyJV4A== + dependencies: + buffer "4.9.1" + events "1.1.1" + ieee754 "1.1.13" + jmespath "0.15.0" + querystring "0.2.0" + sax "1.2.1" + url "0.10.3" + uuid "3.3.2" + xml2js "0.4.19" + aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" @@ -2309,6 +2324,11 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= +base64-js@^1.0.2: + version "1.3.1" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1" + integrity sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g== + base@^0.11.1: version "0.11.2" resolved "https://registry.yarnpkg.com/base/-/base-0.11.2.tgz#7bde5ced145b6d551a90db87f83c558b4eb48a8f" @@ -2462,6 +2482,15 @@ buffer-from@^1.0.0: resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== +buffer@4.9.1: + version "4.9.1" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298" + integrity sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg= + dependencies: + base64-js "^1.0.2" + ieee754 "^1.1.4" + isarray "^1.0.0" + busboy@^0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/busboy/-/busboy-0.3.1.tgz#170899274c5bf38aae27d5c62b71268cd585fd1b" @@ -3811,6 +3840,11 @@ eventemitter3@^3.1.0: resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-3.1.2.tgz#2d3d48f9c346698fce83a85d7d664e98535df6e7" integrity sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q== +events@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" + integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ= + exec-sh@^0.3.2: version "0.3.2" resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.2.tgz#6738de2eb7c8e671d0366aea0b0db8c6f7d7391b" @@ -4822,6 +4856,11 @@ iconv-lite@0.4.24, iconv-lite@^0.4.24, iconv-lite@^0.4.4: dependencies: safer-buffer ">= 2.1.2 < 3" +ieee754@1.1.13, ieee754@^1.1.4: + version "1.1.13" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" + integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== + ienoopen@1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/ienoopen/-/ienoopen-1.1.0.tgz#411e5d530c982287dbdc3bb31e7a9c9e32630974" @@ -5727,6 +5766,11 @@ jest@~25.1.0: import-local "^3.0.2" jest-cli "^25.1.0" +jmespath@0.15.0: + version "0.15.0" + resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" + integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc= + jquery@^3.3.1: version "3.4.1" resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.4.1.tgz#714f1f8d9dde4bdfa55764ba37ef214630d80ef2" @@ -7401,6 +7445,11 @@ punycode2@~1.0.0: resolved "https://registry.yarnpkg.com/punycode2/-/punycode2-1.0.0.tgz#e2b4b9a9a8ff157d0b84438e203181ee7892dfd8" integrity sha1-4rS5qaj/FX0LhEOOIDGB7niS39g= +punycode@1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" + integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0= + punycode@^2.1.0, punycode@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" @@ -7421,6 +7470,11 @@ qs@~6.5.2: resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA== +querystring@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" + integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= + querystringify@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.1.1.tgz#60e5a5fd64a7f8bfa4d2ab2ed6fdf4c85bad154e" @@ -7904,6 +7958,11 @@ sanitize-html@~1.22.0: srcset "^2.0.1" xtend "^4.0.1" +sax@1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" + integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o= + sax@>=0.6.0, sax@^1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -9029,6 +9088,14 @@ url-regex@~5.0.0: ip-regex "^4.1.0" tlds "^1.203.0" +url@0.10.3: + version "0.10.3" + resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" + integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ= + dependencies: + punycode "1.3.2" + querystring "0.2.0" + use@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f" @@ -9064,6 +9131,11 @@ utils-merge@1.0.1: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= +uuid@3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" + integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== + uuid@^3.1.0, uuid@^3.3.2: version "3.4.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" @@ -9308,7 +9380,7 @@ xml-name-validator@^3.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== -xml2js@^0.4.17: +xml2js@0.4.19, xml2js@^0.4.17: version "0.4.19" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==