Start migration to bulk upload images to S3

backend/.env.template

@@ -17,3 +17,8 @@ PRIVATE_KEY_PASSPHRASE="a7dsf78sadg87ad87sfagsadg78"
 SENTRY_DSN_BACKEND=
 COMMIT=
 PUBLIC_REGISTRATION=false
+
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_ENDPOINT=
+AWS_REGION=

backend/package.json

@@ -45,6 +45,7 @@
     "apollo-link-http": "~1.5.16",
     "apollo-server": "~2.11.0",
     "apollo-server-express": "^2.11.0",
+    "aws-sdk": "^2.638.0",
     "babel-plugin-transform-runtime": "^6.23.0",
     "bcryptjs": "~2.4.3",
     "cheerio": "~1.0.0-rc.3",

backend/src/config/index.js

@@ -18,6 +18,11 @@ const {
   SMTP_PASSWORD,
   SENTRY_DSN_BACKEND,
   COMMIT,
+  AWS_ACCESS_KEY_ID,
+  AWS_SECRET_ACCESS_KEY,
+  AWS_ENDPOINT,
+  AWS_REGION,
+  AWS_BUCKET = 'image-upload',
   NEO4J_URI = 'bolt://localhost:7687',
   NEO4J_USERNAME = 'neo4j',
   NEO4J_PASSWORD = 'neo4j',
@@ -64,7 +69,16 @@ export const developmentConfigs = {
 }
 
 export const sentryConfigs = { SENTRY_DSN_BACKEND, COMMIT }
-export const redisConfiig = { REDIS_DOMAIN, REDIS_PORT, REDIS_PASSWORD }
+export const redisConfigs = { REDIS_DOMAIN, REDIS_PORT, REDIS_PASSWORD }
+
+export const s3Configs = {
+  AWS_ACCESS_KEY_ID,
+  AWS_SECRET_ACCESS_KEY,
+  AWS_ENDPOINT,
+  AWS_REGION,
+  AWS_BUCKET,
+}
+
 export default {
   ...requiredConfigs,
   ...smtpConfigs,
@@ -72,5 +86,6 @@ export default {
   ...serverConfigs,
   ...developmentConfigs,
   ...sentryConfigs,
-  ...redisConfiig,
+  ...redisConfigs,
+  ...s3Configs,
 }

backend/src/db/migrations/20200312140328-bulk_upload_to_s3.js

@@ -0,0 +1,97 @@
+import { getDriver } from '../../db/neo4j'
+import { createReadStream } from 'fs'
+import path from 'path'
+import { S3 } from 'aws-sdk'
+import { s3Configs } from '../../config'
+
+export const description = `
+Upload all image files to a S3 compatible object storage in order to reduce
+load on our backend.
+`
+
+export async function up(next) {
+  const driver = getDriver()
+  const session = driver.session()
+  const transaction = session.beginTransaction()
+
+  const {
+    AWS_ACCESS_KEY_ID: accessKeyId,
+    AWS_SECRET_ACCESS_KEY: secretAccessKey,
+    AWS_ENDPOINT: endpoint,
+    AWS_REGION: region,
+    AWS_BUCKET: Bucket,
+  } = s3Configs
+
+  if (!(accessKeyId || secretAccessKey)) {
+    // eslint-disable-next-line no-console
+    console.log('No S3 given, cannot upload image files')
+    return
+  }
+
+  const s3 = new S3({ region, endpoint })
+
+  try {
+    // Implement your migration here.
+    const { records } = await transaction.run('MATCH (image:Image) RETURN image.url as url')
+    let urls = records.map(r => r.get('url'))
+    urls = urls.filter(url => url.startsWith('/uploads'))
+    const locations = await Promise.all(
+      urls
+        .map(url => {
+          return async () => {
+            const { pathname } = new URL(url, 'http://example.org')
+            const fileLocation = path.join(__dirname, `../../../public/${pathname}`)
+            const s3Location = `/original/${pathname}`
+
+            const params = {
+              Bucket,
+              Key: s3Location,
+              ACL: 'public-read',
+              // TODO: check the actual mime type
+              ContentType: 'image/jpg',
+              Body: createReadStream(fileLocation),
+            }
+            const data = await s3.upload(params).promise()
+            const { Location } = data
+            return Location
+          }
+        })
+        .map(p => p()),
+    )
+    // TODO: update the urls in the database
+    // eslint-disable-next-line no-console
+    console.log(locations)
+    await transaction.commit()
+    next()
+  } catch (error) {
+    // eslint-disable-next-line no-console
+    console.log(error)
+    await transaction.rollback()
+    // eslint-disable-next-line no-console
+    console.log('rolled back')
+    throw new Error(error)
+  } finally {
+    session.close()
+  }
+}
+
+export async function down(next) {
+  const driver = getDriver()
+  const session = driver.session()
+  const transaction = session.beginTransaction()
+
+  try {
+    // Implement your migration here.
+    await transaction.run(``)
+    await transaction.commit()
+    next()
+  } catch (error) {
+    // eslint-disable-next-line no-console
+    console.log(error)
+    await transaction.rollback()
+    // eslint-disable-next-line no-console
+    console.log('rolled back')
+  } finally {
+    session.close()
+  }
+}

backend/yarn.lock

@@ -2210,6 +2210,21 @@ audio-extensions@0.0.0:
   resolved "https://registry.yarnpkg.com/audio-extensions/-/audio-extensions-0.0.0.tgz#d0eefe077fb9eb625898eed9985890548cf1f8d2"
   integrity sha1-0O7+B3+562JYmO7ZmFiQVIzx+NI=
 
+aws-sdk@^2.638.0:
+  version "2.638.0"
+  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.638.0.tgz#43df5a956696177c577b841ea21b4007a81dbdaa"
+  integrity sha512-DOSwedH2YkPVs3c2AQezK6FHuGRIDffgULGvmpY9ZmZ/x45Sw+p7WHCYPgWfw/Z1fJWzMjaIpu531xG7pyJV4A==
+  dependencies:
+    buffer "4.9.1"
+    events "1.1.1"
+    ieee754 "1.1.13"
+    jmespath "0.15.0"
+    querystring "0.2.0"
+    sax "1.2.1"
+    url "0.10.3"
+    uuid "3.3.2"
+    xml2js "0.4.19"
+
 aws-sign2@~0.7.0:
   version "0.7.0"
   resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
@@ -2309,6 +2324,11 @@ balanced-match@^1.0.0:
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
   integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c=
 
+base64-js@^1.0.2:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1"
+  integrity sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==
+
 base@^0.11.1:
   version "0.11.2"
   resolved "https://registry.yarnpkg.com/base/-/base-0.11.2.tgz#7bde5ced145b6d551a90db87f83c558b4eb48a8f"
@@ -2462,6 +2482,15 @@ buffer-from@^1.0.0:
   resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef"
   integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==
 
+buffer@4.9.1:
+  version "4.9.1"
+  resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298"
+  integrity sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=
+  dependencies:
+    base64-js "^1.0.2"
+    ieee754 "^1.1.4"
+    isarray "^1.0.0"
+
 busboy@^0.3.1:
   version "0.3.1"
   resolved "https://registry.yarnpkg.com/busboy/-/busboy-0.3.1.tgz#170899274c5bf38aae27d5c62b71268cd585fd1b"
@@ -3811,6 +3840,11 @@ eventemitter3@^3.1.0:
   resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-3.1.2.tgz#2d3d48f9c346698fce83a85d7d664e98535df6e7"
   integrity sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==
 
+events@1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
+  integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=
+
 exec-sh@^0.3.2:
   version "0.3.2"
   resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.2.tgz#6738de2eb7c8e671d0366aea0b0db8c6f7d7391b"
@@ -4822,6 +4856,11 @@ iconv-lite@0.4.24, iconv-lite@^0.4.24, iconv-lite@^0.4.4:
   dependencies:
     safer-buffer ">= 2.1.2 < 3"
 
+ieee754@1.1.13, ieee754@^1.1.4:
+  version "1.1.13"
+  resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84"
+  integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==
+
 ienoopen@1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/ienoopen/-/ienoopen-1.1.0.tgz#411e5d530c982287dbdc3bb31e7a9c9e32630974"
@@ -5727,6 +5766,11 @@ jest@~25.1.0:
     import-local "^3.0.2"
     jest-cli "^25.1.0"
 
+jmespath@0.15.0:
+  version "0.15.0"
+  resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217"
+  integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=
+
 jquery@^3.3.1:
   version "3.4.1"
   resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.4.1.tgz#714f1f8d9dde4bdfa55764ba37ef214630d80ef2"
@@ -7401,6 +7445,11 @@ punycode2@~1.0.0:
   resolved "https://registry.yarnpkg.com/punycode2/-/punycode2-1.0.0.tgz#e2b4b9a9a8ff157d0b84438e203181ee7892dfd8"
   integrity sha1-4rS5qaj/FX0LhEOOIDGB7niS39g=
 
+punycode@1.3.2:
+  version "1.3.2"
+  resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d"
+  integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=
+
 punycode@^2.1.0, punycode@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec"
@@ -7421,6 +7470,11 @@ qs@~6.5.2:
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36"
   integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==
 
+querystring@0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
+  integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=
+
 querystringify@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.1.1.tgz#60e5a5fd64a7f8bfa4d2ab2ed6fdf4c85bad154e"
@@ -7904,6 +7958,11 @@ sanitize-html@~1.22.0:
     srcset "^2.0.1"
     xtend "^4.0.1"
 
+sax@1.2.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
+  integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o=
+
 sax@>=0.6.0, sax@^1.2.4:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
@@ -9029,6 +9088,14 @@ url-regex@~5.0.0:
     ip-regex "^4.1.0"
     tlds "^1.203.0"
 
+url@0.10.3:
+  version "0.10.3"
+  resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64"
+  integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=
+  dependencies:
+    punycode "1.3.2"
+    querystring "0.2.0"
+
 use@^3.1.0:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f"
@@ -9064,6 +9131,11 @@ utils-merge@1.0.1:
   resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713"
   integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=
 
+uuid@3.3.2:
+  version "3.3.2"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131"
+  integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==
+
 uuid@^3.1.0, uuid@^3.3.2:
   version "3.4.0"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
@@ -9308,7 +9380,7 @@ xml-name-validator@^3.0.0:
   resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"
   integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==
 
-xml2js@^0.4.17:
+xml2js@0.4.19, xml2js@^0.4.17:
   version "0.4.19"
   resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
   integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==