Merge pull request #3262 from Human-Connection/3250_upload_images_to_spaces

feat(backend): upload original image files on S3 object storage

backend/.env.template

@@ -17,3 +17,9 @@ PRIVATE_KEY_PASSPHRASE="a7dsf78sadg87ad87sfagsadg78"
 SENTRY_DSN_BACKEND=
 COMMIT=
 PUBLIC_REGISTRATION=false
+
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_ENDPOINT=
+AWS_REGION=
+AWS_BUCKET=

backend/package.json

@@ -45,6 +45,7 @@
     "apollo-link-http": "~1.5.16",
     "apollo-server": "~2.11.0",
     "apollo-server-express": "^2.11.0",
+    "aws-sdk": "^2.638.0",
     "babel-plugin-transform-runtime": "^6.23.0",
     "bcryptjs": "~2.4.3",
     "cheerio": "~1.0.0-rc.3",
@@ -86,6 +87,7 @@
     "metascraper-video": "^5.11.6",
     "metascraper-youtube": "^5.11.6",
     "migrate": "^1.6.2",
+    "mime-types": "^2.1.26",
     "minimatch": "^3.0.4",
     "mustache": "^4.0.1",
     "neo4j-driver": "^4.0.2",

backend/src/config/index.js

@@ -18,6 +18,11 @@ const {
   SMTP_PASSWORD,
   SENTRY_DSN_BACKEND,
   COMMIT,
+  AWS_ACCESS_KEY_ID,
+  AWS_SECRET_ACCESS_KEY,
+  AWS_ENDPOINT,
+  AWS_REGION,
+  AWS_BUCKET,
   NEO4J_URI = 'bolt://localhost:7687',
   NEO4J_USERNAME = 'neo4j',
   NEO4J_PASSWORD = 'neo4j',
@@ -64,7 +69,20 @@ export const developmentConfigs = {
 }
 
 export const sentryConfigs = { SENTRY_DSN_BACKEND, COMMIT }
-export const redisConfiig = { REDIS_DOMAIN, REDIS_PORT, REDIS_PASSWORD }
+export const redisConfigs = { REDIS_DOMAIN, REDIS_PORT, REDIS_PASSWORD }
+
+const S3_CONFIGURED =
+  AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY && AWS_ENDPOINT && AWS_REGION && AWS_BUCKET
+
+export const s3Configs = {
+  AWS_ACCESS_KEY_ID,
+  AWS_SECRET_ACCESS_KEY,
+  AWS_ENDPOINT,
+  AWS_REGION,
+  AWS_BUCKET,
+  S3_CONFIGURED,
+}
+
 export default {
   ...requiredConfigs,
   ...smtpConfigs,
@@ -72,5 +90,6 @@ export default {
   ...serverConfigs,
   ...developmentConfigs,
   ...sentryConfigs,
-  ...redisConfiig,
+  ...redisConfigs,
+  ...s3Configs,
 }

backend/src/db/migrations/20200312140328-bulk_upload_to_s3.js

@@ -0,0 +1,103 @@
+import { getDriver } from '../../db/neo4j'
+import { existsSync, createReadStream } from 'fs'
+import path from 'path'
+import { S3 } from 'aws-sdk'
+import mime from 'mime-types'
+import { s3Configs } from '../../config'
+
+export const description = `
+Upload all image files to a S3 compatible object storage in order to reduce
+load on our backend.
+`
+
+export async function up(next) {
+  const driver = getDriver()
+  const session = driver.session()
+  const transaction = session.beginTransaction()
+
+  const {
+    AWS_ENDPOINT: endpoint,
+    AWS_REGION: region,
+    AWS_BUCKET: Bucket,
+    S3_CONFIGURED,
+  } = s3Configs
+
+  if (!S3_CONFIGURED) {
+    // eslint-disable-next-line no-console
+    console.log('No S3 given, cannot upload image files')
+    return
+  }
+
+  const s3 = new S3({ region, endpoint })
+  try {
+    // Implement your migration here.
+    const { records } = await transaction.run('MATCH (image:Image) RETURN image.url as url')
+    let urls = records.map((r) => r.get('url'))
+    urls = urls.filter((url) => url.startsWith('/uploads'))
+    const locations = await Promise.all(
+      urls
+        .map((url) => {
+          return async () => {
+            const { pathname } = new URL(url, 'http://example.org')
+            const fileLocation = path.join(__dirname, `../../../public/${pathname}`)
+            const s3Location = `original${pathname}`
+            if (existsSync(fileLocation)) {
+              const mimeType = mime.lookup(fileLocation)
+              const params = {
+                Bucket,
+                Key: s3Location,
+                ACL: 'public-read',
+                ContentType: mimeType || 'image/jpeg',
+                Body: createReadStream(fileLocation),
+              }
+
+              const data = await s3.upload(params).promise()
+              const { Location: spacesUrl } = data
+
+              const updatedRecord = await transaction.run(
+                'MATCH (image:Image {url: $url}) SET image.url = $spacesUrl RETURN image.url as url',
+                { url, spacesUrl },
+              )
+              const [updatedUrl] = updatedRecord.records.map((record) => record.get('url'))
+              return updatedUrl
+            }
+          }
+        })
+        .map((p) => p()),
+    )
+    // eslint-disable-next-line no-console
+    console.log('this is locations', locations)
+    await transaction.commit()
+    next()
+  } catch (error) {
+    // eslint-disable-next-line no-console
+    console.log(error)
+    await transaction.rollback()
+    // eslint-disable-next-line no-console
+    console.log('rolled back')
+    throw new Error(error)
+  } finally {
+    session.close()
+  }
+}
+
+export async function down(next) {
+  const driver = getDriver()
+  const session = driver.session()
+  const transaction = session.beginTransaction()
+
+  try {
+    // Implement your migration here.
+    await transaction.run(``)
+    await transaction.commit()
+    next()
+  } catch (error) {
+    // eslint-disable-next-line no-console
+    console.log(error)
+    await transaction.rollback()
+    // eslint-disable-next-line no-console
+    console.log('rolled back')
+  } finally {
+    session.close()
+  }
+}

backend/src/schema/resolvers/images/images.js

@@ -1,11 +1,14 @@
 import path from 'path'
 import { v4 as uuid } from 'uuid'
+import { S3 } from 'aws-sdk'
 import slug from 'slug'
 import { existsSync, unlinkSync, createWriteStream } from 'fs'
-import { getDriver } from '../../../db/neo4j'
 import { UserInputError } from 'apollo-server'
+import { getDriver } from '../../../db/neo4j'
+import { s3Configs } from '../../../config'
 
 // const widths = [34, 160, 320, 640, 1024]
+const { AWS_ENDPOINT: endpoint, AWS_REGION: region, AWS_BUCKET: Bucket, S3_CONFIGURED } = s3Configs
 
 export async function deleteImage(resource, relationshipType, opts = {}) {
   sanitizeRelationshipType(relationshipType)
@@ -79,23 +82,24 @@ const wrapTransaction = async (wrappedCallback, args, opts) => {
   }
 }
 
-const deleteImageFile = (image, deleteCallback = localFileDelete) => {
+const deleteImageFile = (image, deleteCallback) => {
+  if (!deleteCallback) {
+    deleteCallback = S3_CONFIGURED ? s3Delete : localFileDelete
+  }
   const { url } = image
   deleteCallback(url)
   return url
 }
 
-const uploadImageFile = async (upload, uploadCallback = localFileUpload) => {
+const uploadImageFile = async (upload, uploadCallback) => {
   if (!upload) return undefined
+  if (!uploadCallback) {
+    uploadCallback = S3_CONFIGURED ? s3Upload : localFileUpload
+  }
   const { createReadStream, filename, mimetype } = await upload
   const { name, ext } = path.parse(filename)
   const uniqueFilename = `${uuid()}-${slug(name)}${ext}`
-
+  return uploadCallback({ createReadStream, uniqueFilename, mimetype })
-  return uploadCallback({
-    createReadStream,
-    destination: `/uploads/${uniqueFilename}`,
-    mimetype,
-  })
 }
 
 const sanitizeRelationshipType = (relationshipType) => {
@@ -106,7 +110,8 @@ const sanitizeRelationshipType = (relationshipType) => {
   }
 }
 
-const localFileUpload = ({ createReadStream, destination }) => {
+const localFileUpload = ({ createReadStream, uniqueFilename }) => {
+  const destination = `/uploads/${uniqueFilename}`
   return new Promise((resolve, reject) =>
     createReadStream()
       .pipe(createWriteStream(`public${destination}`))
@@ -115,7 +120,34 @@ const localFileUpload = ({ createReadStream, destination }) => {
   )
 }
 
+const s3Upload = async ({ createReadStream, uniqueFilename, mimetype }) => {
+  const s3 = new S3({ region, endpoint })
+  const s3Location = `original/${uniqueFilename}`
+
+  const params = {
+    Bucket,
+    Key: s3Location,
+    ACL: 'public-read',
+    ContentType: mimetype,
+    Body: createReadStream(),
+  }
+  const data = await s3.upload(params).promise()
+  const { Location } = data
+  return Location
+}
+
 const localFileDelete = async (url) => {
   const location = `public${url}`
   if (existsSync(location)) unlinkSync(location)
 }
+
+const s3Delete = async (url) => {
+  const s3 = new S3({ region, endpoint })
+  let { pathname } = new URL(url, 'http://example.org') // dummy domain to avoid invalid URL error
+  pathname = pathname.substring(1) // remove first character '/'
+  const params = {
+    Bucket,
+    Key: pathname,
+  }
+  await s3.deleteObject(params).promise()
+}

backend/src/schema/resolvers/images/images.spec.js

@@ -11,7 +11,7 @@ let deleteCallback
 
 beforeEach(async () => {
   await cleanDatabase()
-  uploadCallback = jest.fn(({ destination }) => destination)
+  uploadCallback = jest.fn(({ uniqueFilename }) => `/uploads/${uniqueFilename}`)
   deleteCallback = jest.fn()
 })
 
@@ -99,19 +99,6 @@ describe('mergeImage', () => {
     }
   })
 
-  describe('on existing resource', () => {
-    beforeEach(async () => {
-      post = await Factory.build(
-        'post',
-        { id: 'p99' },
-        {
-          author: Factory.build('user', {}, { avatar: null }),
-          image: null,
-        },
-      )
-      post = await post.toJson()
-    })
-
   describe('given image.upload', () => {
     beforeEach(() => {
       imageInput = {
@@ -129,6 +116,19 @@ describe('mergeImage', () => {
       }
     })
 
+    describe('on existing resource', () => {
+      beforeEach(async () => {
+        post = await Factory.build(
+          'post',
+          { id: 'p99' },
+          {
+            author: Factory.build('user', {}, { avatar: null }),
+            image: null,
+          },
+        )
+        post = await post.toJson()
+      })
+
       it('returns new image', async () => {
         await expect(
           mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback }),
@@ -330,7 +330,7 @@ describe('mergeImage', () => {
       })
 
       it('updates metadata', async () => {
-        await mergeImage(post, 'HERO_IMAGE', imageInput)
+        await mergeImage(post, 'HERO_IMAGE', imageInput, { uploadCallback, deleteCallback })
         const images = await neode.all('Image')
         expect(images).toHaveLength(1)
         await expect(images.first().toJson()).resolves.toMatchObject({

backend/yarn.lock

@@ -2220,6 +2220,21 @@ audio-extensions@0.0.0:
   resolved "https://registry.yarnpkg.com/audio-extensions/-/audio-extensions-0.0.0.tgz#d0eefe077fb9eb625898eed9985890548cf1f8d2"
   integrity sha1-0O7+B3+562JYmO7ZmFiQVIzx+NI=
 
+aws-sdk@^2.638.0:
+  version "2.638.0"
+  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.638.0.tgz#43df5a956696177c577b841ea21b4007a81dbdaa"
+  integrity sha512-DOSwedH2YkPVs3c2AQezK6FHuGRIDffgULGvmpY9ZmZ/x45Sw+p7WHCYPgWfw/Z1fJWzMjaIpu531xG7pyJV4A==
+  dependencies:
+    buffer "4.9.1"
+    events "1.1.1"
+    ieee754 "1.1.13"
+    jmespath "0.15.0"
+    querystring "0.2.0"
+    sax "1.2.1"
+    url "0.10.3"
+    uuid "3.3.2"
+    xml2js "0.4.19"
+
 aws-sign2@~0.7.0:
   version "0.7.0"
   resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
@@ -2319,6 +2334,11 @@ balanced-match@^1.0.0:
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
   integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c=
 
+base64-js@^1.0.2:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1"
+  integrity sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==
+
 base@^0.11.1:
   version "0.11.2"
   resolved "https://registry.yarnpkg.com/base/-/base-0.11.2.tgz#7bde5ced145b6d551a90db87f83c558b4eb48a8f"
@@ -2472,6 +2492,15 @@ buffer-from@^1.0.0:
   resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef"
   integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==
 
+buffer@4.9.1:
+  version "4.9.1"
+  resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298"
+  integrity sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=
+  dependencies:
+    base64-js "^1.0.2"
+    ieee754 "^1.1.4"
+    isarray "^1.0.0"
+
 busboy@^0.3.1:
   version "0.3.1"
   resolved "https://registry.yarnpkg.com/busboy/-/busboy-0.3.1.tgz#170899274c5bf38aae27d5c62b71268cd585fd1b"
@@ -3821,6 +3850,11 @@ eventemitter3@^3.1.0:
   resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-3.1.2.tgz#2d3d48f9c346698fce83a85d7d664e98535df6e7"
   integrity sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==
 
+events@1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
+  integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=
+
 exec-sh@^0.3.2:
   version "0.3.2"
   resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.2.tgz#6738de2eb7c8e671d0366aea0b0db8c6f7d7391b"
@@ -4832,6 +4866,11 @@ iconv-lite@0.4.24, iconv-lite@^0.4.24, iconv-lite@^0.4.4:
   dependencies:
     safer-buffer ">= 2.1.2 < 3"
 
+ieee754@1.1.13, ieee754@^1.1.4:
+  version "1.1.13"
+  resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84"
+  integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==
+
 ienoopen@1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/ienoopen/-/ienoopen-1.1.0.tgz#411e5d530c982287dbdc3bb31e7a9c9e32630974"
@@ -5741,6 +5780,11 @@ jest@~25.2.0:
     import-local "^3.0.2"
     jest-cli "^25.2.0"
 
+jmespath@0.15.0:
+  version "0.15.0"
+  resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217"
+  integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=
+
 jquery@^3.3.1:
   version "3.4.1"
   resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.4.1.tgz#714f1f8d9dde4bdfa55764ba37ef214630d80ef2"
@@ -6406,7 +6450,7 @@ mime-db@1.43.0:
   resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.43.0.tgz#0a12e0502650e473d735535050e7c8f4eb4fae58"
   integrity sha512-+5dsGEEovYbT8UY9yD7eE4XTc4UwJ1jBYlgaQQF38ENsKR3wj/8q8RFZrF9WIZpB2V1ArTVFUva8sAul1NzRzQ==
 
-mime-types@^2.1.12, mime-types@~2.1.19, mime-types@~2.1.22, mime-types@~2.1.24, mime-types@~2.1.26:
+mime-types@^2.1.12, mime-types@^2.1.26, mime-types@~2.1.19, mime-types@~2.1.22, mime-types@~2.1.24, mime-types@~2.1.26:
   version "2.1.26"
   resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.26.tgz#9c921fc09b7e149a65dfdc0da4d20997200b0a06"
   integrity sha512-01paPWYgLrkqAyrlDorC1uDwl2p3qZT7yl806vW7DvDoxwXi46jsjFbg+WdwotBIk6/MbEhO/dh5aZ5sNj/dWQ==
@@ -7420,6 +7464,11 @@ punycode2@~1.0.0:
   resolved "https://registry.yarnpkg.com/punycode2/-/punycode2-1.0.0.tgz#e2b4b9a9a8ff157d0b84438e203181ee7892dfd8"
   integrity sha1-4rS5qaj/FX0LhEOOIDGB7niS39g=
 
+punycode@1.3.2:
+  version "1.3.2"
+  resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d"
+  integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=
+
 punycode@^2.1.0, punycode@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec"
@@ -7440,6 +7489,11 @@ qs@~6.5.2:
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36"
   integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==
 
+querystring@0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
+  integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=
+
 querystringify@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.1.1.tgz#60e5a5fd64a7f8bfa4d2ab2ed6fdf4c85bad154e"
@@ -7928,6 +7982,11 @@ sanitize-html@~1.22.0:
     srcset "^2.0.1"
     xtend "^4.0.1"
 
+sax@1.2.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
+  integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o=
+
 sax@>=0.6.0, sax@^1.2.4:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
@@ -9048,6 +9107,14 @@ url-regex@~5.0.0:
     ip-regex "^4.1.0"
     tlds "^1.203.0"
 
+url@0.10.3:
+  version "0.10.3"
+  resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64"
+  integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=
+  dependencies:
+    punycode "1.3.2"
+    querystring "0.2.0"
+
 use@^3.1.0:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f"
@@ -9083,6 +9150,11 @@ utils-merge@1.0.1:
   resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713"
   integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=
 
+uuid@3.3.2:
+  version "3.3.2"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131"
+  integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==
+
 uuid@^3.1.0, uuid@^3.3.2:
   version "3.4.0"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
@@ -9327,7 +9399,7 @@ xml-name-validator@^3.0.0:
   resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"
   integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==
 
-xml2js@^0.4.17:
+xml2js@0.4.19, xml2js@^0.4.17:
   version "0.4.19"
   resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
   integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==