Start to mock+test embeds resolver

+ Put oembed providers into a file
+ Add test snapshots
+ Remove unnecessary request-native dependency

backend/package.json

@@ -94,7 +94,6 @@
     "nodemailer": "^6.3.0",
     "npm-run-all": "~4.1.5",
     "request": "~2.88.0",
-    "request-promise-native": "^1.0.5",
     "sanitize-html": "~1.20.1",
     "slug": "~1.1.0",
     "trunc-html": "~1.1.2",

backend/src/schema/resolvers/embeds.spec.js

@@ -0,0 +1,87 @@
+import fetch, { Response } from 'node-fetch'
+
+import fs from 'fs'
+import path from 'path'
+import { createTestClient } from 'apollo-server-testing'
+import createServer from '../../server'
+import { gql } from '../../jest/helpers'
+
+jest.mock('node-fetch')
+
+let variables = {}
+
+describe('Query', () => {
+  describe('embed', () => {
+    let embedAction
+
+    beforeEach(() => {
+      embedAction = async variables => {
+        const { server } = createServer({
+          context: () => {},
+        })
+        const { query } = createTestClient(server)
+        const embed = gql`
+          query($url: String!) {
+            embed(url: $url) {
+              type
+              title
+              author
+              publisher
+              date
+              description
+              url
+              image
+              audio
+              video
+              lang
+              logo
+              sources
+            }
+          }
+        `
+        return query({ query: embed, variables })
+      }
+    })
+
+    describe('given a youtube link', () => {
+      beforeEach(() => {
+        const youtubeHtml = fs.readFileSync(
+          path.join(__dirname, './embeds/snapshots/babyLovesCat.html'),
+          'utf8',
+        )
+        const embedJson = fs.readFileSync(
+          path.join(__dirname, './embeds/snapshots/oembed/babyLovesCat.json'),
+          'utf8',
+        )
+        fetch
+          .mockReturnValueOnce(Promise.resolve(new Response(youtubeHtml)))
+          .mockReturnValueOnce(Promise.resolve(new Response(embedJson)))
+        variables = { url: 'https://www.youtube.com/watch?v=qkdXAtO40Fo&t=18s' }
+      })
+
+      it('returns meta data', async () => {
+        const expected = expect.objectContaining({
+          data: {
+            embed: {
+              type: 'link',
+              title: 'Baby Loves Cat',
+              author: 'Merkley Family',
+              publisher: 'YouTube',
+              date: '2015-08-16T00:00:00.000Z',
+              description:
+                'She’s incapable of controlling her limbs when her kitty is around. The obsession grows every day. Ps. That’s a sleep sack she’s in. Not a starfish outfit. Al...',
+              url: 'https://www.youtube.com/watch?v=qkdXAtO40Fo',
+              image: 'https://i.ytimg.com/vi/qkdXAtO40Fo/maxresdefault.jpg',
+              audio: null,
+              video: null,
+              lang: 'de',
+              logo: 'https://www.youtube.com/yts/img/favicon_144-vfliLAfaB.png',
+              sources: ['resource'],
+            },
+          },
+        })
+        await expect(embedAction(variables)).resolves.toEqual(expected)
+      })
+    })
+  })
+})

backend/src/schema/resolvers/embeds/scraper.js

@@ -1,15 +1,13 @@
 import Metascraper from 'metascraper'
 import fetch from 'node-fetch'
+import fs from 'fs'
+import path from 'path'
 
 import { ApolloError } from 'apollo-server'
-import request from 'request-promise-native'
-import find from 'lodash/find'
 import isEmpty from 'lodash/isEmpty'
 import isArray from 'lodash/isArray'
 import mergeWith from 'lodash/mergeWith'
 
-const error = require('debug')('embed:error')
-
 const metascraper = Metascraper([
   require('metascraper-author')(),
   require('metascraper-date')(),
@@ -31,47 +29,36 @@ const metascraper = Metascraper([
   // require('./rules/metascraper-embed')()
 ])
 
-let oEmbedProviders = []
-const getEmbedProviders = async () => {
-  let providers = await request('https://oembed.com/providers.json')
-  providers = JSON.parse(providers)
-  oEmbedProviders = providers
-  return providers
-}
-getEmbedProviders()
+let oEmbedProvidersFile = fs.readFileSync(path.join(__dirname, './providers.json'), 'utf8')
+
+// some providers allow a format parameter
+// we need JSON
+oEmbedProvidersFile = oEmbedProvidersFile.replace('{format}', 'json')
+
+const oEmbedProviders = JSON.parse(oEmbedProvidersFile)
 
 const fetchEmbed = async targetUrl => {
   const url = new URL(targetUrl)
-  const embedMeta = find(oEmbedProviders, provider => {
-    return provider.provider_url.indexOf(url.hostname) >= 0
+  const {
+    endpoints: [endpoint],
+  } = oEmbedProviders.find(provider => {
+    return provider.provider_url.includes(url.hostname)
   })
-  if (!embedMeta) {
-    return {}
-  }
-  const embedUrl = embedMeta.endpoints[0].url.replace('{format}', 'json')
+  const endpointUrl = new URL(endpoint.url)
+  endpointUrl.searchParams.append('url', targetUrl)
+  endpointUrl.searchParams.append('format', 'json')
+  const response = await fetch(endpointUrl)
+  const {
+    type = 'link',
+    html,
+    author_name, // eslint-disable-line camelcase
+    upload_date, // eslint-disable-line camelcase
+    sources = ['oembed'],
+  } = await response.json()
 
-  let data
-  try {
-    data = await request(`${embedUrl}?url=${targetUrl}`)
-    data = JSON.parse(data)
-  } catch (err) {
-    data = await request(`${embedUrl}?url=${targetUrl}&format=json`)
-    data = JSON.parse(data)
-  }
-  if (data) {
-    let output = {
-      type: data.type || 'link',
-      embed: data.html,
-      author: data.author_name,
-      date: data.upload_date ? new Date(data.upload_date).toISOString() : null,
-    }
-
-    output.sources = ['oembed']
-
-    return output
-  }
-  return {}
+  return { type, html, author: author_name, date: upload_date, sources }
 }
+
 const fetchMeta = async targetUrl => {
   const response = await fetch(targetUrl)
   const html = await response.text()
@@ -89,34 +76,7 @@ export default async function scrape(targetUrl) {
     targetUrl.hostname = 'youtube.com'
   }
 
-  let meta = {}
-  let embed = {}
-
-  // only get data from requested services
-  await Promise.all([
-    new Promise(async (resolve, reject) => {
-      try {
-        meta = await fetchMeta(targetUrl)
-        resolve()
-      } catch (err) {
-        if (process.env.DEBUG) {
-          error(`ERROR at fetchMeta | ${err.message}`)
-        }
-        resolve()
-      }
-    }),
-    new Promise(async (resolve, reject) => {
-      try {
-        embed = await fetchEmbed(targetUrl)
-        resolve()
-      } catch (err) {
-        if (process.env.DEBUG) {
-          error(`ERROR at fetchEmbed | ${err.message}`)
-        }
-        resolve()
-      }
-    }),
-  ])
+  const [meta, embed] = await Promise.all([fetchMeta(targetUrl), fetchEmbed(targetUrl)])
 
   const output = mergeWith(meta, embed, (objValue, srcValue) => {
     if (isArray(objValue)) {

backend/src/schema/resolvers/embeds/snapshots/oembed/babyLovesCat.json

@@ -0,0 +1 @@
+{"type":"video","provider_url":"https:\/\/www.youtube.com\/","thumbnail_url":"https:\/\/i.ytimg.com\/vi\/qkdXAtO40Fo\/hqdefault.jpg","author_name":"Merkley Family","title":"Baby Loves Cat","thumbnail_width":480,"width":480,"version":"1.0","html":"\u003ciframe width=\"480\" height=\"270\" src=\"https:\/\/www.youtube.com\/embed\/qkdXAtO40Fo?feature=oembed\" frameborder=\"0\" allow=\"accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture\" allowfullscreen\u003e\u003c\/iframe\u003e","thumbnail_height":360,"height":270,"author_url":"https:\/\/www.youtube.com\/channel\/UC5P8yei950tif7UmdPpkJLQ","provider_name":"YouTube"}