Merge pull request #960 from Human-Connection/256-editor-embeds-merge-in-nitro-embed

Editor embeds merge in nitro embed

backend/package.json

@@ -9,6 +9,7 @@
     "dev": "nodemon --exec babel-node src/ -e js,gql",
     "dev:debug": "nodemon --exec babel-node --inspect=0.0.0.0:9229 src/index.js -e js,gql",
     "lint": "eslint src --config .eslintrc.js",
+    "jest": "jest --forceExit --detectOpenHandles --runInBand",
     "test": "run-s test:jest test:cucumber",
     "test:before:server": "cross-env GRAPHQL_URI=http://localhost:4123 GRAPHQL_PORT=4123 yarn run dev 2> /dev/null",
     "test:before:seeder": "cross-env GRAPHQL_URI=http://localhost:4001 GRAPHQL_PORT=4001 DISABLED_MIDDLEWARES=permissions,activityPub yarn run dev 2> /dev/null",
@@ -69,6 +70,22 @@
     "linkifyjs": "~2.1.8",
     "lodash": "~4.17.14",
     "merge-graphql-schemas": "^1.6.1",
+    "metascraper": "^4.10.3",
+    "metascraper-audio": "^5.5.0",
+    "metascraper-author": "^4.8.5",
+    "metascraper-clearbit-logo": "^5.3.0",
+    "metascraper-date": "^4.8.5",
+    "metascraper-description": "^5.5.0",
+    "metascraper-image": "^4.8.5",
+    "metascraper-lang": "^4.8.5",
+    "metascraper-lang-detector": "^4.8.5",
+    "metascraper-logo": "^5.5.0",
+    "metascraper-publisher": "^4.8.5",
+    "metascraper-soundcloud": "^5.5.3",
+    "metascraper-title": "^4.8.5",
+    "metascraper-url": "^5.5.0",
+    "metascraper-video": "^4.8.5",
+    "metascraper-youtube": "^4.8.5",
     "neo4j-driver": "~1.7.4",
     "neo4j-graphql-js": "^2.6.3",
     "neode": "^0.2.16",

backend/src/middleware/index.js

@@ -52,10 +52,12 @@ export default schema => {
   if (CONFIG.DISABLED_MIDDLEWARES) {
     const disabledMiddlewares = CONFIG.DISABLED_MIDDLEWARES.split(',')
     order = order.filter(key => {
+      if (disabledMiddlewares.includes(key)) {
+        /* eslint-disable-next-line no-console */
+        console.log(`Warning: Disabled "${disabledMiddlewares}" middleware.`)
+      }
       return !disabledMiddlewares.includes(key)
     })
-    /* eslint-disable-next-line no-console */
-    console.log(`Warning: "${disabledMiddlewares}" middlewares have been disabled.`)
   }
 
   const appliedMiddlewares = order.map(key => middlewares[key])

backend/src/middleware/permissionsMiddleware.js

@@ -136,6 +136,7 @@ const permissions = shield(
     Query: {
       '*': deny,
       findPosts: allow,
+      embed: allow,
       Category: allow,
       Tag: allow,
       Report: isModerator,

backend/src/schema/helpers.js

@@ -0,0 +1,9 @@
+export const undefinedToNull = list => {
+  const resolvers = {}
+  list.forEach(key => {
+    resolvers[key] = async (parent, params, context, resolveInfo) => {
+      return typeof parent[key] === 'undefined' ? null : parent[key]
+    }
+  })
+  return resolvers
+}

backend/src/schema/resolvers/embeds.js

@@ -0,0 +1,29 @@
+import scrape from './embeds/scraper.js'
+import { undefinedToNull } from '../helpers'
+
+export default {
+  Query: {
+    embed: async (object, { url }, context, resolveInfo) => {
+      return scrape(url)
+    },
+  },
+  Embed: {
+    ...undefinedToNull([
+      'type',
+      'title',
+      'author',
+      'publisher',
+      'date',
+      'description',
+      'url',
+      'image',
+      'audio',
+      'video',
+      'lang',
+      'html',
+    ]),
+    sources: async (parent, params, context, resolveInfo) => {
+      return typeof parent.sources === 'undefined' ? [] : parent.sources
+    },
+  },
+}

backend/src/schema/resolvers/embeds.spec.js

@@ -0,0 +1,216 @@
+import fetch from 'node-fetch'
+import fs from 'fs'
+import path from 'path'
+import { createTestClient } from 'apollo-server-testing'
+import createServer from '../../server'
+import { gql } from '../../jest/helpers'
+
+jest.mock('node-fetch')
+const { Response } = jest.requireActual('node-fetch')
+
+afterEach(() => {
+  fetch.mockRestore()
+})
+
+let variables = {}
+
+const HumanConnectionOrg = fs.readFileSync(
+  path.join(__dirname, '../../jest/snapshots/embeds/HumanConnectionOrg.html'),
+  'utf8',
+)
+const pr960 = fs.readFileSync(
+  path.join(__dirname, '../../jest/snapshots/embeds/pr960.html'),
+  'utf8',
+)
+const babyLovesCat = fs.readFileSync(
+  path.join(__dirname, '../../jest/snapshots/embeds/babyLovesCat.html'),
+  'utf8',
+)
+
+const babyLovesCatEmbedResponse = new Response(
+  JSON.stringify({
+    height: 270,
+    provider_name: 'YouTube',
+    title: 'Baby Loves Cat',
+    type: 'video',
+    width: 480,
+    thumbnail_height: 360,
+    provider_url: 'https://www.youtube.com/',
+    thumbnail_width: 480,
+    html:
+      '<iframe width="480" height="270" src="https://www.youtube.com/embed/qkdXAtO40Fo?start=18&feature=oembed" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>',
+    thumbnail_url: 'https://i.ytimg.com/vi/qkdXAtO40Fo/hqdefault.jpg',
+    version: '1.0',
+    author_name: 'Merkley Family',
+    author_url: 'https://www.youtube.com/channel/UC5P8yei950tif7UmdPpkJLQ',
+  }),
+)
+
+describe('Query', () => {
+  describe('embed', () => {
+    let embedAction
+
+    beforeEach(() => {
+      embedAction = async variables => {
+        const { server } = createServer({
+          context: () => {},
+        })
+        const { query } = createTestClient(server)
+        const embed = gql`
+          query($url: String!) {
+            embed(url: $url) {
+              type
+              title
+              author
+              publisher
+              date
+              description
+              url
+              image
+              audio
+              video
+              lang
+              sources
+              html
+            }
+          }
+        `
+        return query({ query: embed, variables })
+      }
+    })
+
+    describe('given a video link', () => {
+      beforeEach(() => {
+        fetch
+          .mockReturnValueOnce(Promise.resolve(new Response('')))
+          .mockReturnValueOnce(Promise.resolve(JSON.stringify({})))
+        variables = { url: 'https://www.w3schools.com/html/mov_bbb.mp4' }
+      })
+
+      it('shows some default data', async () => {
+        const expected = expect.objectContaining({
+          data: {
+            embed: {
+              audio: null,
+              author: null,
+              date: null,
+              description: null,
+              html: null,
+              image: null,
+              lang: null,
+              publisher: null,
+              sources: ['resource'],
+              title: null,
+              type: 'link',
+              url: 'https://www.w3schools.com/html/mov_bbb.mp4',
+              video: null,
+            },
+          },
+        })
+        await expect(embedAction(variables)).resolves.toEqual(expected)
+      })
+    })
+
+    describe('given a Facebook link', () => {
+      beforeEach(() => {
+        fetch
+          .mockReturnValueOnce(Promise.resolve(new Response(HumanConnectionOrg)))
+          .mockReturnValueOnce(Promise.resolve('invalid json'))
+        variables = { url: 'https://www.facebook.com/HumanConnectionOrg/' }
+      })
+
+      it('does not crash if embed provider returns invalid JSON', async () => {
+        const expected = expect.objectContaining({
+          data: {
+            embed: {
+              audio: null,
+              author: null,
+              date: expect.any(String),
+              description:
+                'Human Connection, Weilheim an der Teck. Gefällt 24.407 Mal. An upcoming non-profit social network focused on local and global positive change. Twitter accounts : @hc_world (EN), @hc_deutschland (GE),...',
+              html: null,
+              image:
+                'https://scontent.ftxl3-1.fna.fbcdn.net/v/t1.0-1/c5.0.200.200a/p200x200/12108307_997373093648222_70057205881020137_n.jpg?_nc_cat=110&_nc_oc=AQnPPYQlR0dU556gOfl4xkXr7IPZdRIAUfQeXl3fpUv4DAsFN8T4PfgOjPwuq85GPKGZ5S5E5mWQ8IVV1UiRBAIZ&_nc_ht=scontent.ftxl3-1.fna&oh=90309adddaab38839782f16e7d4b7bcf&oe=5DEEDFE5',
+              lang: 'de',
+              publisher: 'Facebook',
+              sources: ['resource'],
+              title: 'Human Connection',
+              type: 'link',
+              url: 'https://www.facebook.com/HumanConnectionOrg/',
+              video: null,
+            },
+          },
+        })
+        await expect(embedAction(variables)).resolves.toEqual(expected)
+      })
+    })
+
+    describe('given a Github link', () => {
+      beforeEach(() => {
+        fetch
+          .mockReturnValueOnce(Promise.resolve(new Response(pr960)))
+          .mockReturnValueOnce(Promise.resolve(JSON.stringify({})))
+        variables = { url: 'https://github.com/Human-Connection/Human-Connection/pull/960' }
+      })
+
+      it('returns meta data even if no embed html can be retrieved', async () => {
+        const expected = expect.objectContaining({
+          data: {
+            embed: {
+              type: 'link',
+              title:
+                'Editor embeds merge in nitro embed by mattwr18 · Pull Request #960 · Human-Connection/Human-Connection',
+              author: 'Human-Connection',
+              publisher: 'GitHub',
+              date: expect.any(String),
+              description: '🍰 Pullrequest Issues fixes #256',
+              url: 'https://github.com/Human-Connection/Human-Connection/pull/960',
+              image:
+                'https://repository-images.githubusercontent.com/112590397/52c9a000-7e11-11e9-899d-aaa55f3a3d72',
+              audio: null,
+              video: null,
+              lang: 'en',
+              sources: ['resource'],
+              html: null,
+            },
+          },
+        })
+        await expect(embedAction(variables)).resolves.toEqual(expected)
+      })
+    })
+
+    describe('given a youtube link', () => {
+      beforeEach(() => {
+        fetch
+          .mockReturnValueOnce(Promise.resolve(new Response(babyLovesCat)))
+          .mockReturnValueOnce(Promise.resolve(babyLovesCatEmbedResponse))
+        variables = { url: 'https://www.youtube.com/watch?v=qkdXAtO40Fo&t=18s' }
+      })
+
+      it('returns meta data plus youtube iframe html', async () => {
+        const expected = expect.objectContaining({
+          data: {
+            embed: {
+              type: 'video',
+              title: 'Baby Loves Cat',
+              author: 'Merkley Family',
+              publisher: 'YouTube',
+              date: expect.any(String),
+              description:
+                'She’s incapable of controlling her limbs when her kitty is around. The obsession grows every day. Ps. That’s a sleep sack she’s in. Not a starfish outfit. Al...',
+              url: 'https://www.youtube.com/watch?v=qkdXAtO40Fo',
+              image: 'https://i.ytimg.com/vi/qkdXAtO40Fo/maxresdefault.jpg',
+              audio: null,
+              video: null,
+              lang: 'de',
+              sources: ['resource', 'oembed'],
+              html:
+                '<iframe width="480" height="270" src="https://www.youtube.com/embed/qkdXAtO40Fo?start=18&feature=oembed" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>',
+            },
+          },
+        })
+        await expect(embedAction(variables)).resolves.toEqual(expected)
+      })
+    })
+  })
+})

backend/src/schema/resolvers/embeds/scraper.js

@@ -0,0 +1,102 @@
+import Metascraper from 'metascraper'
+import fetch from 'node-fetch'
+import fs from 'fs'
+import path from 'path'
+
+import { ApolloError } from 'apollo-server'
+import isEmpty from 'lodash/isEmpty'
+import isArray from 'lodash/isArray'
+import mergeWith from 'lodash/mergeWith'
+
+const error = require('debug')('embed:error')
+
+const metascraper = Metascraper([
+  require('metascraper-author')(),
+  require('metascraper-date')(),
+  require('metascraper-description')(),
+  require('metascraper-image')(),
+  require('metascraper-lang')(),
+  require('metascraper-lang-detector')(),
+  require('metascraper-logo')(),
+  // require('metascraper-clearbit-logo')(),
+  require('metascraper-publisher')(),
+  require('metascraper-title')(),
+  require('metascraper-url')(),
+  require('metascraper-audio')(),
+  require('metascraper-soundcloud')(),
+  require('metascraper-video')(),
+  require('metascraper-youtube')(),
+
+  // require('./rules/metascraper-embed')()
+])
+
+let oEmbedProvidersFile = fs.readFileSync(path.join(__dirname, './providers.json'), 'utf8')
+
+// some providers allow a format parameter
+// we need JSON
+oEmbedProvidersFile = oEmbedProvidersFile.replace('{format}', 'json')
+
+const oEmbedProviders = JSON.parse(oEmbedProvidersFile)
+
+const fetchEmbed = async url => {
+  const provider = oEmbedProviders.find(provider => {
+    return provider.provider_url.includes(url.hostname)
+  })
+  if (!provider) return {}
+  const {
+    endpoints: [endpoint],
+  } = provider
+  const endpointUrl = new URL(endpoint.url)
+  endpointUrl.searchParams.append('url', url.href)
+  endpointUrl.searchParams.append('format', 'json')
+  let json
+  try {
+    const response = await fetch(endpointUrl)
+    json = await response.json()
+  } catch (err) {
+    error(`Error fetching embed data: ${err.message}`)
+    return {}
+  }
+
+  return {
+    type: json.type,
+    html: json.html,
+    author: json.author_name,
+    date: json.upload_date,
+    sources: ['oembed'],
+  }
+}
+
+const fetchResource = async url => {
+  const response = await fetch(url)
+  const html = await response.text()
+  const resource = await metascraper({ html, url: url.href })
+  return {
+    sources: ['resource'],
+    ...resource,
+  }
+}
+
+export default async function scrape(url) {
+  url = new URL(url)
+  if (url.hostname === 'youtu.be') {
+    // replace youtu.be to get proper results
+    url.hostname = 'youtube.com'
+  }
+
+  const [meta, embed] = await Promise.all([fetchResource(url), fetchEmbed(url)])
+  const output = mergeWith(meta, embed, (objValue, srcValue) => {
+    if (isArray(objValue)) {
+      return objValue.concat(srcValue)
+    }
+  })
+
+  if (isEmpty(output)) {
+    throw new ApolloError('Not found', 'NOT_FOUND')
+  }
+
+  return {
+    type: 'link',
+    ...output,
+  }
+}

backend/src/schema/resolvers/users.js

@@ -2,6 +2,7 @@ import { neo4jgraphql } from 'neo4j-graphql-js'
 import fileUpload from './fileUpload'
 import { neode } from '../../bootstrap/neo4j'
 import { UserInputError } from 'apollo-server'
+import { undefinedToNull } from '../helpers'
 
 const instance = neode()
 
@@ -36,16 +37,6 @@ const count = obj => {
   return resolvers
 }
 
-const undefinedToNull = list => {
-  const resolvers = {}
-  list.forEach(key => {
-    resolvers[key] = async (parent, params, context, resolveInfo) => {
-      return typeof parent[key] === 'undefined' ? null : parent[key]
-    }
-  })
-  return resolvers
-}
-
 export const hasMany = obj => {
   const resolvers = {}
   for (const [key, connection] of Object.entries(obj)) {

backend/src/schema/types/embed.gql

@@ -0,0 +1,19 @@
+type Embed {
+  type: String
+  title: String
+  author: String
+  publisher: String
+  date: String
+  description: String
+  url: String
+  image: String
+  audio: String
+  video: String
+  lang: String
+  html: String
+  sources: [String]
+}
+
+type Query {
+  embed(url: String!): Embed
+}

docker-compose.yml

@@ -42,6 +42,5 @@ services:
       context: neo4j
     networks:
       - hc-network
-
 networks:
   hc-network:

webapp/styleguide/src/system/icons/svg/bold.svg

@@ -1,5 +0,0 @@
-<!-- Generated by IcoMoon.io -->
-<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
-<title>bold</title>
-<path d="M16 7h-9v18h11c2.8 0 5-2.2 5-5 0-2.2-1.4-4-3.3-4.7 0.8-0.9 1.3-2 1.3-3.3 0-2.8-2.2-5-5-5zM9 15v-6h7c1.7 0 3 1.3 3 3s-1.3 3-3 3h-7zM9 23v-6h9c1.7 0 3 1.3 3 3s-1.3 3-3 3h-9zM16 5v0c3.9 0 7 3.1 7 7 0 0.9-0.2 1.8-0.5 2.6 1.5 1.3 2.5 3.3 2.5 5.4 0 3.9-3.1 7-7 7h-13v-22h11zM11 11v0 2h5c0.6 0 1-0.4 1-1s-0.4-1-1-1h-5zM11 19v0 2h7c0.6 0 1-0.4 1-1s-0.4-1-1-1h-7z"></path>
-</svg>

webapp/styleguide/src/system/icons/svg/italic.svg

@@ -1,5 +0,0 @@
-<!-- Generated by IcoMoon.io -->
-<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
-<title>italic</title>
-<path d="M11.75 5h10.031l-0.094 1.063-0.188 3-0.063 0.938h-2l-0.875 12h2l-0.063 1.063-0.188 3-0.063 0.938h-10.031l0.094-1.063 0.188-3 0.063-0.938h2l0.875-12h-2l0.063-1.063 0.188-3zM13.625 7l-0.063 1h2l-0.063 1.063-1 14-0.063 0.938h-2l-0.063 1h6l0.063-1h-2l0.063-1.063 1-14 0.063-0.938h2l0.063-1h-6z"></path>
-</svg>

webapp/styleguide/src/system/icons/svg/list-ol.svg

@@ -1,5 +0,0 @@
-<!-- Generated by IcoMoon.io -->
-<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
-<title>list-ol</title>
-<path d="M5.969 3h2.031v7h-2v-4.531c-0.444 0.255-0.913 0.531-1.594 0.531v-2c0.494 0 1.25-0.656 1.25-0.656zM11 6h17v2h-17v-2zM6.5 12c1.383 0 2.5 1.117 2.5 2.5 0 0.481-0.248 1.090-0.75 1.5l0.031 0.031-0.125 0.094-0.875 0.875h1.719v2h-5v-1.625l0.313-0.281 2.688-2.594c0-0.217-0.283-0.5-0.5-0.5s-0.5 0.283-0.5 0.5v0.5h-2v-0.5c0-1.383 1.117-2.5 2.5-2.5zM11 15h17v2h-17v-2zM4 21h4v1.469l-0.125 0.25-0.406 0.688c0.853 0.398 1.531 1.089 1.531 2.094 0 1.383-1.117 2.5-2.5 2.5h-2.5v-2h2.5c0.217 0 0.5-0.283 0.5-0.5s-0.283-0.5-0.5-0.5h-1.5v-1.375l0.125-0.219 0.25-0.406h-1.375v-2zM11 24h17v2h-17v-2z"></path>
-</svg>

webapp/styleguide/src/system/icons/svg/list-ul.svg

@@ -1,5 +0,0 @@
-<!-- Generated by IcoMoon.io -->
-<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
-<title>list-ul</title>
-<path d="M4 5h6v6h-6v-6zM6 7v2h2v-2h-2zM12 7h15v2h-15v-2zM4 13h6v6h-6v-6zM6 15v2h2v-2h-2zM12 15h15v2h-15v-2zM4 21h6v6h-6v-6zM6 23v2h2v-2h-2zM12 23h15v2h-15v-2z"></path>
-</svg>

webapp/styleguide/src/system/icons/svg/paragraph.svg

@@ -1,5 +0,0 @@
-<!-- Generated by IcoMoon.io -->
-<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
-<title>paragraph</title>
-<path d="M12 5h12v2h-2v20h-2v-20h-2v20h-2v-10h-4c-3.302 0-6-2.698-6-6s2.698-6 6-6zM12 7c-2.22 0-4 1.78-4 4s1.78 4 4 4h4v-8h-4z"></path>
-</svg>

webapp/styleguide/src/system/icons/svg/quote-right.svg

@@ -1,5 +0,0 @@
-<!-- Generated by IcoMoon.io -->
-<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
-<title>quote-right</title>
-<path d="M4 8h10v10c0 3.302-2.698 6-6 6v-2c2.22 0 4-1.78 4-4h-8v-10zM18 8h10v10c0 3.302-2.698 6-6 6v-2c2.22 0 4-1.78 4-4h-8v-10zM6 10v6h6v-6h-6zM20 10v6h6v-6h-6z"></path>
-</svg>