IT4Change/Ocelot-Social
3
0
Fork 0
mirror of https://github.com/IT4Change/Ocelot-Social.git synced 2025-12-13 07:45:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

added sources to results from which data was aggregated

Browse Source
This commit is contained in:
Grzegorz Leoniec 2018-12-28 16:35:36 +01:00
parent 32fad6118c
commit f8f0ff3d06
No known key found for this signature in database
GPG Key ID: 3AA43686D4EB1377
2 changed files with 23 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/schema.graphql
View File

@ -12,6 +12,7 @@ type Embed {
lang: String lang: String
logo: String logo: String
embed: String embed: String
sources: [String]
} }
type Query { type Query {

31
src/scraper.js
View File

@ -26,6 +26,8 @@ const request = require('request-promise-native')
const find = require('lodash/find') const find = require('lodash/find')
const isEmpty = require('lodash/isEmpty') const isEmpty = require('lodash/isEmpty')
const each = require('lodash/each') const each = require('lodash/each')
const isArray = require('lodash/isArray')
const mergeWith = require('lodash/mergeWith')
const urlParser = require('url') const urlParser = require('url')
// quick in memory cache // quick in memory cache
@ -52,12 +54,15 @@ const removeEmptyAttrs = obj => {
const scraper = { const scraper = {
async fetch(targetUrl) { async fetch(targetUrl) {
if (targetUrl.indexOf('//youtu.be/')) { if (targetUrl.indexOf('//youtu.be/')) {
// replace youtu.be to get proper results // replace youtu.be to get proper results
targetUrl = targetUrl.replace('//youtu.be/', '//youtube.com/') targetUrl = targetUrl.replace('//youtu.be/', '//youtube.com/')
} }
if (cache[targetUrl]) {
return cache[targetUrl]
}
const url = parseUrl.parse(targetUrl, true) const url = parseUrl.parse(targetUrl, true)
let meta = {} let meta = {}
@ -89,10 +94,15 @@ const scraper = {
}) })
]) ])
const output = { const output = mergeWith(
...(removeEmptyAttrs(meta)), meta,
...(removeEmptyAttrs(embed)) embed,
(objValue, srcValue) => {
if (isArray(objValue)) {
return objValue.concat(srcValue);
} }
}
)
if (isEmpty(output)) { if (isEmpty(output)) {
throw new ApolloError('Not found', 404) throw new ApolloError('Not found', 404)
@ -105,6 +115,9 @@ const scraper = {
output.url += `&start=${YouTubeStartParam}` output.url += `&start=${YouTubeStartParam}`
} }
// write to cache
cache[targetUrl] = output
return output return output
}, },
async fetchEmbed(targetUrl) { async fetchEmbed(targetUrl) {
@ -133,6 +146,8 @@ const scraper = {
date: data.upload_date ? new Date(data.upload_date).toISOString() : null date: data.upload_date ? new Date(data.upload_date).toISOString() : null
} }
output.sources = ['oembed']
return output return output
} }
return {} return {}
@ -142,16 +157,12 @@ const scraper = {
// const parsedURL = urlParser.parse(targetUrl) // const parsedURL = urlParser.parse(targetUrl)
// console.log(parsedURL) // console.log(parsedURL)
// get from cache // get from cach
if (cache[targetUrl]) {
return cache[targetUrl]
}
const { body: html, url } = await got(targetUrl) const { body: html, url } = await got(targetUrl)
const metadata = await metascraper({ html, url }) const metadata = await metascraper({ html, url })
// write to cache metadata.sources = ['resource']
cache[targetUrl] = metadata
return metadata return metadata
} }