ocelot-Social-Community/Ocelot-Social
1
0
Fork 0
mirror of https://github.com/Ocelot-Social-Community/Ocelot-Social.git synced 2025-12-13 07:46:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove metascraper-logo-favicon altogether

Browse Source 
It's doing http requests which are hard to test and hard to maintain.
Therefore let's remove it until we really need it.
This commit is contained in:
Robert Schäfer 2019-07-21 22:20:44 +02:00
parent 75743cae32
commit 3d7a30d419
5 changed files with 12 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backend/package.json
View File

@ -81,7 +81,6 @@
"metascraper-lang": "^4.8.5",
"metascraper-lang-detector": "^4.8.5",
"metascraper-logo": "^5.5.0",
"metascraper-logo-favicon": "^4.8.5",
"metascraper-publisher": "^4.8.5",
"metascraper-soundcloud": "^5.5.3",
"metascraper-title": "^4.8.5",

2
backend/src/schema/resolvers/embeds.spec.js
View File

@ -58,7 +58,6 @@ describe('Query', () => {
audio
video
lang
logo
sources
html
}
@ -92,7 +91,6 @@ describe('Query', () => {
audio: null,
video: null,
lang: 'de',
logo: 'https://www.youtube.com/favicon.ico',
sources: ['resource', 'oembed'],
html:
'<iframe width="480" height="270" src="https://www.youtube.com/embed/qkdXAtO40Fo?start=18&feature=oembed" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>',

25
backend/src/schema/resolvers/embeds/scraper.js
View File

@ -16,7 +16,6 @@ const metascraper = Metascraper([
require('metascraper-lang')(),
require('metascraper-lang-detector')(),
require('metascraper-logo')(),
require('metascraper-logo-favicon')(),
// require('metascraper-clearbit-logo')(),
require('metascraper-publisher')(),
require('metascraper-title')(),
@ -37,16 +36,16 @@ oEmbedProvidersFile = oEmbedProvidersFile.replace('{format}', 'json')
const oEmbedProviders = JSON.parse(oEmbedProvidersFile)
const fetchEmbed = async targetUrl => {
const fetchEmbed = async url => {
const provider = oEmbedProviders.find(provider => {
return provider.provider_url.includes(targetUrl.hostname)
return provider.provider_url.includes(url.hostname)
})
if (!provider) return {}
const {
endpoints: [endpoint],
} = provider
const endpointUrl = new URL(endpoint.url)
endpointUrl.searchParams.append('url', targetUrl)
endpointUrl.searchParams.append('url', url)
endpointUrl.searchParams.append('format', 'json')
const response = await fetch(endpointUrl)
const json = await response.json()
@ -61,26 +60,26 @@ const fetchEmbed = async targetUrl => {
return { type, html, author: author_name, date: upload_date, sources }
}
const fetchMeta = async targetUrl => {
const response = await fetch(targetUrl)
const fetchMeta = async url => {
const response = await fetch(url)
const html = await response.text()
const metadata = await metascraper({ html, url: targetUrl })
const metadata = await metascraper({ html, url })
return {
sources: ['resource'],
type: 'link',
sources: ['resource'],
...metadata,
}
}
export default async function scrape(targetUrl) {
targetUrl = new URL(targetUrl)
if (targetUrl.hostname === 'youtu.be') {
export default async function scrape(url) {
url = new URL(url)
if (url.hostname === 'youtu.be') {
// replace youtu.be to get proper results
targetUrl.hostname = 'youtube.com'
url.hostname = 'youtube.com'
}
const [meta, embed] = await Promise.all([fetchMeta(targetUrl), fetchEmbed(targetUrl)])
const [meta, embed] = await Promise.all([fetchMeta(url), fetchEmbed(url)])
const output = mergeWith(meta, embed, (objValue, srcValue) => {
if (isArray(objValue)) {

1
backend/src/schema/types/embed.gql
View File

@ -10,7 +10,6 @@ type Embed {
audio: String
video: String
lang: String
logo: String
html: String
sources: [String]
}

9
backend/yarn.lock
View File

@ -5691,15 +5691,6 @@ metascraper-lang@^4.8.5:
dependencies:
"@metascraper/helpers" "^4.10.2"
metascraper-logo-favicon@^4.8.5:
version "4.10.2"
resolved "https://registry.yarnpkg.com/metascraper-logo-favicon/-/metascraper-logo-favicon-4.10.2.tgz#b85176ee72b6d066a398661cc418140b7935deb7"
integrity sha512-CKdeaeIChTqRKQmd9pXSax9Us3bqWGdhKGkU2drgKptrcigsudjEp2rTyv5lDGDeMHLLD21xx7KuYr/R0Ysxtw==
dependencies:
"@metascraper/helpers" "^4.10.2"
got "~9.6.0"
lodash "~4.17.11"
metascraper-logo@^5.5.0:
version "5.6.2"
resolved "https://registry.yarnpkg.com/metascraper-logo/-/metascraper-logo-5.6.2.tgz#2677d3d00dc716cf8a62741e0b1770e93c201439"