Infrastructure/ghost
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2025-01-06 22:40:14 -05:00
Code Issues Activity

Added support for custom OEmbed providers

Browse source 
refs https://github.com/TryGhost/Team/issues/1211

In order to override the default OEmbeds for OpenSea NFT's we need a way
to provide out own OEmbed data. We will want this in future too for
custom Twitter embeds, so this has been built in a way which allows
extension.
This commit is contained in:
Fabien O'Carroll 2021-11-10 16:14:04 +02:00
parent 4b78586ad1
commit e69d112c77
2 changed files with 148 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

261
core/server/services/oembed.js
View file

@ -1,4 +1,3 @@
const Promise = require('bluebird');
const errors = require('@tryghost/errors');
const tpl = require('@tryghost/tpl');
const {extract, hasProvider} = require('oembed-parser');
@ -11,6 +10,10 @@ const messages = {
insufficientMetadata: 'URL contains insufficient metadata.'
};
/**
* @param {string} url
* @returns {{url: string, provider: boolean}}
*/
const findUrlWithProvider = (url) => {
let provider;
@ -44,6 +47,12 @@ const findUrlWithProvider = (url) => {
* @typedef {(url: string, config: Object) => Promise} IExternalRequest
*/
/**
* @typedef {object} ICustomProvider
* @prop {(url: URL) => Promise<boolean>} canSupportRequest
* @prop {(url: URL, externalRequest: IExternalRequest) => Promise<import('oembed-parser').OembedData>} getOEmbedData
*/
class OEmbed {
/**
*
@ -53,29 +62,62 @@ class OEmbed {
*/
constructor({config, externalRequest}) {
this.config = config;
this.externalRequest = externalRequest;
/** @type {IExternalRequest} */
this.externalRequest = async (url, requestConfig) => {
if (this.isIpOrLocalhost(url)) {
return this.unknownProvider(url);
}
const response = await externalRequest(url, requestConfig);
if (this.isIpOrLocalhost(response.url)) {
return this.unknownProvider(url);
}
return response;
};
/** @type {ICustomProvider[]} */
this.customProviders = [];
}
unknownProvider(url) {
return Promise.reject(new errors.ValidationError({
/**
* @param {ICustomProvider} provider
*/
registerProvider(provider) {
this.customProviders.push(provider);
}
/**
* @param {string} url
*/
async unknownProvider(url) {
throw new errors.ValidationError({
message: tpl(messages.unknownProvider),
context: url
}));
}
knownProvider(url) {
return extract(url).catch((err) => {
return Promise.reject(new errors.InternalServerError({
message: err.message
}));
});
}
/**
* @param {string} url
*/
async knownProvider(url) {
try {
return await extract(url);
} catch (err) {
throw new errors.InternalServerError({
message: err.message
});
}
}
/**
* @param {string} url
*/
errorHandler(url) {
return (err) => {
/**
* @param {Error|errors.GhostError} err
*/
return async (err) => {
// allow specific validation errors through for better error messages
if (errors.utils.isIgnitionError(err) && err.errorType === 'ValidationError') {
return Promise.reject(err);
throw err;
}
// default to unknown provider to avoid leaking any app specifics
@ -97,19 +139,11 @@ class OEmbed {
let scraperResponse;
try {
const cookieJar = new CookieJar();
const response = await this.externalRequest(url, {cookieJar});
const cookieJar = new CookieJar();
const response = await this.externalRequest(url, {cookieJar});
if (this.isIpOrLocalhost(response.url)) {
scraperResponse = {};
} else {
const html = response.body;
scraperResponse = await metascraper({html, url});
}
} catch (err) {
return Promise.reject(err);
}
const html = response.body;
scraperResponse = await metascraper({html, url});
const metadata = Object.assign({}, scraperResponse, {
thumbnail: scraperResponse.image,
@ -119,20 +153,25 @@ class OEmbed {
delete metadata.image;
delete metadata.logo;
if (metadata.title) {
return Promise.resolve({
type: 'bookmark',
url,
metadata
if (!metadata.title) {
throw new errors.ValidationError({
message: tpl(messages.insufficientMetadata),
context: url
});
}
return Promise.reject(new errors.ValidationError({
message: tpl(messages.insufficientMetadata),
context: url
}));
return {
version: '1.0',
type: 'bookmark',
url,
metadata
};
}
/**
* @param {string} url
* @returns {boolean}
*/
isIpOrLocalhost(url) {
try {
const IPV4_REGEX = /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
@ -163,14 +202,7 @@ class OEmbed {
*
* @returns {Promise<Object>}
*/
fetchOembedData(_url, cardType) {
// parse the url then validate the protocol and host to make sure it's
// http(s) and not an IP address or localhost to avoid potential access to
// internal network endpoints
if (this.isIpOrLocalhost(_url)) {
return this.unknownProvider();
}
async fetchOembedData(_url, cardType) {
// check against known oembed list
let {url, provider} = findUrlWithProvider(_url);
if (provider) {
@ -180,88 +212,81 @@ class OEmbed {
// url not in oembed list so fetch it in case it's a redirect or has a
// <link rel="alternate" type="application/json+oembed"> element
const cookieJar = new CookieJar();
return this.externalRequest(url, {
const pageResponse = await this.externalRequest(url, {
method: 'GET',
timeout: 2 * 1000,
followRedirect: true,
cookieJar
}).then((pageResponse) => {
// url changed after fetch, see if we were redirected to a known oembed
if (pageResponse.url !== url) {
({url, provider} = findUrlWithProvider(pageResponse.url));
if (provider) {
return this.knownProvider(url);
}
});
// url changed after fetch, see if we were redirected to a known oembed
if (pageResponse.url !== url) {
({url, provider} = findUrlWithProvider(pageResponse.url));
if (provider) {
return this.knownProvider(url);
}
}
// check for <link rel="alternate" type="application/json+oembed"> element
let oembedUrl;
try {
oembedUrl = cheerio('link[type="application/json+oembed"]', pageResponse.body).attr('href');
} catch (e) {
return this.unknownProvider(url);
}
if (oembedUrl) {
// for standard WP oembed's we want to insert a bookmark card rather than their blockquote+script
// which breaks in the editor and most Ghost themes. Only fallback if card type was not explicitly chosen
if (!cardType && oembedUrl.match(/wp-json\/oembed/)) {
return;
}
// check for <link rel="alternate" type="application/json+oembed"> element
let oembedUrl;
try {
oembedUrl = cheerio('link[type="application/json+oembed"]', pageResponse.body).attr('href');
} catch (e) {
return this.unknownProvider(url);
}
// fetch oembed response from embedded rel="alternate" url
const oembedResponse = await this.externalRequest(oembedUrl, {
method: 'GET',
json: true,
timeout: 2 * 1000,
followRedirect: true,
cookieJar
});
// validate the fetched json against the oembed spec to avoid
// leaking non-oembed responses
const body = oembedResponse.body;
const hasRequiredFields = body.type && body.version;
const hasValidType = ['photo', 'video', 'link', 'rich'].includes(body.type);
if (oembedUrl) {
// make sure the linked url is not an ip address or localhost
if (this.isIpOrLocalhost(oembedUrl)) {
return this.unknownProvider(oembedUrl);
if (hasRequiredFields && hasValidType) {
// extract known oembed fields from the response to limit leaking of unrecognised data
const knownFields = [
'type',
'version',
'html',
'url',
'title',
'width',
'height',
'author_name',
'author_url',
'provider_name',
'provider_url',
'thumbnail_url',
'thumbnail_width',
'thumbnail_height'
];
const oembed = _.pick(body, knownFields);
// ensure we have required data for certain types
if (oembed.type === 'photo' && !oembed.url) {
return;
}
// for standard WP oembed's we want to insert a bookmark card rather than their blockquote+script
// which breaks in the editor and most Ghost themes. Only fallback if card type was not explicitly chosen
if (!cardType && oembedUrl.match(/wp-json\/oembed/)) {
if ((oembed.type === 'video' || oembed.type === 'rich') && (!oembed.html || !oembed.width || !oembed.height)) {
return;
}
// fetch oembed response from embedded rel="alternate" url
return this.externalRequest(oembedUrl, {
method: 'GET',
json: true,
timeout: 2 * 1000,
followRedirect: true,
cookieJar
}).then((oembedResponse) => {
// validate the fetched json against the oembed spec to avoid
// leaking non-oembed responses
const body = oembedResponse.body;
const hasRequiredFields = body.type && body.version;
const hasValidType = ['photo', 'video', 'link', 'rich'].includes(body.type);
if (hasRequiredFields && hasValidType) {
// extract known oembed fields from the response to limit leaking of unrecognised data
const knownFields = [
'type',
'version',
'html',
'url',
'title',
'width',
'height',
'author_name',
'author_url',
'provider_name',
'provider_url',
'thumbnail_url',
'thumbnail_width',
'thumbnail_height'
];
const oembed = _.pick(body, knownFields);
// ensure we have required data for certain types
if (oembed.type === 'photo' && !oembed.url) {
return;
}
if ((oembed.type === 'video' || oembed.type === 'rich') && (!oembed.html || !oembed.width || !oembed.height)) {
return;
}
// return the extracted object, don't pass through the response body
return oembed;
}
}).catch(() => {});
// return the extracted object, don't pass through the response body
return oembed;
}
});
}
}
/**
@ -274,6 +299,16 @@ class OEmbed {
let data;
try {
const urlObject = new URL(url);
for (const provider of this.customProviders) {
if (await provider.canSupportRequest(urlObject)) {
const result = await provider.getOEmbedData(urlObject, this.externalRequest);
if (result !== null) {
return result;
}
}
}
if (type === 'bookmark') {
return this.fetchBookmarkData(url);
}

1
test/e2e-api/admin/oembed.test.js
View file

@ -149,7 +149,6 @@ describe('Oembed API', function () {
pageMock.isDone().should.be.true();
should.exist(res.body.errors);
res.body.errors[0].context.should.match(/insufficient metadata/i);
});
});