diff --git a/ghost/core/core/server/services/mentions/WebmentionMetadata.js b/ghost/core/core/server/services/mentions/WebmentionMetadata.js index 42df7eef5f..a91586132d 100644 --- a/ghost/core/core/server/services/mentions/WebmentionMetadata.js +++ b/ghost/core/core/server/services/mentions/WebmentionMetadata.js @@ -34,7 +34,13 @@ module.exports = class WebmentionMetadata { */ async fetch(url) { const mappedUrl = this.#getMappedUrl(url); - const data = await oembedService.fetchOembedDataFromUrl(mappedUrl.href, 'mention'); + const data = await oembedService.fetchOembedDataFromUrl(mappedUrl.href, 'mention', { + timeout: 15000, + retry: { + // Only retry on network issues, or specific HTTP status codes + limit: 3 + } + }); const result = { siteTitle: data.metadata.publisher, @@ -50,7 +56,13 @@ module.exports = class WebmentionMetadata { if (mappedUrl.href !== url.href) { // Still need to fetch body and contentType separately now // For verification - const {body, contentType} = await oembedService.fetchPageHtml(url); + const {body, contentType} = await oembedService.fetchPageHtml(url, { + timeout: 15000, + retry: { + // Only retry on network issues, or specific HTTP status codes + limit: 3 + } + }); result.body = body; result.contentType = contentType; } diff --git a/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js b/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js index dc5bab43df..88139781e3 100644 --- a/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js +++ b/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js @@ -45,7 +45,7 @@ class RecommendationServiceWrapper { const mentions = require('../mentions'); - if (!mentions.sendingService) { + if (!mentions.sendingService || !mentions.api) { // eslint-disable-next-line ghost/ghost-custom/no-native-error throw new Error('MentionSendingService not intialized, but this is a dependency of RecommendationServiceWrapper. Check boot order.'); } @@ -75,7 +75,8 @@ class RecommendationServiceWrapper { wellknownService, mentionSendingService: mentions.sendingService, clickEventRepository: this.clickEventRepository, - subscribeEventRepository: this.subscribeEventRepository + subscribeEventRepository: this.subscribeEventRepository, + mentionsApi: mentions.api }); this.controller = new RecommendationController({ service: this.service diff --git a/ghost/i18n/locales/hu/comments.json b/ghost/i18n/locales/hu/comments.json index 85d72208f3..40fdffd059 100644 --- a/ghost/i18n/locales/hu/comments.json +++ b/ghost/i18n/locales/hu/comments.json @@ -29,7 +29,7 @@ "Expertise": "Foglalkozás", "Founder @ Acme Inc": "Acme Kft. alapító", "Full-time parent": "Főállású szülő", - "Head of Marketing at Acme, Inc": "Marketing vezető — Acme Kft.", + "Head of Marketing at Acme, Inc": "Marketing vezető —\u00a0Acme Kft.", "Hide": "Elrejtés", "Hide comment": "Hozzászólás elrejtése", "Jamie Larson": "Kiss Sára", diff --git a/ghost/i18n/locales/hu/ghost.json b/ghost/i18n/locales/hu/ghost.json index 4a4e52b086..2b509de7fc 100644 --- a/ghost/i18n/locales/hu/ghost.json +++ b/ghost/i18n/locales/hu/ghost.json @@ -1,6 +1,6 @@ { "All the best!": "Üdvözlettel,", - "Complete signup for {{siteTitle}}!": "{{siteTitle}} — Regisztráció", + "Complete signup for {{siteTitle}}!": "{{siteTitle}} —\u00a0Regisztráció", "Complete your sign up to {{siteTitle}}!": "{{siteTitle}} — Regisztráció", "Confirm email address": "Kérjük hagyja jóvá meg email címét", "Confirm signup": "Regisztráció jóváhagyása", diff --git a/ghost/oembed-service/lib/OEmbedService.js b/ghost/oembed-service/lib/OEmbedService.js index 3834f3d931..5255e8eb28 100644 --- a/ghost/oembed-service/lib/OEmbedService.js +++ b/ghost/oembed-service/lib/OEmbedService.js @@ -131,17 +131,19 @@ class OEmbedService { /** * @param {string} url + * @param {Object} options * * @returns {Promise<{url: string, body: string, contentType: string|undefined}>} */ - async fetchPageHtml(url) { + async fetchPageHtml(url, options = {}) { // Fetch url and get response as binary buffer to // avoid implicit cast let {headers, body, url: responseUrl} = await this.fetchPage( url, { encoding: 'binary', - responseType: 'buffer' + responseType: 'buffer', + ...options }); try { @@ -328,10 +330,12 @@ class OEmbedService { /** * @param {string} url - oembed URL * @param {string} type - card type + * @param {Object} [options] Specific fetch options + * @param {number} [options.timeout] Change the default timeout for fetching html * * @returns {Promise} */ - async fetchOembedDataFromUrl(url, type) { + async fetchOembedDataFromUrl(url, type, options = {}) { try { const urlObject = new URL(url); @@ -358,7 +362,7 @@ class OEmbedService { } // Not in the list, we need to fetch the content - const {url: pageUrl, body, contentType} = await this.fetchPageHtml(url); + const {url: pageUrl, body, contentType} = await this.fetchPageHtml(url, options); // fetch only bookmark when explicitly requested if (type === 'bookmark') { diff --git a/ghost/recommendations/src/RecommendationService.ts b/ghost/recommendations/src/RecommendationService.ts index b127c0d367..fdbfbd82c9 100644 --- a/ghost/recommendations/src/RecommendationService.ts +++ b/ghost/recommendations/src/RecommendationService.ts @@ -6,6 +6,7 @@ import errors from '@tryghost/errors'; import tpl from '@tryghost/tpl'; import {ClickEvent} from './ClickEvent'; import {SubscribeEvent} from './SubscribeEvent'; +import logging from '@tryghost/logging'; export type RecommendationIncludeTypes = { 'count.clicks': number, @@ -26,6 +27,10 @@ type MentionSendingService = { sendAll(options: {url: URL, links: URL[]}): Promise } +type MentionsAPI = { + refreshMentions(options: {filter: string, limit: number|'all'}): Promise +} + type RecommendationEnablerService = { getSetting(): string, setSetting(value: string): Promise @@ -43,6 +48,7 @@ export class RecommendationService { wellknownService: WellknownService; mentionSendingService: MentionSendingService; recommendationEnablerService: RecommendationEnablerService; + mentionsApi: MentionsAPI; constructor(deps: { repository: RecommendationRepository, @@ -51,6 +57,7 @@ export class RecommendationService { wellknownService: WellknownService, mentionSendingService: MentionSendingService, recommendationEnablerService: RecommendationEnablerService, + mentionsApi: MentionsAPI }) { this.repository = deps.repository; this.wellknownService = deps.wellknownService; @@ -58,11 +65,31 @@ export class RecommendationService { this.recommendationEnablerService = deps.recommendationEnablerService; this.clickEventRepository = deps.clickEventRepository; this.subscribeEventRepository = deps.subscribeEventRepository; + this.mentionsApi = deps.mentionsApi; } async init() { const recommendations = await this.#listRecommendations(); await this.updateWellknown(recommendations); + + // When we boot, it is possible that we missed some webmentions from other sites recommending you + // More importantly, we might have missed some deletes which we can detect. + // So we do a slow revalidation of all incoming recommendations + // This also prevents doing multiple external fetches when doing quick reboots of Ghost after each other (requires Ghost to be up for at least 15 seconds) + if (!process.env.NODE_ENV?.startsWith('test')) { + setTimeout(() => { + logging.info('Updating incoming recommendations on boot'); + this.#updateIncomingRecommendations().catch((err) => { + logging.error('Failed to update incoming recommendations on boot', err); + }); + }, 15 * 1000 + Math.random() * 5 * 60 * 1000); + } + } + + async #updateIncomingRecommendations() { + // Note: we also recheck recommendations that were not verified (verification could have failed) + const filter = `source:~$'/.well-known/recommendations.json'`; + await this.mentionsApi.refreshMentions({filter, limit: 100}); } async updateWellknown(recommendations: Recommendation[]) { @@ -86,7 +113,9 @@ export class RecommendationService { links: [ recommendation.url ] - }).catch(console.error); // eslint-disable-line no-console + }).catch((err) => { + logging.error('Failed to send mention to recommendation', err); + }); } async readRecommendation(id: string): Promise { diff --git a/ghost/webmentions/lib/Mention.js b/ghost/webmentions/lib/Mention.js index fc1600c53d..7c97edc9fe 100644 --- a/ghost/webmentions/lib/Mention.js +++ b/ghost/webmentions/lib/Mention.js @@ -19,6 +19,17 @@ module.exports = class Mention { return this.#verified; } + /** @type {boolean} */ + #deleted = false; + + get deleted() { + return this.#deleted; + } + + delete() { + this.#deleted = true; + } + /** * @param {string} html * @param {string} contentType @@ -177,11 +188,6 @@ module.exports = class Mention { this.#sourceFeaturedImage = sourceFeaturedImage; } - #deleted = false; - delete() { - this.#deleted = true; - } - toJSON() { return { id: this.id, diff --git a/ghost/webmentions/lib/MentionDiscoveryService.js b/ghost/webmentions/lib/MentionDiscoveryService.js index 70185c0ac8..991d578116 100644 --- a/ghost/webmentions/lib/MentionDiscoveryService.js +++ b/ghost/webmentions/lib/MentionDiscoveryService.js @@ -1,5 +1,4 @@ const cheerio = require('cheerio'); -const errors = require('@tryghost/errors'); module.exports = class MentionDiscoveryService { #externalRequest; @@ -16,16 +15,15 @@ module.exports = class MentionDiscoveryService { async getEndpoint(url) { try { const response = await this.#externalRequest(url.href, { - throwHttpErrors: false, + throwHttpErrors: true, followRedirect: true, - maxRedirects: 10 + maxRedirects: 10, + timeout: 15000, + retry: { + // Only retry on network issues, or specific HTTP status codes + limit: 3 + } }); - if (response.statusCode === 404) { - throw new errors.BadRequestError({ - message: 'Webmention discovery service could not find target site', - statusCode: response.statusCode - }); - } return this.getEndpointFromResponse(response); } catch (error) { return null; diff --git a/ghost/webmentions/lib/MentionSendingService.js b/ghost/webmentions/lib/MentionSendingService.js index 22fc3dfbc8..7151cf68e3 100644 --- a/ghost/webmentions/lib/MentionSendingService.js +++ b/ghost/webmentions/lib/MentionSendingService.js @@ -97,7 +97,11 @@ module.exports = class MentionSendingService { throwHttpErrors: false, maxRedirects: 10, followRedirect: true, - timeout: 10000 + timeout: 15000, + retry: { + // Only retry on network issues, or specific HTTP status codes + limit: 3 + } }); if (response.statusCode >= 200 && response.statusCode < 300) { diff --git a/ghost/webmentions/lib/MentionsAPI.js b/ghost/webmentions/lib/MentionsAPI.js index ad952d3eca..165d918d3b 100644 --- a/ghost/webmentions/lib/MentionsAPI.js +++ b/ghost/webmentions/lib/MentionsAPI.js @@ -163,6 +163,101 @@ module.exports = class MentionsAPI { return page; } + /** + * Update the metadata of the webmentions in the database, and delete them if they are no longer valid. + * @param {object} options + * @param {number|'all'} [options.limit] + * @param {number} [options.page] + * @param {string} [options.filter] + */ + async refreshMentions(options) { + const mentions = await this.#repository.getAll(options); + + for (const mention of mentions) { + await this.#updateWebmention(mention, { + source: mention.source, + target: mention.target + }); + } + } + + async #updateWebmention(mention, webmention) { + const isNew = !mention; + const targetExists = await this.#routingService.pageExists(webmention.target); + + if (!targetExists) { + if (!mention) { + throw new errors.BadRequestError({ + message: `${webmention.target} is not a valid URL for this site.` + }); + } else { + mention.delete(); + } + } + + if (targetExists) { + const resourceInfo = await this.#resourceService.getByURL(webmention.target); + let metadata; + try { + metadata = await this.#webmentionMetadata.fetch(webmention.source); + if (mention) { + mention.setSourceMetadata({ + sourceTitle: metadata.title, + sourceSiteTitle: metadata.siteTitle, + sourceAuthor: metadata.author, + sourceExcerpt: metadata.excerpt, + sourceFavicon: metadata.favicon, + sourceFeaturedImage: metadata.image + }); + } + } catch (err) { + if (!mention) { + throw err; + } + mention.delete(); + } + + if (!mention) { + mention = await Mention.create({ + source: webmention.source, + target: webmention.target, + timestamp: new Date(), + payload: webmention.payload, + resourceId: resourceInfo.id ? resourceInfo.id.toHexString() : null, + resourceType: resourceInfo.type, + sourceTitle: metadata.title, + sourceSiteTitle: metadata.siteTitle, + sourceAuthor: metadata.author, + sourceExcerpt: metadata.excerpt, + sourceFavicon: metadata.favicon, + sourceFeaturedImage: metadata.image + }); + } + + if (metadata?.body) { + try { + mention.verify(metadata.body, metadata.contentType); + } catch (e) { + logging.error(e); + } + } + } + + await this.#repository.save(mention); + + if (isNew) { + logging.info('[Webmention] Created ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified); + } else { + if (mention.deleted) { + logging.info('[Webmention] Deleted ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified); + } else { + logging.info('[Webmention] Updated ' + webmention.source + ' to ' + webmention.target + ', verified: ' + mention.verified); + } + } + + return mention; + } + /** * @param {object} webmention * @param {URL} webmention.source @@ -177,65 +272,6 @@ module.exports = class MentionsAPI { webmention.target ); - const targetExists = await this.#routingService.pageExists(webmention.target); - - if (!targetExists) { - if (!mention) { - throw new errors.BadRequestError({ - message: `${webmention.target} is not a valid URL for this site.` - }); - } else { - mention.delete(); - } - } - - const resourceInfo = await this.#resourceService.getByURL(webmention.target); - let metadata; - try { - metadata = await this.#webmentionMetadata.fetch(webmention.source); - if (mention) { - mention.setSourceMetadata({ - sourceTitle: metadata.title, - sourceSiteTitle: metadata.siteTitle, - sourceAuthor: metadata.author, - sourceExcerpt: metadata.excerpt, - sourceFavicon: metadata.favicon, - sourceFeaturedImage: metadata.image - }); - } - } catch (err) { - if (!mention) { - throw err; - } - mention.delete(); - } - - if (!mention) { - mention = await Mention.create({ - source: webmention.source, - target: webmention.target, - timestamp: new Date(), - payload: webmention.payload, - resourceId: resourceInfo.id ? resourceInfo.id.toHexString() : null, - resourceType: resourceInfo.type, - sourceTitle: metadata.title, - sourceSiteTitle: metadata.siteTitle, - sourceAuthor: metadata.author, - sourceExcerpt: metadata.excerpt, - sourceFavicon: metadata.favicon, - sourceFeaturedImage: metadata.image - }); - } - - if (metadata?.body) { - try { - mention.verify(metadata.body, metadata.contentType); - } catch (e) { - logging.error(e); - } - } - - await this.#repository.save(mention); - return mention; + return await this.#updateWebmention(mention, webmention); } }; diff --git a/ghost/webmentions/test/MentionsAPI.test.js b/ghost/webmentions/test/MentionsAPI.test.js index de53f4d87b..037d5ff0ab 100644 --- a/ghost/webmentions/test/MentionsAPI.test.js +++ b/ghost/webmentions/test/MentionsAPI.test.js @@ -19,6 +19,7 @@ const mockResourceService = { }; } }; + const mockWebmentionMetadata = { async fetch() { return { @@ -223,6 +224,52 @@ describe('MentionsAPI', function () { assert(page.data[1].id === mentionTwo.id, 'Second mention should be the second one in ascending order'); }); + it('Can update recommendations', async function () { + const repository = new InMemoryMentionRepository(); + const api = new MentionsAPI({ + repository, + routingService: mockRoutingService, + resourceService: mockResourceService, + webmentionMetadata: { + fetch: sinon.stub() + .onFirstCall().resolves(mockWebmentionMetadata.fetch()) + .onSecondCall().resolves(mockWebmentionMetadata.fetch()) + .onThirdCall().resolves(mockWebmentionMetadata.fetch()) + .onCall(3).rejects() + } + }); + + await api.processWebmention({ + source: new URL('https://source.com'), + target: new URL('https://target.com'), + payload: {} + }); + + sinon.useFakeTimers(addMinutes(new Date(), 10).getTime()); + + await api.processWebmention({ + source: new URL('https://source2.com'), + target: new URL('https://target.com'), + payload: {} + }); + + let page = await api.listMentions({ + limit: 'all' + }); + assert(page.meta.pagination.total === 2); + + // Now we invalidate the second mention + + await api.refreshMentions({ + limit: 'all' + }); + + page = await api.listMentions({ + limit: 'all' + }); + assert(page.meta.pagination.total === 1); + }); + it('Can handle updating mentions', async function () { const repository = new InMemoryMentionRepository(); const api = new MentionsAPI({