Added mention discovery service (#16154)

fixes https://github.com/TryGhost/Team/issues/2407

The MentionDiscoveryService fetches mentioned sites to return the webmention endpoint.

ghost/core/core/server/services/mentions/service.js

@@ -3,7 +3,8 @@ const WebmentionMetadata = require('./WebmentionMetadata');
 const {
     InMemoryMentionRepository,
     MentionsAPI,
-    MentionSendingService
+    MentionSendingService,
+    MentionDiscoveryService
 } = require('@tryghost/webmentions');
 const events = require('../../lib/common/events');
 const externalRequest = require('../../../server/lib/request-external.js');
@@ -21,6 +22,7 @@ module.exports = {
     async init() {
         const repository = new InMemoryMentionRepository();
         const webmentionMetadata = new WebmentionMetadata();
+        const discoveryService = new MentionDiscoveryService({externalRequest});
         const api = new MentionsAPI({
             repository,
             webmentionMetadata,
@@ -66,11 +68,7 @@ module.exports = {
         });
 
         const sendingService = new MentionSendingService({
-            discoveryService: {
+            discoveryService,
-                getEndpoint: async () => {
-                    return new URL('https://site.ghost/webmentions/receive');
-                }
-            },
             externalRequest,
             getSiteUrl: () => urlUtils.urlFor('home', true),
             getPostUrl: post => getPostUrl(post),

ghost/webmentions/lib/MentionDiscoveryService.js

@@ -0,0 +1,73 @@
+const cheerio = require('cheerio');
+const errors = require('@tryghost/errors');
+
+module.exports = class MentionDiscoveryService {
+    #externalRequest;
+
+    constructor({externalRequest}) {
+        this.#externalRequest = externalRequest;
+    }
+
+    /**
+     * Fetches the given URL to identify the webmention endpoint
+     * @param {URL} url
+     * @returns {Promise<URL|null>}
+     */
+    async getEndpoint(url) {
+        try {
+            const response = await this.#externalRequest(url.href, {
+                throwHttpErrors: false,
+                followRedirects: true,
+                maxRedirects: 10
+            });
+            if (response.statusCode === 404) {
+                throw new errors.BadRequestError({
+                    message: 'Webmention discovery service could not find target site',
+                    statusCode: response.statusCode
+                });
+            }
+            return this.getEndpointFromResponse(response);
+        } catch (error) {
+            return null;
+        }
+    }
+
+    /**
+     * @private
+     * Parses the given response for the first webmention endpoint
+     * @param {Object} response
+     * @returns {Promise<URL|null>}
+     */
+    async getEndpointFromResponse(response) {
+        let href;
+        let endpoint;
+        // Link: <uri-reference>; param1=value1; param2="value2"
+        // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Link
+        const linkHeader = response.headers.link;
+        if (linkHeader && linkHeader.includes('rel="webmention"')) {
+            linkHeader.split(',').forEach((p) => {
+                if (p.includes('rel="webmention"')) {
+                    href = p.substring(p.indexOf('<') + 1, p.indexOf('>'));
+                    endpoint = new URL(href);
+                    return;
+                }
+            });
+        }
+        if (endpoint) {
+            return endpoint;
+        }
+
+        // must be html to find links/tags
+        if (!response.headers['content-type'].includes('text/html')) {
+            return null;
+        }
+
+        const $ = cheerio.load(response.body);
+
+        // must be first <link> OR <a> element with rel=webmention
+        href = $('a[rel="webmention"],link[rel="webmention"]').first().attr('href');
+
+        endpoint = href ? new URL(href) : null;
+        return endpoint;
+    }
+};

ghost/webmentions/lib/webmentions.js

@@ -1,4 +1,5 @@
 module.exports.InMemoryMentionRepository = require('./InMemoryMentionRepository');
 module.exports.MentionsAPI = require('./MentionsAPI');
+module.exports.MentionDiscoveryService = require('./MentionDiscoveryService');
 module.exports.Mention = require('./Mention');
 module.exports.MentionSendingService = require('./MentionSendingService');

ghost/webmentions/package.json

@@ -21,11 +21,12 @@
     "c8": "7.12.0",
     "mocha": "10.2.0",
     "nock": "13.3.0",
-    "sinon": "15.0.1",
+    "bson-objectid": "2.0.4",
-    "bson-objectid": "2.0.4"
+    "sinon": "15.0.1"
   },
   "dependencies": {
     "@tryghost/errors": "1.2.20",
-    "@tryghost/logging": "2.3.6"
+    "@tryghost/logging": "2.3.6",
+    "cheerio": "0.22.0"
   }
 }

ghost/webmentions/test/MentionDiscoveryService.test.js

@@ -0,0 +1,180 @@
+const MentionDiscoveryService = require('../lib/MentionDiscoveryService');
+// non-standard to use externalRequest here, but this is required for the overrides in the libary, which we want to test for security reasons in combination with the package
+const externalRequest = require('../../core/core/server/lib/request-external.js');
+const assert = require('assert');
+const nock = require('nock');
+
+describe('MentionDiscoveryService', function () {
+    const service = new MentionDiscoveryService({externalRequest});
+
+    it('Returns null from a bad URL', async function () {
+        const url = new URL('http://www.fake.com/');
+        nock(url.href)
+            .get('/')
+            .reply(404);
+        let endpoint = await service.getEndpoint(url);
+
+        assert.equal(endpoint, null);
+    });
+
+    // TODO: need to support redirects
+    // it('Follows redirects', async function () {
+
+    //     let url = new URL('http://redirector.io/');
+    //     let nextUrl = new URL('http://testpage.com/');
+
+    //     let mockRedirect = nock(url.href)
+    //         .intercept("/", "HEAD")
+    //         .reply(301, undefined, { location: nextUrl.href })
+    //         .get('/')
+    //         .reply(200, '<link rel="webmention" href="http://valid.site.org" />Very cool site', { 'content-type': 'text/html' });
+
+    //     let endpoint = await service.getEndpoint(url);
+
+    //     assert(endpoint instanceof URL);
+    // });
+
+    describe('Can parse headers', function () {
+        it('Returns null for a valid non-html site', async function () {
+            const url = new URL('http://www.veryrealsite.com');
+            nock(url.href)
+                .get('/')
+                .reply(200, {}, {'content-type': 'application/json'});
+            const endpoint = await service.getEndpoint(url);
+
+            assert.equal(endpoint, null);
+        });
+
+        it('Returns an endpoint from a site with a webmentions Link in the header', async function () {
+            const url = new URL('http://testpage.com/');
+            nock(url.href)
+                .get('/')
+                .reply(200, {}, {Link: '<http://webmentions.endpoint.io>; rel="webmention"'});
+            const endpoint = await service.getEndpoint(url);
+
+            assert(endpoint instanceof URL);
+            assert.equal(endpoint, 'http://webmentions.endpoint.io/');
+        });
+
+        it('Returns null with Links in the header that are not for webmentions', async function () {
+            const url = new URL('http://testpage.com/');
+            nock(url.href)
+                .get('/')
+                .reply(200, {}, {Link: '<http://not.your.endpoint>; rel="preconnect"'});
+            const endpoint = await service.getEndpoint(url);
+
+            assert.equal(endpoint, null);
+        });
+
+        it('Returns with multiple Links in the header, one of which is for webmentions', async function () {
+            const url = new URL('http://testpage.com/');
+            nock(url.href)
+                .get('/')
+                .reply(200, {}, {Link: '<http://not.your.endpoint>; rel="preconnect",<http://webmentions.endpoint.io>; rel="webmention"'});
+            const endpoint = await service.getEndpoint(url);
+
+            assert(endpoint instanceof URL);
+            assert.equal(endpoint, 'http://webmentions.endpoint.io/');
+        });
+    });
+
+    describe('Can parse html', function () {
+        it('Returns endpoint for valid html site with <link rel="webmention"> tag in body', async function () {
+            const url = new URL('http://testpage.com/');
+            nock(url.href)
+                .get('/')
+                .reply(200, '<link rel="webmention" href="http://webmentions.endpoint.io" />', {'content-type': 'text/html'});
+            const endpoint = await service.getEndpoint(url);
+
+            assert(endpoint instanceof URL);
+            assert.equal(endpoint, 'http://webmentions.endpoint.io/');
+        });
+
+        it('Returns endpoint for valid html site with <a rel="webmention"> tag in body', async function () {
+            const url = new URL('http://testpage.com/');
+            nock(url.href)
+                .get('/')
+                .reply(200, '<a rel="webmention" href="http://valid.site.org">webmention</a>', {'content-type': 'text/html'});
+            const endpoint = await service.getEndpoint(url);
+
+            assert(endpoint instanceof URL);
+            assert.equal(endpoint, 'http://valid.site.org/');
+        });
+
+        it('Returns first endpoint for valid html site with multiple <a> tags in body', async function () {
+            const url = new URL('http://testpage.com/');
+            const html = `
+                    <a rel="bookmark" href="http://not.an.endpoint">kewl link 1</a>
+                    <a rel="webmention" href="http://first.webmention.endpoint">kewl link 2</a>
+                    <a rel="webmention" href="http://second.webmention.endpoint">kewl link 3</a>
+                    <a rel="icon" href="http://not.an.endpoint">kewl link 4</a>
+                `;
+            nock(url.href)
+                .get('/')
+                .reply(200, html, {'content-type': 'text/html'});
+            const endpoint = await service.getEndpoint(url);
+
+            assert(endpoint instanceof URL);
+            assert.equal(endpoint.href, 'http://first.webmention.endpoint/');
+        });
+
+        it('Returns first endpoint for valid html site with multiple <link> tags in the header', async function () {
+            const url = new URL('http://testpage.com/');
+            const html = `
+                    <link rel="bookmark" href="http://not.an.endpoint" />
+                    <link rel="webmention" href="http://first.webmention.endpoint" />
+                    <link rel="webmention" href="http://second.webmention.endpoint" />
+                `;
+            nock(url.href)
+                .get('/')
+                .reply(200, html, {'content-type': 'text/html'});
+            const endpoint = await service.getEndpoint(url);
+
+            assert(endpoint instanceof URL);
+            assert.equal(endpoint.href, 'http://first.webmention.endpoint/');
+        });
+
+        it('Ignores link without href', async function () {
+            const url = new URL('http://testpage.com/');
+            const html = `
+                    <link rel="webmention" />
+                `;
+            nock(url.href)
+                .get('/')
+                .reply(200, html, {'content-type': 'text/html'});
+            const endpoint = await service.getEndpoint(url);
+
+            assert.equal(endpoint, null);
+        });
+
+        it('Returns first endpoint for valid html site with multiple <link> and <a> tags', async function () {
+            // note - link tags are in the header and should come first
+            const url = new URL('http://testpage.com/');
+            const html = `
+                    <link rel="bookmark" href="http://not.an.endpoint" />
+                    <link rel="webmention" href="http://first.link.endpoint" />
+                    <link rel="webmention" href="http://second.link.endpoint" />
+                    <a rel="bookmark" href="http://not.an.endpoint">kewl link 1</a>
+                    <a rel="webmention" href="http://first.a.endpoint">kewl link 2</a>
+                `;
+            nock(url.href)
+                .get('/')
+                .reply(200, html, {'content-type': 'text/html'});
+
+            const endpoint = await service.getEndpoint(url);
+
+            assert(endpoint instanceof URL);
+            assert.equal(endpoint.href, 'http://first.link.endpoint/');
+        });
+
+        it('Returns null for a valid html site with no endpoint', async function () {
+            const url = new URL('http://www.veryrealsite.com');
+            nock(url.href)
+                .get('/')
+                .reply(200, {}, {'content-type': 'text/html'});
+            const endpoint = await service.getEndpoint(url);
+
+            assert.equal(endpoint, null);
+        });
+    });
+});

ghost/webmentions/test/MentionSendingService.test.js

@@ -1,4 +1,4 @@
-const MentionSendingService = require('../lib/MentionSendingService.js');
+const {MentionSendingService} = require('../');
 const assert = require('assert');
 const nock = require('nock');
 // non-standard to use externalRequest here, but this is required for the overrides in the libary, which we want to test for security reasons in combination with the package

ghost/webmentions/test/hello.test.js

@@ -1,8 +0,0 @@
-const assert = require('assert');
-
-describe('Hello world', function () {
-    it('Runs a test', function () {
-        // TODO: Write me!
-        assert.ok(require('../index'));
-    });
-});