Improved validation of fetched urls and responses in canary oembed endpoint

refs 5efef45dd0

- backports security fixes implemented in https://github.com/TryGhost/Ghost/commit/477393967 from v3 endpoint in Ghost 3.x to the Ghost 2.x canary endpoint

core/server/api/canary/oembed.js

@@ -3,27 +3,36 @@ const {extract, hasProvider} = require('oembed-parser');
 const Promise = require('bluebird');
 const request = require('../../lib/request');
 const cheerio = require('cheerio');
-const metascraper = require('metascraper')([
-    require('metascraper-url')(),
-    require('metascraper-title')(),
-    require('metascraper-description')(),
-    require('metascraper-author')(),
-    require('metascraper-publisher')(),
-    require('metascraper-image')(),
-    require('metascraper-logo-favicon')(),
-    require('metascraper-logo')()
-]);
+const _ = require('lodash');
 
 async function fetchBookmarkData(url, html) {
-    if (!html) {
-        const response = await request(url, {
-            headers: {
-                'user-agent': 'Ghost(https://github.com/TryGhost/Ghost)'
-            }
-        });
-        html = response.body;
+    const metascraper = require('metascraper')([
+        require('metascraper-url')(),
+        require('metascraper-title')(),
+        require('metascraper-description')(),
+        require('metascraper-author')(),
+        require('metascraper-publisher')(),
+        require('metascraper-image')(),
+        require('metascraper-logo-favicon')(),
+        require('metascraper-logo')()
+    ]);
+
+    let scraperResponse;
+
+    try {
+        if (!html) {
+            const response = await request(url, {
+                headers: {
+                    'user-agent': 'Ghost(https://github.com/TryGhost/Ghost)'
+                }
+            });
+            html = response.body;
+        }
+        scraperResponse = await metascraper({html, url});
+    } catch (e) {
+        return Promise.reject();
     }
-    const scraperResponse = await metascraper({html, url});
+
     const metadata = Object.assign({}, scraperResponse, {
         thumbnail: scraperResponse.image,
         icon: scraperResponse.logo
@@ -66,10 +75,6 @@ const findUrlWithProvider = (url) => {
     return {url, provider};
 };
 
-const getOembedUrlFromHTML = (html) => {
-    return cheerio('link[type="application/json+oembed"]', html).attr('href');
-};
-
 function unknownProvider(url) {
     return Promise.reject(new common.errors.ValidationError({
         message: common.i18n.t('errors.api.oembed.unknownProvider'),
@@ -85,12 +90,40 @@ function knownProvider(url) {
     });
 }
 
-function fetchOembedData(url) {
-    let provider;
-    ({url, provider} = findUrlWithProvider(url));
+function isIpOrLocalhost(url) {
+    try {
+        const IPV4_REGEX = /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
+        const IPV6_REGEX = /:/; // fqdns will not have colons
+        const HTTP_REGEX = /^https?:/i;
+
+        let {protocol, hostname} = new URL(url);
+
+        if (!HTTP_REGEX.test(protocol) || hostname === 'localhost' || IPV4_REGEX.test(hostname) || IPV6_REGEX.test(hostname)) {
+            return true;
+        }
+
+        return false;
+    } catch (e) {
+        return true;
+    }
+}
+
+function fetchOembedData(_url) {
+    // parse the url then validate the protocol and host to make sure it's
+    // http(s) and not an IP address or localhost to avoid potential access to
+    // internal network endpoints
+    if (isIpOrLocalhost(_url)) {
+        return unknownProvider();
+    }
+
+    // check against known oembed list
+    let {url, provider} = findUrlWithProvider(_url);
     if (provider) {
         return knownProvider(url);
     }
+
+    // url not in oembed list so fetch it in case it's a redirect or has a
+    // <link rel="alternate" type="application/json+oembed"> element
     return request(url, {
         method: 'GET',
         timeout: 2 * 1000,
@@ -99,19 +132,74 @@ function fetchOembedData(url) {
             'user-agent': 'Ghost(https://github.com/TryGhost/Ghost)'
         }
     }).then((response) => {
+        // url changed after fetch, see if we were redirected to a known oembed
         if (response.url !== url) {
             ({url, provider} = findUrlWithProvider(response.url));
+            if (provider) {
+                return knownProvider(url);
+            }
         }
-        if (provider) {
-            return knownProvider(url);
+
+        // check for <link rel="alternate" type="application/json+oembed"> element
+        let oembedUrl;
+        try {
+            oembedUrl = cheerio('link[type="application/json+oembed"]', response.body).attr('href');
+        } catch (e) {
+            return unknownProvider(url);
         }
-        const oembedUrl = getOembedUrlFromHTML(response.body);
+
         if (oembedUrl) {
+            // make sure the linked url is not an ip address or localhost
+            if (isIpOrLocalhost(oembedUrl)) {
+                return unknownProvider(oembedUrl);
+            }
+
+            // fetch oembed response from embedded rel="alternate" url
             return request(oembedUrl, {
                 method: 'GET',
-                json: true
+                json: true,
+                timeout: 2 * 1000,
+                headers: {
+                    'user-agent': 'Ghost(https://github.com/TryGhost/Ghost)'
+                }
             }).then((response) => {
-                return response.body;
+                // validate the fetched json against the oembed spec to avoid
+                // leaking non-oembed responses
+                const body = response.body;
+                const hasRequiredFields = body.type && body.version;
+                const hasValidType = ['photo', 'video', 'link', 'rich'].includes(body.type);
+
+                if (hasRequiredFields && hasValidType) {
+                    // extract known oembed fields from the response to limit leaking of unrecognised data
+                    const knownFields = [
+                        'type',
+                        'version',
+                        'html',
+                        'url',
+                        'title',
+                        'width',
+                        'height',
+                        'author_name',
+                        'author_url',
+                        'provider_name',
+                        'provider_url',
+                        'thumbnail_url',
+                        'thumbnail_width',
+                        'thumbnail_height'
+                    ];
+                    const oembed = _.pick(body, knownFields);
+
+                    // ensure we have required data for certain types
+                    if (oembed.type === 'photo' && !oembed.url) {
+                        return;
+                    }
+                    if ((oembed.type === 'video' || oembed.type === 'rich') && (!oembed.html || !oembed.width || !oembed.height)) {
+                        return;
+                    }
+
+                    // return the extracted object, don't pass through the response body
+                    return oembed;
+                }
             }).catch(() => {});
         }
     });
@@ -131,7 +219,8 @@ module.exports = {
             let {url, type} = data;
 
             if (type === 'bookmark') {
-                return fetchBookmarkData(url);
+                return fetchBookmarkData(url)
+                    .catch(() => unknownProvider(url));
             }
 
             return fetchOembedData(url).then((response) => {

core/test/regression/api/canary/admin/oembed_spec.js

@@ -0,0 +1,333 @@
+const nock = require('nock');
+const should = require('should');
+const supertest = require('supertest');
+const testUtils = require('../../../../utils');
+const config = require('../../../../../server/config');
+const localUtils = require('./utils');
+
+const ghost = testUtils.startGhost;
+
+describe('Oembed API (canary)', function () {
+    let ghostServer, request;
+
+    before(function () {
+        return ghost()
+            .then((_ghostServer) => {
+                ghostServer = _ghostServer;
+                request = supertest.agent(config.get('url'));
+            })
+            .then(() => {
+                return localUtils.doAuth(request);
+            });
+    });
+
+    it('can fetch an embed', function (done) {
+        let requestMock = nock('https://www.youtube.com')
+            .get('/oembed')
+            .query(true)
+            .reply(200, {
+                html: '<iframe width="480" height="270" src="https://www.youtube.com/embed/E5yFcdPAGv0?feature=oembed" frameborder="0" allow="autoplay; encrypted-media" allowfullscreen></iframe>',
+                thumbnail_width: 480,
+                width: 480,
+                author_url: 'https://www.youtube.com/user/gorillaz',
+                height: 270,
+                thumbnail_height: 360,
+                provider_name: 'YouTube',
+                title: 'Gorillaz - Humility (Official Video)',
+                provider_url: 'https://www.youtube.com/',
+                author_name: 'Gorillaz',
+                version: '1.0',
+                thumbnail_url: 'https://i.ytimg.com/vi/E5yFcdPAGv0/hqdefault.jpg',
+                type: 'video'
+            });
+
+        request.get(localUtils.API.getApiQuery('oembed/?url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DE5yFcdPAGv0'))
+            .set('Origin', config.get('url'))
+            .expect('Content-Type', /json/)
+            .expect('Cache-Control', testUtils.cacheRules.private)
+            .expect(200)
+            .end(function (err, res) {
+                if (err) {
+                    return done(err);
+                }
+
+                requestMock.isDone().should.be.true();
+                should.exist(res.body.html);
+                done();
+            });
+    });
+
+    describe('with unknown provider', function () {
+        it('fetches url and follows <link rel="alternate">', function (done) {
+            const pageMock = nock('http://test.com')
+                .get('/')
+                .reply(200, '<html><head><link rel="alternate" type="application/json+oembed" href="http://test.com/oembed"></head></html>');
+
+            const oembedMock = nock('http://test.com')
+                .get('/oembed')
+                .reply(200, {
+                    version: '1.0',
+                    type: 'link'
+                });
+
+            const url = encodeURIComponent('http://test.com');
+            request.get(localUtils.API.getApiQuery(`oembed/?url=${url}`))
+                .set('Origin', config.get('url'))
+                .expect('Content-Type', /json/)
+                .expect('Cache-Control', testUtils.cacheRules.private)
+                .expect(200)
+                .end(function (err, res) {
+                    if (err) {
+                        return done(err);
+                    }
+                    pageMock.isDone().should.be.true();
+                    oembedMock.isDone().should.be.true();
+                    done();
+                });
+        });
+
+        it('rejects invalid oembed responses', function (done) {
+            const pageMock = nock('http://test.com')
+                .get('/')
+                .reply(200, '<html><head><link rel="alternate" type="application/json+oembed" href="http://test.com/oembed"></head></html>');
+
+            const oembedMock = nock('http://test.com')
+                .get('/oembed')
+                .reply(200, {
+                    version: '1.0',
+                    html: 'test'
+                });
+
+            const url = encodeURIComponent('http://test.com');
+            request.get(localUtils.API.getApiQuery(`oembed/?url=${url}`))
+                .set('Origin', config.get('url'))
+                .expect('Content-Type', /json/)
+                .expect('Cache-Control', testUtils.cacheRules.private)
+                .expect(422)
+                .end(function (err, res) {
+                    if (err) {
+                        return done(err);
+                    }
+                    pageMock.isDone().should.be.true();
+                    oembedMock.isDone().should.be.true();
+                    done();
+                });
+        });
+
+        it('rejects unknown oembed types', function (done) {
+            const pageMock = nock('http://test.com')
+                .get('/')
+                .reply(200, '<html><head><link rel="alternate" type="application/json+oembed" href="http://test.com/oembed"></head></html>');
+
+            const oembedMock = nock('http://test.com')
+                .get('/oembed')
+                .reply(200, {
+                    version: '1.0',
+                    type: 'unknown'
+                });
+
+            const url = encodeURIComponent('http://test.com');
+            request.get(localUtils.API.getApiQuery(`oembed/?url=${url}`))
+                .set('Origin', config.get('url'))
+                .expect('Content-Type', /json/)
+                .expect('Cache-Control', testUtils.cacheRules.private)
+                .expect(422)
+                .end(function (err, res) {
+                    if (err) {
+                        return done(err);
+                    }
+                    pageMock.isDone().should.be.true();
+                    oembedMock.isDone().should.be.true();
+                    done();
+                });
+        });
+
+        it('rejects invalid photo responses', function (done) {
+            const pageMock = nock('http://test.com')
+                .get('/')
+                .reply(200, '<html><head><link rel="alternate" type="application/json+oembed" href="http://test.com/oembed"></head></html>');
+
+            const oembedMock = nock('http://test.com')
+                .get('/oembed')
+                .reply(200, {
+                    // no `url` field
+                    version: '1.0',
+                    type: 'photo',
+                    thumbnail_url: 'https://test.com/thumbnail.jpg'
+                });
+
+            const url = encodeURIComponent('http://test.com');
+            request.get(localUtils.API.getApiQuery(`oembed/?url=${url}`))
+                .set('Origin', config.get('url'))
+                .expect('Content-Type', /json/)
+                .expect('Cache-Control', testUtils.cacheRules.private)
+                .expect(422)
+                .end(function (err, res) {
+                    if (err) {
+                        return done(err);
+                    }
+                    pageMock.isDone().should.be.true();
+                    oembedMock.isDone().should.be.true();
+                    done();
+                });
+        });
+
+        it('rejects invalid video responses', function (done) {
+            const pageMock = nock('http://test.com')
+                .get('/')
+                .reply(200, '<html><head><link rel="alternate" type="application/json+oembed" href="http://test.com/oembed"></head></html>');
+
+            const oembedMock = nock('http://test.com')
+                .get('/oembed')
+                .reply(200, {
+                    // no `html` field
+                    version: '1.0',
+                    type: 'video',
+                    thumbnail_url: 'https://test.com/thumbnail.jpg'
+                });
+
+            const url = encodeURIComponent('http://test.com');
+            request.get(localUtils.API.getApiQuery(`oembed/?url=${url}`))
+                .set('Origin', config.get('url'))
+                .expect('Content-Type', /json/)
+                .expect('Cache-Control', testUtils.cacheRules.private)
+                .expect(422)
+                .end(function (err, res) {
+                    if (err) {
+                        return done(err);
+                    }
+                    pageMock.isDone().should.be.true();
+                    oembedMock.isDone().should.be.true();
+                    done();
+                });
+        });
+
+        it('strips unknown response fields', function (done) {
+            const pageMock = nock('http://test.com')
+                .get('/')
+                .reply(200, '<html><head><link rel="alternate" type="application/json+oembed" href="http://test.com/oembed"></head></html>');
+
+            const oembedMock = nock('http://test.com')
+                .get('/oembed')
+                .reply(200, {
+                    version: '1.0',
+                    type: 'video',
+                    html: '<p>Test</p>',
+                    width: 200,
+                    height: 100,
+                    unknown: 'test'
+                });
+
+            const url = encodeURIComponent('http://test.com');
+            request.get(localUtils.API.getApiQuery(`oembed/?url=${url}`))
+                .set('Origin', config.get('url'))
+                .expect('Content-Type', /json/)
+                .expect('Cache-Control', testUtils.cacheRules.private)
+                .expect(200)
+                .end(function (err, res) {
+                    if (err) {
+                        return done(err);
+                    }
+                    pageMock.isDone().should.be.true();
+                    oembedMock.isDone().should.be.true();
+
+                    res.body.should.deepEqual({
+                        version: '1.0',
+                        type: 'video',
+                        html: '<p>Test</p>',
+                        width: 200,
+                        height: 100
+                    });
+                    should.not.exist(res.body.unknown);
+
+                    done();
+                });
+        });
+
+        it('skips fetching IPv4 addresses', function (done) {
+            const pageMock = nock('http://test.com')
+                .get('/')
+                .reply(200, '<html><head><link rel="alternate" type="application/json+oembed" href="http://192.168.0.1/oembed"></head></html>');
+
+            const oembedMock = nock('http://192.168.0.1')
+                .get('/oembed')
+                .reply(200, {
+                    version: '1.0',
+                    type: 'link'
+                });
+
+            const url = encodeURIComponent('http://test.com');
+            request.get(localUtils.API.getApiQuery(`oembed/?url=${url}`))
+                .set('Origin', config.get('url'))
+                .expect('Content-Type', /json/)
+                .expect('Cache-Control', testUtils.cacheRules.private)
+                .expect(422)
+                .end(function (err, res) {
+                    if (err) {
+                        return done(err);
+                    }
+                    pageMock.isDone().should.be.true();
+                    oembedMock.isDone().should.be.false();
+                    done();
+                });
+        });
+
+        it('skips fetching IPv6 addresses', function (done) {
+            const pageMock = nock('http://test.com')
+                .get('/')
+                .reply(200, '<html><head><link rel="alternate" type="application/json+oembed" href="http://[2607:f0d0:1002:51::4]:9999/oembed"></head></html>');
+
+            const oembedMock = nock('http://[2607:f0d0:1002:51::4]:9999')
+                .get('/oembed')
+                .reply(200, {
+                    version: '1.0',
+                    type: 'link'
+                });
+
+            const url = encodeURIComponent('http://test.com');
+            request.get(localUtils.API.getApiQuery(`oembed/?url=${url}`))
+                .set('Origin', config.get('url'))
+                .expect('Content-Type', /json/)
+                .expect('Cache-Control', testUtils.cacheRules.private)
+                .expect(422)
+                .end(function (err, res) {
+                    if (err) {
+                        return done(err);
+                    }
+                    pageMock.isDone().should.be.true();
+                    oembedMock.isDone().should.be.false();
+                    done();
+                });
+        });
+
+        it('skips fetching localhost', function (done) {
+            const pageMock = nock('http://test.com')
+                .get('/')
+                .reply(200, '<html><head><link rel="alternate" type="application/json+oembed" href="http://localhost:9999/oembed"></head></html>');
+
+            const oembedMock = nock('http://localhost:9999')
+                .get('/oembed')
+                .reply(200, {
+                    // no `html` field
+                    version: '1.0',
+                    type: 'video',
+                    thumbnail_url: 'https://test.com/thumbnail.jpg'
+                });
+
+            const url = encodeURIComponent('http://test.com');
+            request.get(localUtils.API.getApiQuery(`oembed/?url=${url}`))
+                .set('Origin', config.get('url'))
+                .expect('Content-Type', /json/)
+                .expect('Cache-Control', testUtils.cacheRules.private)
+                .expect(422)
+                .end(function (err, res) {
+                    if (err) {
+                        return done(err);
+                    }
+                    pageMock.isDone().should.be.true();
+                    oembedMock.isDone().should.be.false();
+                    done();
+                });
+        });
+    });
+});