Merge pull request from GHSA-4m2q-w26j-h268

back-ported from 64ed246d03

- added an `externalRequest` lib
  - uses same underlying `got` module as our `request` lib
  - uses `got`'s `beforeRequest` and `beforeRedirect` hooks to perform it's own dns resolution for each url that's encountered and aborts with an error if it resolves to a private IP address block
  - includes a bypass for Ghost's configured url so that requests to it's own hostname+port are not blocked
- updated v0.1, v2 and canary oembed controllers to use the `externalRequest` lib

core/server/api/canary/oembed.js

@@ -1,9 +1,10 @@
 const common = require('../../lib/common');
 const {extract, hasProvider} = require('oembed-parser');
 const Promise = require('bluebird');
-const request = require('../../lib/request');
+const externalRequest = require('../../lib/request-external');
 const cheerio = require('cheerio');
 const _ = require('lodash');
+const {URL} = require('url');
 
 async function fetchBookmarkData(url, html) {
     const metascraper = require('metascraper')([
@@ -21,7 +22,7 @@ async function fetchBookmarkData(url, html) {
 
     try {
         if (!html) {
-            const response = await request(url, {
+            const response = await externalRequest(url, {
                 headers: {
                     'user-agent': 'Ghost(https://github.com/TryGhost/Ghost)'
                 }
@@ -124,7 +125,7 @@ function fetchOembedData(_url) {
 
     // url not in oembed list so fetch it in case it's a redirect or has a
     // <link rel="alternate" type="application/json+oembed"> element
-    return request(url, {
+    return externalRequest(url, {
         method: 'GET',
         timeout: 2 * 1000,
         followRedirect: true,
@@ -155,7 +156,7 @@ function fetchOembedData(_url) {
             }
 
             // fetch oembed response from embedded rel="alternate" url
-            return request(oembedUrl, {
+            return externalRequest(oembedUrl, {
                 method: 'GET',
                 json: true,
                 timeout: 2 * 1000,

core/server/api/v0.1/oembed.js

@@ -1,7 +1,7 @@
 const common = require('../../lib/common');
 const {extract, hasProvider} = require('oembed-parser');
 const Promise = require('bluebird');
-const request = require('../../lib/request');
+const externalRequest = require('../../lib/request-external');
 const cheerio = require('cheerio');
 const _ = require('lodash');
 const {URL} = require('url');
@@ -84,7 +84,7 @@ function fetchOembedData(_url) {
 
     // url not in oembed list so fetch it in case it's a redirect or has a
     // <link rel="alternate" type="application/json+oembed"> element
-    return request(url, {
+    return externalRequest(url, {
         method: 'GET',
         timeout: 2 * 1000,
         followRedirect: true,
@@ -115,7 +115,7 @@ function fetchOembedData(_url) {
             }
 
             // fetch oembed response from embedded rel="alternate" url
-            return request(oembedUrl, {
+            return externalRequest(oembedUrl, {
                 method: 'GET',
                 json: true,
                 timeout: 2 * 1000,

core/server/api/v2/oembed.js

@@ -1,7 +1,7 @@
 const common = require('../../lib/common');
 const {extract, hasProvider} = require('oembed-parser');
 const Promise = require('bluebird');
-const request = require('../../lib/request');
+const externalRequest = require('../../lib/request-external');
 const cheerio = require('cheerio');
 const _ = require('lodash');
 const {URL} = require('url');
@@ -79,7 +79,7 @@ function fetchOembedData(_url) {
 
     // url not in oembed list so fetch it in case it's a redirect or has a
     // <link rel="alternate" type="application/json+oembed"> element
-    return request(url, {
+    return externalRequest(url, {
         method: 'GET',
         timeout: 2 * 1000,
         followRedirect: true,
@@ -110,7 +110,7 @@ function fetchOembedData(_url) {
             }
 
             // fetch oembed response from embedded rel="alternate" url
-            return request(oembedUrl, {
+            return externalRequest(oembedUrl, {
                 method: 'GET',
                 json: true,
                 timeout: 2 * 1000,

core/server/lib/request-external.js

@@ -0,0 +1,67 @@
+const got = require('got');
+const {URL} = require('url');
+const dns = require('dns');
+const common = require('./common');
+const ghostVersion = require('./ghost-version');
+const config = require('../config');
+const validator = require('../data/validation').validator;
+
+function isPrivateIp(addr) {
+    return /^(::f{4}:)?10\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$/i.test(addr) ||
+      /^(::f{4}:)?192\.168\.([0-9]{1,3})\.([0-9]{1,3})$/i.test(addr) ||
+      /^(::f{4}:)?172\.(1[6-9]|2\d|30|31)\.([0-9]{1,3})\.([0-9]{1,3})$/i.test(addr) ||
+      /^(::f{4}:)?127\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$/i.test(addr) ||
+      /^(::f{4}:)?169\.254\.([0-9]{1,3})\.([0-9]{1,3})$/i.test(addr) ||
+      /^f[cd][0-9a-f]{2}:/i.test(addr) ||
+      /^fe80:/i.test(addr) ||
+      /^::1$/.test(addr) ||
+      /^::$/.test(addr);
+}
+
+function errorIfHostnameResolvesToPrivateIp(options) {
+    // allow requests through to local Ghost instance
+    const siteUrl = new URL(config.get('url'));
+    const requestUrl = new URL(options.href);
+    if (requestUrl.host === siteUrl.host) {
+        return Promise.resolve();
+    }
+
+    dns.lookup(options.hostname, {}, (err, address) => {
+        if (err) {
+            throw err;
+        }
+
+        if (isPrivateIp(address)) {
+            throw new common.errors.InternalServerError({
+                message: 'URL resolves to a non-permitted private IP block',
+                code: 'URL_PRIVATE_INVALID',
+                context: options.href
+            });
+        }
+
+        return Promise.resolve();
+    });
+}
+
+// same as our normal request lib but if any request in a redirect chain resolves
+// to a private IP address it will be blocked before the request is made.
+const externalRequest = got.extend({
+    headers: {
+        'user-agent': 'Ghost/' + ghostVersion.original + ' (https://github.com/TryGhost/Ghost)'
+    },
+    hooks: {
+        init: [(options) => {
+            if (!options.hostname || !validator.isURL(options.hostname)) {
+                throw new common.errors.InternalServerError({
+                    message: 'URL empty or invalid.',
+                    code: 'URL_MISSING_INVALID',
+                    context: options.href
+                });
+            }
+        }],
+        beforeRequest: [errorIfHostnameResolvesToPrivateIp],
+        beforeRedirect: [errorIfHostnameResolvesToPrivateIp]
+    }
+});
+
+module.exports = externalRequest;

core/test/acceptance/old/admin/oembed_spec.js

@@ -1,10 +1,14 @@
 const nock = require('nock');
+const sinon = require('sinon');
 const should = require('should');
 const supertest = require('supertest');
 const testUtils = require('../../../utils/index');
 const config = require('../../../../server/config/index');
 const localUtils = require('./utils');
 
+// for sinon
+const dns = require('dns');
+
 const ghost = testUtils.startGhost;
 
 describe('Oembed API', function () {
@@ -21,6 +25,15 @@ describe('Oembed API', function () {
             });
     });
 
+    beforeEach(function () {
+        sinon.stub(dns, 'lookup').yields(null, '123.123.123.123', 4);
+    });
+
+    afterEach(function () {
+        sinon.restore();
+        nock.cleanAll();
+    });
+
     it('can fetch an embed', function (done) {
         let requestMock = nock('https://www.youtube.com')
             .get('/oembed')

core/test/regression/api/canary/admin/oembed_spec.js

@@ -1,10 +1,14 @@
 const nock = require('nock');
+const sinon = require('sinon');
 const should = require('should');
 const supertest = require('supertest');
 const testUtils = require('../../../../utils');
 const config = require('../../../../../server/config');
 const localUtils = require('./utils');
 
+// for sinon
+const dns = require('dns');
+
 const ghost = testUtils.startGhost;
 
 describe('Oembed API (canary)', function () {
@@ -21,6 +25,15 @@ describe('Oembed API (canary)', function () {
             });
     });
 
+    beforeEach(function () {
+        sinon.stub(dns, 'lookup').yields(null, '123.123.123.123', 4);
+    });
+
+    afterEach(function () {
+        sinon.restore();
+        nock.cleanAll();
+    });
+
     it('can fetch an embed', function (done) {
         let requestMock = nock('https://www.youtube.com')
             .get('/oembed')

core/test/regression/api/v0.1/oembed_spec.js

@@ -1,10 +1,14 @@
 const config = require('../../../../server/config/index');
+const sinon = require('sinon');
 const nock = require('nock');
 const should = require('should');
 const supertest = require('supertest');
 const testUtils = require('../../../utils/index');
 const localUtils = require('./utils');
 
+// for sinon
+const dns = require('dns');
+
 const ghost = testUtils.startGhost;
 
 describe('Oembed API', function () {
@@ -24,6 +28,15 @@ describe('Oembed API', function () {
             });
     });
 
+    beforeEach(function () {
+        sinon.stub(dns, 'lookup').yields(null, '123.123.123.123', 4);
+    });
+
+    afterEach(function () {
+        sinon.restore();
+        nock.cleanAll();
+    });
+
     describe('success', function () {
         it('can fetch an embed', function (done) {
             let requestMock = nock('https://www.youtube.com')

core/test/regression/api/v2/admin/oembed_spec.js

@@ -1,10 +1,14 @@
 const nock = require('nock');
+const sinon = require('sinon');
 const should = require('should');
 const supertest = require('supertest');
 const testUtils = require('../../../../utils/index');
 const config = require('../../../../../server/config/index');
 const localUtils = require('./utils');
 
+// for sinon
+const dns = require('dns');
+
 const ghost = testUtils.startGhost;
 
 describe('Oembed API (v2)', function () {
@@ -21,6 +25,15 @@ describe('Oembed API (v2)', function () {
             });
     });
 
+    beforeEach(function () {
+        sinon.stub(dns, 'lookup').yields(null, '123.123.123.123', 4);
+    });
+
+    afterEach(function () {
+        sinon.restore();
+        nock.cleanAll();
+    });
+
     it('can fetch an embed', function (done) {
         let requestMock = nock('https://www.youtube.com')
             .get('/oembed')

core/test/unit/api/v0.1/oembed_spec.js

@@ -1,9 +1,22 @@
 const common = require('../../../../server/lib/common');
+const sinon = require('sinon');
 const nock = require('nock');
 const OembedAPI = require('../../../../server/api/v0.1/oembed');
 const should = require('should');
 
+// for sinon
+const dns = require('dns');
+
 describe('API: oembed', function () {
+    beforeEach(function () {
+        sinon.stub(dns, 'lookup').yields(null, '123.123.123.123', 4);
+    });
+
+    afterEach(function () {
+        sinon.restore();
+        nock.cleanAll();
+    });
+
     describe('fn: read', function () {
         // https://oembed.com/providers.json only has schemes for https://reddit.com
         it('finds match for unlisted http scheme', function (done) {

core/test/unit/lib/request-external_spec.js

@@ -0,0 +1,328 @@
+const sinon = require('sinon');
+const should = require('should');
+const rewire = require('rewire');
+const nock = require('nock');
+const externalRequest = rewire('../../../../core/server/lib/request-external');
+const configUtils = require('../../utils/configUtils');
+
+// for sinon stubs
+const dns = require('dns');
+
+describe('External Request', function () {
+    describe('with private ip', function () {
+        beforeEach(function () {
+            sinon.stub(dns, 'lookup').yields(null, '192.168.0.1', 4);
+        });
+
+        afterEach(function () {
+            configUtils.restore();
+            sinon.restore();
+            nock.cleanAll();
+        });
+
+        it('allows configured hostname', function () {
+            configUtils.set('url', 'http://example.com');
+
+            const url = 'http://example.com/endpoint/';
+            const expectedResponse = {
+                body: 'Response body',
+                url: 'http://example.com/endpoint/',
+                statusCode: 200
+            };
+            const options = {
+                headers: {
+                    'User-Agent': 'Mozilla/5.0'
+                }
+            };
+
+            const requestMock = nock('http://example.com')
+                .get('/endpoint/')
+                .reply(200, 'Response body');
+
+            return externalRequest(url, options).then(function (res) {
+                requestMock.isDone().should.be.true();
+                should.exist(res);
+                should.exist(res.body);
+                res.body.should.be.equal(expectedResponse.body);
+                should.exist(res.url);
+                res.statusCode.should.be.equal(expectedResponse.statusCode);
+                should.exist(res.statusCode);
+                res.url.should.be.equal(expectedResponse.url);
+            });
+        });
+
+        it('allows configured hostname+port', function () {
+            configUtils.set('url', 'http://example.com:2368');
+
+            const url = 'http://example.com:2368/endpoint/';
+            const expectedResponse = {
+                body: 'Response body',
+                url: 'http://example.com:2368/endpoint/',
+                statusCode: 200
+            };
+            const options = {
+                headers: {
+                    'User-Agent': 'Mozilla/5.0'
+                }
+            };
+
+            const requestMock = nock('http://example.com:2368')
+                .get('/endpoint/')
+                .reply(200, 'Response body');
+
+            return externalRequest(url, options).then(function (res) {
+                requestMock.isDone().should.be.true();
+                should.exist(res);
+                should.exist(res.body);
+                res.body.should.be.equal(expectedResponse.body);
+                should.exist(res.url);
+                res.statusCode.should.be.equal(expectedResponse.statusCode);
+                should.exist(res.statusCode);
+                res.url.should.be.equal(expectedResponse.url);
+            });
+        });
+
+        it('blocks configured hostname with incorrect port', function () {
+            configUtils.set('url', 'http://example.com');
+
+            const url = 'http://example.com:1234/endpoint/';
+            const options = {
+                headers: {
+                    'User-Agent': 'Mozilla/5.0'
+                }
+            };
+
+            return externalRequest(url, options).then(() => {
+                throw new Error('Request should have rejected with non-permitted IP message');
+            }, (err) => {
+                should.exist(err);
+                err.message.should.be.equal('URL resolves to a non-permitted private IP block');
+            });
+        });
+
+        it('blocks configured hostname+port with incorrect port', function () {
+            configUtils.set('url', 'http://example.com:2368');
+
+            const url = 'http://example.com:1234/endpoint/';
+            const options = {
+                headers: {
+                    'User-Agent': 'Mozilla/5.0'
+                }
+            };
+
+            return externalRequest(url, options).then(() => {
+                throw new Error('Request should have rejected with non-permitted IP message');
+            }, (err) => {
+                should.exist(err);
+                err.message.should.be.equal('URL resolves to a non-permitted private IP block');
+            });
+        });
+
+        it('blocks on request', function () {
+            const url = 'http://some-website.com/';
+            const options = {
+                headers: {
+                    'User-Agent': 'Mozilla/5.0'
+                }
+            };
+
+            const requestMock = nock('http://some-website.com')
+                .get('/files/')
+                .reply(200, 'Response');
+
+            return externalRequest(url, options).then(function () {
+                throw new Error('Request should have rejected with non-permitted IP message');
+            }, (err) => {
+                should.exist(err);
+                err.message.should.be.equal('URL resolves to a non-permitted private IP block');
+                requestMock.isDone().should.be.false();
+            });
+        });
+
+        it('blocks on redirect', function () {
+            configUtils.set('url', 'http://some-website.com');
+
+            const url = 'http://some-website.com/endpoint/';
+            const options = {
+                headers: {
+                    'User-Agent': 'Mozilla/5.0'
+                }
+            };
+
+            const requestMock = nock('http://some-website.com')
+                .get('/endpoint/')
+                .reply(301, 'Oops, got redirected',
+                    {
+                        location: 'http://someredirectedurl.com/files/'
+                    });
+
+            const secondRequestMock = nock('http://someredirectedurl.com')
+                .get('/files/')
+                .reply(200, 'Redirected response');
+
+            return externalRequest(url, options).then(function () {
+                throw new Error('Request should have rejected with non-permitted IP message');
+            }, (err) => {
+                should.exist(err);
+                err.message.should.be.equal('URL resolves to a non-permitted private IP block');
+                requestMock.isDone().should.be.true();
+                secondRequestMock.isDone().should.be.false();
+            });
+        });
+    });
+
+    describe('general behaviour', function () {
+        beforeEach(function () {
+            sinon.stub(dns, 'lookup').yields(null, '123.123.123.123', 4);
+        });
+
+        afterEach(function () {
+            configUtils.restore();
+            sinon.restore();
+            nock.cleanAll();
+        });
+
+        it('[success] should return response for http request', function () {
+            const url = 'http://some-website.com/endpoint/';
+            const expectedResponse = {
+                body: 'Response body',
+                url: 'http://some-website.com/endpoint/',
+                statusCode: 200
+            };
+            const options = {
+                headers: {
+                    'User-Agent': 'Mozilla/5.0'
+                }
+            };
+
+            const requestMock = nock('http://some-website.com')
+                .get('/endpoint/')
+                .reply(200, 'Response body');
+
+            return externalRequest(url, options).then(function (res) {
+                requestMock.isDone().should.be.true();
+                should.exist(res);
+                should.exist(res.body);
+                res.body.should.be.equal(expectedResponse.body);
+                should.exist(res.url);
+                res.statusCode.should.be.equal(expectedResponse.statusCode);
+                should.exist(res.statusCode);
+                res.url.should.be.equal(expectedResponse.url);
+            });
+        });
+
+        it('[success] can handle redirect', function () {
+            const url = 'http://some-website.com/endpoint/';
+            const expectedResponse = {
+                body: 'Redirected response',
+                url: 'http://someredirectedurl.com/files/',
+                statusCode: 200
+            };
+            const options = {
+                headers: {
+                    'User-Agent': 'Mozilla/5.0'
+                }
+            };
+
+            const requestMock = nock('http://some-website.com')
+                .get('/endpoint/')
+                .reply(301, 'Oops, got redirected',
+                    {
+                        location: 'http://someredirectedurl.com/files/'
+                    });
+
+            const secondRequestMock = nock('http://someredirectedurl.com')
+                .get('/files/')
+                .reply(200, 'Redirected response');
+
+            return externalRequest(url, options).then(function (res) {
+                requestMock.isDone().should.be.true();
+                secondRequestMock.isDone().should.be.true();
+                should.exist(res);
+                should.exist(res.body);
+                res.body.should.be.equal(expectedResponse.body);
+                should.exist(res.url);
+                res.statusCode.should.be.equal(expectedResponse.statusCode);
+                should.exist(res.statusCode);
+                res.url.should.be.equal(expectedResponse.url);
+            });
+        });
+
+        it('[failure] can handle invalid url', function () {
+            const url = 'test';
+            const options = {
+                headers: {
+                    'User-Agent': 'Mozilla/5.0'
+                }
+            };
+
+            return externalRequest(url, options).then(() => {
+                throw new Error('Request should have rejected with invalid url message');
+            }, (err) => {
+                should.exist(err);
+                err.message.should.be.equal('URL empty or invalid.');
+            });
+        });
+
+        it('[failure] can handle empty url', function () {
+            const url = '';
+            const options = {
+                headers: {
+                    'User-Agent': 'Mozilla/5.0'
+                }
+            };
+
+            return externalRequest(url, options).then(() => {
+                throw new Error('Request should have rejected with invalid url message');
+            }, (err) => {
+                should.exist(err);
+                err.message.should.be.equal('URL empty or invalid.');
+            });
+        });
+
+        it('[failure] can handle an error with statuscode not 200', function () {
+            const url = 'http://nofilehere.com/files/test.txt';
+            const options = {
+                headers: {
+                    'User-Agent': 'Mozilla/5.0'
+                }
+            };
+
+            const requestMock = nock('http://nofilehere.com')
+                .get('/files/test.txt')
+                .reply(404);
+
+            return externalRequest(url, options).then(() => {
+                throw new Error('Request should have errored');
+            }, (err) => {
+                requestMock.isDone().should.be.true();
+                should.exist(err);
+                err.statusMessage.should.be.equal('Not Found');
+            });
+        });
+
+        it('[failure] returns error if request errors', function () {
+            const url = 'http://nofilehere.com/files/test.txt';
+            const options = {
+                headers: {
+                    'User-Agent': 'Mozilla/5.0'
+                }
+            };
+
+            const requestMock = nock('http://nofilehere.com')
+                .get('/files/test.txt')
+                .times(3) // 1 original request + 2 default retries
+                .reply(500, {message: 'something awful happened', code: 'AWFUL_ERROR'});
+
+            return externalRequest(url, options).then(() => {
+                throw new Error('Request should have errored with an awful error');
+            }, (err) => {
+                requestMock.isDone().should.be.true();
+                should.exist(err);
+                err.statusMessage.should.be.equal('Internal Server Error');
+                err.body.should.match(/something awful happened/);
+                err.body.should.match(/AWFUL_ERROR/);
+            });
+        });
+    });
+});