Added queueing middleware to handle high request volume (#19887)

ref https://linear.app/tryghost/issue/CFR-4/ - added request queueing middleware (express-queue) to handle high request volume - added new config option `optimization.requestQueue` - added new config option `optimization.requestConcurrency` - added logging of request queue depth - `req.queueDepth` We've done a fair amount of investigation around improving Ghost's resiliency to high request volume. While we believe this to be partly due to database connection contention, it also seems Ghost gets overwhelmed by the requests themselves. Implementing a simple queueing system allows us a simple lever to change the volume of requests Ghost is actually ingesting at any given time and gives us options besides simply increasing database connection pool size. --------- Co-authored-by: Michael Barrett <mike@ghost.org>
2024-12-30 22:34:01 -05:00 · 2024-03-21 09:25:07 -05:00 · 2024-03-21 09:25:07 -05:00 · a1c4e64994
commit a1c4e64994
parent dfdd4e5cfa
21 changed files with 312 additions and 61 deletions
--- a/ghost/bootstrap-socket/package.json
+++ b/ghost/bootstrap-socket/package.json
@ -24,6 +24,6 @@
    "sinon": "15.2.0"
  },
  "dependencies": {
-    "@tryghost/logging": "2.4.8"
+    "@tryghost/logging": "2.4.13"
  }
 }
--- a/ghost/collections/package.json
+++ b/ghost/collections/package.json
@ -29,7 +29,7 @@
        "@tryghost/debug": "0.1.26",
        "@tryghost/errors": "1.3.1",
        "@tryghost/in-memory-repository": "0.0.0",
-        "@tryghost/logging": "2.4.8",
+        "@tryghost/logging": "2.4.13",
        "@tryghost/nql": "0.12.1",
        "@tryghost/nql-filter-expansions": "0.0.0",
        "@tryghost/post-events": "0.0.0",
--- a/ghost/core/core/server/web/parent/app.js
+++ b/ghost/core/core/server/web/parent/app.js
@ -23,6 +23,13 @@ module.exports = function setupParentApp() {
    // @TODO: figure out if this is really needed everywhere? Is it not frontend only...
    parentApp.use(mw.ghostLocals);

+    // Enable request queuing if configured
+    const queueConfig = config.get('optimization:requestQueue');
+
+    if (queueConfig?.enabled === true) {
+        parentApp.use(mw.queueRequest(queueConfig));
+    }
+
    debug('ParentApp setup end');

    return parentApp;
--- a/ghost/core/core/server/web/parent/middleware/index.js
+++ b/ghost/core/core/server/web/parent/middleware/index.js
@ -2,5 +2,6 @@ module.exports = {
    emitEvents: require('./emit-events'),
    ghostLocals: require('./ghost-locals'),
    logRequest: require('./log-request'),
+    queueRequest: require('./queue-request'),
    requestId: require('./request-id')
 };
--- a/ghost/core/core/server/web/parent/middleware/queue-request.js
+++ b/ghost/core/core/server/web/parent/middleware/queue-request.js
@ -0,0 +1,60 @@
+const errors = require('@tryghost/errors');
+const logging = require('@tryghost/logging');
+const path = require('node:path');
+const expressQueue = require('express-queue');
+
+const debug = (message) => {
+    logging.debug(`[queue-request] ${message}`);
+};
+
+module.exports = function queueRequest(
+    config,
+    queueFactory = expressQueue
+) {
+    if (config.concurrencyLimit === undefined) {
+        throw new errors.IncorrectUsageError({
+            message: 'concurrencyLimit must be defined when using queueRequest middleware'
+        });
+    }
+
+    debug(`Initialising queueRequest middleware with config: ${JSON.stringify(config)}`);
+
+    // @see https://github.com/alykoshin/express-queue#usage
+    const queue = queueFactory({
+        activeLimit: config.concurrencyLimit,
+        queuedLimit: -1 // Do not limit the number of queued requests
+    });
+
+    /**
+     * Available events:
+     * - queue - when a request is queued
+     * - dequeue - when a request is dequeued
+     * - process - when a request is being processed
+     * - reject - when a request is rejected
+     * - cancel - when a request is cancelled
+     * - complete - when a request has completed
+     *
+     * @see https://github.com/search?q=repo:alykoshin/mini-queue%20job._toState&type=code
+     */
+    queue.queue.on('queue', (job) => {
+        debug(`Request queued: ${job.data.req.path}`);
+
+        job.data.req.queueDepth = job.queue.getLength();
+    });
+
+    queue.queue.on('complete', (job) => {
+        debug(`Request completed: ${job.data.req.path}`);
+    });
+
+    return (req, res, next) => {
+        // Do not queue requests for static assets - We assume that any path
+        // with a file extension is a static asset
+        if (path.extname(req.path)) {
+            debug(`Request for assumed static asset skipping queue: ${req.path}`);
+
+            return next();
+        }
+
+        return queue(req, res, next);
+    };
+};
--- a/ghost/core/package.json
+++ b/ghost/core/package.json
@ -110,7 +110,7 @@
    "@tryghost/link-redirects": "0.0.0",
    "@tryghost/link-replacer": "0.0.0",
    "@tryghost/link-tracking": "0.0.0",
-    "@tryghost/logging": "2.4.8",
+    "@tryghost/logging": "2.4.13",
    "@tryghost/magic-link": "0.0.0",
    "@tryghost/mail-events": "0.0.0",
    "@tryghost/mailgun-client": "0.0.0",
@ -182,6 +182,7 @@
    "express-jwt": "8.4.1",
    "express-lazy-router": "1.0.6",
    "express-query-boolean": "2.0.0",
+    "express-queue": "0.0.13",
    "express-session": "1.18.0",
    "fs-extra": "11.2.0",
    "ghost-storage-base": "1.0.0",
@ -260,7 +261,7 @@
  },
  "resolutions": {
    "@tryghost/errors": "1.3.1",
-    "@tryghost/logging": "2.4.8",
+    "@tryghost/logging": "2.4.13",
    "moment": "2.24.0",
    "moment-timezone": "0.5.23"
  },
--- a/ghost/core/test/unit/server/web/parent/middleware/queue-request.test.js
+++ b/ghost/core/test/unit/server/web/parent/middleware/queue-request.test.js
@ -0,0 +1,92 @@
+const assert = require('node:assert');
+const sinon = require('sinon');
+
+const queueRequest = require('../../../../../../core/server/web/parent/middleware/queue-request');
+
+describe('Queue request middleware', function () {
+    let req, res, next, config, queueFactory, queue;
+
+    beforeEach(function () {
+        req = {};
+        res = {};
+        next = sinon.stub();
+        config = {
+            concurrencyLimit: 123
+        };
+
+        queue = sinon.stub();
+        queue.queue = {
+            on: sinon.stub()
+        };
+
+        queueFactory = sinon.stub().returns(queue);
+    });
+
+    it('should configure the queue using the concurrency limit defined in the config', function () {
+        queueRequest(config, queueFactory);
+
+        assert.deepEqual(queueFactory.callCount, 1, 'queueFactory should be called once');
+        assert.deepEqual(queueFactory.getCall(0).args[0], {
+            activeLimit: config.concurrencyLimit,
+            queuedLimit: -1
+        }, 'queueFactory should be called with the queue limit from the config');
+    });
+
+    it('should throw an error if the concurrency limit is not defined in the config', function () {
+        assert.throws(() => {
+            queueRequest({}, queueFactory);
+        }, /concurrencyLimit must be defined when using queueRequest middleware/, 'error should be thrown');
+    });
+
+    it('should not queue requests for static assets', function () {
+        req.path = '/foo/bar.css'; // Assume any path with a file extension is a static asset
+
+        const mw = queueRequest(config, queueFactory);
+
+        mw(req, res, next);
+
+        assert(next.calledOnce, 'next should be called once');
+        assert.equal(queue.calledOnce, 0, 'queue should not be called');
+    });
+
+    it('should queue the request', function () {
+        req.path = '/foo/bar';
+
+        const mw = queueRequest(config, queueFactory);
+
+        mw(req, res, next);
+
+        assert(queue.calledOnce, 'queue should be called once');
+        assert(queue.calledWith(req, res, next), 'queue should be called with the correct arguments');
+    });
+
+    it('should record the queue depth on a request when it has queued', function () {
+        const queueEvent = 'queue';
+        const queueLength = 123;
+
+        // Assert event listener is added
+        queueRequest(config, queueFactory);
+
+        assert(queue.queue.on.calledWith(queueEvent), `"${queueEvent}" event listener should be added`);
+
+        const listener = queue.queue.on.args.find(arg => arg[0] === queueEvent)[1];
+
+        // Assert event listener implementation
+        const queueJob = {
+            data: {
+                req: {
+                    path: '/foo/bar'
+                }
+            },
+            queue: {
+                getLength() {
+                    return queueLength;
+                }
+            }
+        };
+
+        listener(queueJob);
+
+        assert(queueJob.data.req.queueDepth === queueLength, 'queueDepth should be set on the request');
+    });
+});
--- a/ghost/domain-events/package.json
+++ b/ghost/domain-events/package.json
@ -19,7 +19,7 @@
    "lib"
  ],
  "devDependencies": {
-    "@tryghost/logging": "2.4.8",
+    "@tryghost/logging": "2.4.13",
    "c8": "8.0.1",
    "mocha": "10.2.0",
    "should": "13.2.3"
--- a/ghost/email-service/package.json
+++ b/ghost/email-service/package.json
@ -30,7 +30,7 @@
    "@tryghost/errors": "1.3.1",
    "@tryghost/html-to-plaintext": "0.0.0",
    "@tryghost/kg-default-cards": "10.0.2",
-    "@tryghost/logging": "2.4.8",
+    "@tryghost/logging": "2.4.13",
    "@tryghost/tpl": "0.1.26",
    "@tryghost/validator": "0.2.6",
    "bson-objectid": "2.0.4",
--- a/ghost/job-manager/package.json
+++ b/ghost/job-manager/package.json
@ -29,7 +29,7 @@
  "dependencies": {
    "@breejs/later": "4.2.0",
    "@tryghost/errors": "1.3.1",
-    "@tryghost/logging": "2.4.8",
+    "@tryghost/logging": "2.4.13",
    "bree": "6.5.0",
    "cron-validate": "1.4.5",
    "fastq": "1.17.1",
--- a/ghost/mailgun-client/package.json
+++ b/ghost/mailgun-client/package.json
@ -25,7 +25,7 @@
  },
  "dependencies": {
    "@tryghost/debug": "0.1.26",
-    "@tryghost/logging": "2.4.8",
+    "@tryghost/logging": "2.4.13",
    "@tryghost/metrics": "1.0.27",
    "form-data": "4.0.0",
    "lodash": "4.17.21",
--- a/ghost/members-api/package.json
+++ b/ghost/members-api/package.json
@ -32,7 +32,7 @@
  "dependencies": {
    "@tryghost/domain-events": "0.0.0",
    "@tryghost/errors": "1.3.1",
-    "@tryghost/logging": "2.4.8",
+    "@tryghost/logging": "2.4.13",
    "@tryghost/magic-link": "0.0.0",
    "@tryghost/member-events": "0.0.0",
    "@tryghost/members-payments": "0.0.0",
--- a/ghost/members-events-service/package.json
+++ b/ghost/members-events-service/package.json
@ -27,7 +27,7 @@
  "dependencies": {
    "@tryghost/domain-events": "0.0.0",
    "@tryghost/errors": "1.3.1",
-    "@tryghost/logging": "2.4.8",
+    "@tryghost/logging": "2.4.13",
    "@tryghost/member-events": "0.0.0",
    "moment-timezone": "0.5.34"
  }
--- a/ghost/members-importer/package.json
+++ b/ghost/members-importer/package.json
@ -26,7 +26,7 @@
  },
  "dependencies": {
    "@tryghost/errors": "1.3.1",
-    "@tryghost/logging": "2.4.8",
+    "@tryghost/logging": "2.4.13",
    "@tryghost/members-csv": "0.0.0",
    "@tryghost/metrics": "1.0.27",
    "@tryghost/tpl": "0.1.26",
--- a/ghost/oembed-service/package.json
+++ b/ghost/oembed-service/package.json
@ -24,7 +24,7 @@
  "dependencies": {
    "@extractus/oembed-extractor": "3.2.1",
    "@tryghost/errors": "1.3.1",
-    "@tryghost/logging": "2.4.8",
+    "@tryghost/logging": "2.4.13",
    "@tryghost/tpl": "0.1.26",
    "charset": "1.0.1",
    "cheerio": "0.22.0",
--- a/ghost/recommendations/package.json
+++ b/ghost/recommendations/package.json
@ -34,6 +34,6 @@
    "@tryghost/errors": "1.3.1",
    "@tryghost/in-memory-repository": "0.0.0",
    "@tryghost/bookshelf-repository": "0.0.0",
-    "@tryghost/logging": "2.4.8"
+    "@tryghost/logging": "2.4.13"
  }
 }
--- a/ghost/stripe/package.json
+++ b/ghost/stripe/package.json
@ -28,7 +28,7 @@
    "@tryghost/debug": "0.1.26",
    "@tryghost/domain-events": "0.0.0",
    "@tryghost/errors": "1.3.1",
-    "@tryghost/logging": "2.4.8",
+    "@tryghost/logging": "2.4.13",
    "@tryghost/member-events": "0.0.0",
    "leaky-bucket": "2.2.0",
    "lodash": "4.17.21",
--- a/ghost/update-check-service/package.json
+++ b/ghost/update-check-service/package.json
@ -27,7 +27,7 @@
  "dependencies": {
    "@tryghost/debug": "0.1.26",
    "@tryghost/errors": "1.3.1",
-    "@tryghost/logging": "2.4.8",
+    "@tryghost/logging": "2.4.13",
    "@tryghost/tpl": "0.1.26",
    "lodash": "4.17.21",
    "moment": "2.24.0"
--- a/ghost/webmentions/package.json
+++ b/ghost/webmentions/package.json
@ -26,7 +26,7 @@
  },
  "dependencies": {
    "@tryghost/errors": "1.3.1",
-    "@tryghost/logging": "2.4.8",
+    "@tryghost/logging": "2.4.13",
    "cheerio": "0.22.0"
  }
 }
--- a/package.json
+++ b/package.json
@ -43,7 +43,7 @@
  },
  "resolutions": {
    "@tryghost/errors": "1.3.1",
-    "@tryghost/logging": "2.4.8",
+    "@tryghost/logging": "2.4.13",
    "moment": "2.24.0",
    "moment-timezone": "0.5.23"
  },
--- a/yarn.lock
+++ b/yarn.lock
@ -2234,6 +2234,14 @@
    "@elastic/transport" "^8.3.4"
    tslib "^2.4.0"

+"@elastic/elasticsearch@8.12.2":
+  version "8.12.2"
+  resolved "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-8.12.2.tgz#7a241f739a509cc59faee85f79a4c9e9e5ba9128"
+  integrity sha512-04NvH3LIgcv1Uwguorfw2WwzC9Lhfsqs9f0L6uq6MrCw0lqe/HOQ6E8vJ6EkHAA15iEfbhtxOtenbZVVcE+mAQ==
+  dependencies:
+    "@elastic/transport" "^8.4.1"
+    tslib "^2.4.0"
+
 "@elastic/transport@^8.3.4":
  version "8.4.0"
  resolved "https://registry.npmjs.org/@elastic/transport/-/transport-8.4.0.tgz#e1ec05f7a2857162c161e2c97008f9b21301a673"
@ -2246,6 +2254,18 @@
    tslib "^2.4.0"
    undici "^5.22.1"

+"@elastic/transport@^8.4.1":
+  version "8.4.1"
+  resolved "https://registry.npmjs.org/@elastic/transport/-/transport-8.4.1.tgz#f98c5a5e2156bcb3f01170b4aca7e7de4d8b61b8"
+  integrity sha512-/SXVuVnuU5b4dq8OFY4izG+dmGla185PcoqgK6+AJMpmOeY1QYVNbWtCwvSvoAANN5D/wV+EBU8+x7Vf9EphbA==
+  dependencies:
+    debug "^4.3.4"
+    hpagent "^1.0.0"
+    ms "^2.1.3"
+    secure-json-parse "^2.4.0"
+    tslib "^2.4.0"
+    undici "^5.22.1"
+
 "@ember-data/adapter@3.24.0":
  version "3.24.0"
  resolved "https://registry.yarnpkg.com/@ember-data/adapter/-/adapter-3.24.0.tgz#995c19bc6fb95c94cbb83b8c3c7bc08253346cba"
@ -6856,6 +6876,14 @@
    "@tryghost/root-utils" "^0.3.25"
    debug "^4.3.1"

+"@tryghost/debug@^0.1.28":
+  version "0.1.28"
+  resolved "https://registry.npmjs.org/@tryghost/debug/-/debug-0.1.28.tgz#498ef3450aa654ebb15a47553c2478a33164c0e6"
+  integrity sha512-iZKKlDDcZZa77GCgZ+o/Vp5oz520SOOpKCnoapgKGkFLRFT/0/D54jw/KY2pHGTFBXrcrE8kqTulgeuMNP+ABA==
+  dependencies:
+    "@tryghost/root-utils" "^0.3.26"
+    debug "^4.3.1"
+
 "@tryghost/elasticsearch@^3.0.15":
  version "3.0.15"
  resolved "https://registry.yarnpkg.com/@tryghost/elasticsearch/-/elasticsearch-3.0.15.tgz#d4be60b79155d95de063e17ea90ff0151a0a35d9"
@ -6865,6 +6893,15 @@
    "@tryghost/debug" "^0.1.26"
    split2 "4.2.0"

+"@tryghost/elasticsearch@^3.0.17":
+  version "3.0.17"
+  resolved "https://registry.npmjs.org/@tryghost/elasticsearch/-/elasticsearch-3.0.17.tgz#408e8ba7ce35c9357f6814bf0fd9b88cb56c2ebb"
+  integrity sha512-4uYnFJQ0QDNleko1J26E0byWnHrEBZzd3S1WVTbCztlC14KQweZxmfou3fc5JmcT/GNiyXd5Pgx+bLMtVi017g==
+  dependencies:
+    "@elastic/elasticsearch" "8.12.2"
+    "@tryghost/debug" "^0.1.28"
+    split2 "4.2.0"
+
 "@tryghost/email-mock-receiver@0.3.2":
  version "0.3.2"
  resolved "https://registry.yarnpkg.com/@tryghost/email-mock-receiver/-/email-mock-receiver-0.3.2.tgz#abd8086935a95a996b6c5c803478a9f81dcae19a"
@ -6886,7 +6923,7 @@
    focus-trap "^6.7.2"
    postcss-preset-env "^7.3.1"

-"@tryghost/errors@1.2.26", "@tryghost/errors@1.3.1", "@tryghost/errors@^1.2.26", "@tryghost/errors@^1.2.27", "@tryghost/errors@^1.2.3":
+"@tryghost/errors@1.2.26", "@tryghost/errors@1.3.1", "@tryghost/errors@^1.2.26", "@tryghost/errors@^1.2.27", "@tryghost/errors@^1.2.3", "@tryghost/errors@^1.3.1":
  version "1.3.1"
  resolved "https://registry.npmjs.org/@tryghost/errors/-/errors-1.3.1.tgz#32a00c5e5293c46e54d03a66da871ac34b2ab35c"
  integrity sha512-iZqT0vZ3NVZNq9o1HYxW00k1mcUAC+t5OLiI8O29/uQwAfy7NemY+Cabl9mWoIwgvBmw7l0Z8pHTcXMo1c+xMw==
@ -6924,13 +6961,13 @@
  resolved "https://registry.yarnpkg.com/@tryghost/http-cache-utils/-/http-cache-utils-0.1.11.tgz#9b09921828f4772ac26c6d55b438a59e776e2b04"
  integrity sha512-PbUfViKtY0mxzOpC5Pdkx26R4jcYfWCvSWiTNIW2OG2k1CtE83nIRD/AanIcNMXxrRNnT/hdG9Yu3+gOhEqpmA==

-"@tryghost/http-stream@^0.1.25":
-  version "0.1.25"
-  resolved "https://registry.yarnpkg.com/@tryghost/http-stream/-/http-stream-0.1.25.tgz#cbe87996c67a2814c4a6706cd7e445a2b069246e"
-  integrity sha512-aeuGSvoXS1rjY+LupkqzJ9osKEkouzs/LrVqvOe7x9b/c8VUWIh6SrQ2KegKXtkvBggI8dBCH39bJiheZea9/Q==
+"@tryghost/http-stream@^0.1.28":
+  version "0.1.28"
+  resolved "https://registry.npmjs.org/@tryghost/http-stream/-/http-stream-0.1.28.tgz#ae38ef2e113e23b582d194fc76c5196a54171a11"
+  integrity sha512-E1TyZE5e121TPjMgJvlSPTcxJl3Xdq78kugmx2deLxADa5W/vuchidZ6yYQQzxCXPciQFUciVYT8B5tggZb+Xw==
  dependencies:
-    "@tryghost/errors" "^1.2.26"
-    "@tryghost/request" "^1.0.0"
+    "@tryghost/errors" "^1.3.1"
+    "@tryghost/request" "^1.0.3"

 "@tryghost/image-transform@1.2.11":
  version "1.2.11"
@ -7104,16 +7141,16 @@
    lodash "^4.17.21"
    luxon "^1.26.0"

-"@tryghost/logging@2.4.10", "@tryghost/logging@2.4.8", "@tryghost/logging@^2.4.7":
-  version "2.4.8"
-  resolved "https://registry.yarnpkg.com/@tryghost/logging/-/logging-2.4.8.tgz#a9e9abdbec823f0c6a009aa2f6847ce454b35475"
-  integrity sha512-/pIeTcw9jpqWJ5/VyUn5sa3rsUxUHortykB4oYd5vKr16KgnrVOuGPCg4JqmdGfz2zrkgKaGd9cAsTNE++0Deg==
+"@tryghost/logging@2.4.10", "@tryghost/logging@2.4.13", "@tryghost/logging@^2.4.7":
+  version "2.4.13"
+  resolved "https://registry.npmjs.org/@tryghost/logging/-/logging-2.4.13.tgz#61a6ab80d6367d8344e8779e67cb490ebd050f3f"
+  integrity sha512-BuYV1Wufds6TEeb+muRYJURD3YN216ZJBSZnOAFTWE4oC0q58o21VvQNFk8B/FJP9xnDF1Me6gOuutKQMWhSMg==
  dependencies:
    "@tryghost/bunyan-rotating-filestream" "^0.0.7"
-    "@tryghost/elasticsearch" "^3.0.15"
-    "@tryghost/http-stream" "^0.1.25"
-    "@tryghost/pretty-stream" "^0.1.20"
-    "@tryghost/root-utils" "^0.3.24"
+    "@tryghost/elasticsearch" "^3.0.17"
+    "@tryghost/http-stream" "^0.1.28"
+    "@tryghost/pretty-stream" "^0.1.22"
+    "@tryghost/root-utils" "^0.3.26"
    bunyan "^1.8.15"
    bunyan-loggly "^1.4.2"
    fs-extra "^11.0.0"
@ -7219,6 +7256,15 @@
    moment "^2.29.1"
    prettyjson "^1.2.5"

+"@tryghost/pretty-stream@^0.1.22":
+  version "0.1.22"
+  resolved "https://registry.npmjs.org/@tryghost/pretty-stream/-/pretty-stream-0.1.22.tgz#856a6b8eb3bd17b2fdc5ca668ac27bdd2f55b04b"
+  integrity sha512-97/JRI7rmdQIG6zwPzux58Kfc/UJfdvKiJgYgH7+CuNQqdl0Zy2+X0wlnRnYjck7tj781/rhGTEGGZg6PHZbaw==
+  dependencies:
+    lodash "^4.17.21"
+    moment "^2.29.1"
+    prettyjson "^1.2.5"
+
 "@tryghost/promise@0.3.4":
  version "0.3.4"
  resolved "https://registry.yarnpkg.com/@tryghost/promise/-/promise-0.3.4.tgz#b5437eb14a3d06e7d32f277e10975ff77061e16e"
@ -7229,7 +7275,7 @@
  resolved "https://registry.yarnpkg.com/@tryghost/promise/-/promise-0.3.6.tgz#b6b36442100564eab3d3de46595646722754113c"
  integrity sha512-ZCNc3GUxMd8A7PXKso1s3kCfQfXZgm+e7TCZ2bog/mfpU08EHWVOnAeRTygzlbooTTxTRsqUnMfaXnD5ZyOAdw==

-"@tryghost/request@1.0.0", "@tryghost/request@^1.0.0":
+"@tryghost/request@1.0.0":
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/@tryghost/request/-/request-1.0.0.tgz#2fc95c1a119e70a1e1cadd7c173beb810cf49809"
  integrity sha512-budMHtaaysFZT5gCIkbbsILiSXOxeiPKNGfUNtjKByuheLDhrQnz7E7+C0FWIIniMoJ6nI8FJdbixS9glpDOgQ==
@ -7241,6 +7287,18 @@
    got "13.0.0"
    lodash "^4.17.21"

+"@tryghost/request@^1.0.3":
+  version "1.0.3"
+  resolved "https://registry.npmjs.org/@tryghost/request/-/request-1.0.3.tgz#73f7417bd4eb382133ea1eaa092ff14d070c45aa"
+  integrity sha512-ruHs3omvxTYGIm87gGJSRx0r64y4mBWV52d0wiwgOeCyyYL2WDYQ1dTgHWZbSjl8YHc2p0lc8gkPPxBZ+9ZnUA==
+  dependencies:
+    "@tryghost/errors" "^1.3.1"
+    "@tryghost/validator" "^0.2.9"
+    "@tryghost/version" "^0.1.26"
+    cacheable-lookup "7.0.0"
+    got "13.0.0"
+    lodash "^4.17.21"
+
 "@tryghost/root-utils@0.3.24":
  version "0.3.24"
  resolved "https://registry.yarnpkg.com/@tryghost/root-utils/-/root-utils-0.3.24.tgz#91653fbadc882fb8510844f163a2231c87f30fab"
@ -7257,6 +7315,14 @@
    caller "^1.0.1"
    find-root "^1.1.0"

+"@tryghost/root-utils@^0.3.26":
+  version "0.3.26"
+  resolved "https://registry.npmjs.org/@tryghost/root-utils/-/root-utils-0.3.26.tgz#26e626706b67dddd13f59812826a806c45722122"
+  integrity sha512-akbI+mmnU6mlwbVAEy8Ay1MYbMEocKEVP+QHbYCKk3d++2TM2lmSZm6CFIP5+10NeFiP2Em6jSaGXBpaalR9VQ==
+  dependencies:
+    caller "^1.0.1"
+    find-root "^1.1.0"
+
 "@tryghost/server@^0.1.37":
  version "0.1.37"
  resolved "https://registry.yarnpkg.com/@tryghost/server/-/server-0.1.37.tgz#04ee5671b19a4a5be05e361e293d47eb9c6c2482"
@ -7296,6 +7362,13 @@
  dependencies:
    lodash.template "^4.5.0"

+"@tryghost/tpl@^0.1.28":
+  version "0.1.28"
+  resolved "https://registry.npmjs.org/@tryghost/tpl/-/tpl-0.1.28.tgz#c1453eedf33da7010b1c556f2e4d92f656351fd9"
+  integrity sha512-z8DBIDntaJQMHEmp/ZhCpPjc5TXIsu7ZdnOVbAVK2YnzhLcjDl/JPpmt2FXLV3VBo7VM1XBT9nptiYd2kFnZFg==
+  dependencies:
+    lodash.template "^4.5.0"
+
 "@tryghost/url-utils@4.4.6", "@tryghost/url-utils@^4.0.0":
  version "4.4.6"
  resolved "https://registry.yarnpkg.com/@tryghost/url-utils/-/url-utils-4.4.6.tgz#4938e55fcc11620fd17c64346249d420f6f97129"
@ -7320,6 +7393,17 @@
    moment-timezone "^0.5.23"
    validator "7.2.0"

+"@tryghost/validator@^0.2.9":
+  version "0.2.9"
+  resolved "https://registry.npmjs.org/@tryghost/validator/-/validator-0.2.9.tgz#6b33d884d96e0bca20a750d9dd1ed534a0efbab6"
+  integrity sha512-7EBFiXUGhU6ReuryAnqh5BM0Fa918NSEN3UR2dqrgk861W/pwofmx8r179jVKBNA0cSYot/DVj5bWlUV25cWvQ==
+  dependencies:
+    "@tryghost/errors" "^1.3.1"
+    "@tryghost/tpl" "^0.1.28"
+    lodash "^4.17.21"
+    moment-timezone "^0.5.23"
+    validator "7.2.0"
+
 "@tryghost/version@0.1.24", "@tryghost/version@^0.1.24":
  version "0.1.24"
  resolved "https://registry.yarnpkg.com/@tryghost/version/-/version-0.1.24.tgz#eb8bc345929ba8f67c3f36757bd91c12f701a5f5"
@ -7328,6 +7412,14 @@
    "@tryghost/root-utils" "^0.3.24"
    semver "^7.3.5"

+"@tryghost/version@^0.1.26":
+  version "0.1.26"
+  resolved "https://registry.npmjs.org/@tryghost/version/-/version-0.1.26.tgz#00829961bfef66b0aae01f6b866068df63859236"
+  integrity sha512-um1GihMBOs+1+p6tPGgIOHGlPeYyj0w+JxRHSIstPyywMmuM+kH3LUrHt3N3hb7zzgWfehM0L81k28MCGTqV5Q==
+  dependencies:
+    "@tryghost/root-utils" "^0.3.26"
+    semver "^7.3.5"
+
 "@tryghost/webhook-mock-receiver@0.2.8":
  version "0.2.8"
  resolved "https://registry.yarnpkg.com/@tryghost/webhook-mock-receiver/-/webhook-mock-receiver-0.2.8.tgz#1cb3bb5de667f597f2eaa25aff3e9e572212cafa"
@ -16449,6 +16541,13 @@ express-brute@1.0.1, express-brute@^1.0.1:
    long-timeout "~0.1.1"
    underscore "~1.8.3"

+express-end@0.0.8:
+  version "0.0.8"
+  resolved "https://registry.npmjs.org/express-end/-/express-end-0.0.8.tgz#0c8fd95428932158f2b4cf91f4045346bf2c5323"
+  integrity sha512-PPntzICAq006LBpXKBVJtmRUiCRqTMZ+OB8L2RFXgx+OmkMWU66IL4DTEPF/DOcxmsuC7Y0NdbT2R71lb+pBpg==
+  dependencies:
+    debug "^2.2.0"
+
 express-hbs@2.5.0, express-hbs@^2.4.2:
  version "2.5.0"
  resolved "https://registry.yarnpkg.com/express-hbs/-/express-hbs-2.5.0.tgz#28ed0b8507bb7bcceb102b63fe863585f13a4d89"
@ -16479,6 +16578,15 @@ express-query-boolean@2.0.0:
  resolved "https://registry.yarnpkg.com/express-query-boolean/-/express-query-boolean-2.0.0.tgz#ea56ac8138e2b95b171b8eee2af88738302941c3"
  integrity sha512-4dU/1HPm8lkTPR12+HFUXqCarcsC19OKOkb4otLOuADfPYrQMaugPJkSmxNsqwmWYjozvT6vdTiqkgeBHkzOow==

+express-queue@0.0.13:
+  version "0.0.13"
+  resolved "https://registry.npmjs.org/express-queue/-/express-queue-0.0.13.tgz#e9800d67749d4dfab7c34223f00595af933ce5df"
+  integrity sha512-C4OEDasGDqpXLrZICSUxbY47p5c0bKqf/3/3hwauSCmI+jVVxKBWU2w39BuKLP6nF65z87uDFBbJMPAn2ZrG3g==
+  dependencies:
+    debug "^4.3.4"
+    express-end "0.0.8"
+    mini-queue "0.0.14"
+
 express-session@1.18.0:
  version "1.18.0"
  resolved "https://registry.yarnpkg.com/express-session/-/express-session-1.18.0.tgz#a6ae39d9091f2efba5f20fc5c65a3ce7c9ce16a3"
@ -22282,6 +22390,13 @@ mini-css-extract-plugin@^2.5.2:
  dependencies:
    schema-utils "^4.0.0"

+mini-queue@0.0.14:
+  version "0.0.14"
+  resolved "https://registry.npmjs.org/mini-queue/-/mini-queue-0.0.14.tgz#83d2f3f908e3cac5390bd986d1e6fbbfa0d95b93"
+  integrity sha512-DNh9Wn8U1jrmn1yVfpviwClyER/Y4ltgGbG+LF/KIdKJ8BEo2Q9jDDPG7tEhz6F/DTZ/ohv5D7AAXFVSFyP05Q==
+  dependencies:
+    debug "^3.1.0"
+
 mini-svg-data-uri@^1.2.3:
  version "1.4.4"
  resolved "https://registry.yarnpkg.com/mini-svg-data-uri/-/mini-svg-data-uri-1.4.4.tgz#8ab0aabcdf8c29ad5693ca595af19dd2ead09939"
@ -28088,7 +28203,7 @@ string-template@~0.2.1:
  resolved "https://registry.yarnpkg.com/string-template/-/string-template-0.2.1.tgz#42932e598a352d01fc22ec3367d9d84eec6c9add"
  integrity sha512-Yptehjogou2xm4UJbxJ4CxgZx12HBfeystp0y3x7s4Dj32ltVVG1Gg8YhKjHZkHicuKpZX/ffilA8505VbUbpw==

-"string-width-cjs@npm:string-width@^4.2.0":
+"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
  version "4.2.3"
  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@ -28106,15 +28221,6 @@ string-width@^1.0.1:
    is-fullwidth-code-point "^1.0.0"
    strip-ansi "^3.0.0"

-"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
-  version "4.2.3"
-  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
-  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
-  dependencies:
-    emoji-regex "^8.0.0"
-    is-fullwidth-code-point "^3.0.0"
-    strip-ansi "^6.0.1"
-
 string-width@^2.1.0:
  version "2.1.1"
  resolved "https://registry.yarnpkg.com/string-width/-/string-width-2.1.1.tgz#ab93f27a8dc13d28cac815c462143a6d9012ae9e"
@ -28203,7 +28309,7 @@ stringify-entities@^2.0.0:
    is-decimal "^1.0.2"
    is-hexadecimal "^1.0.0"

-"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
  version "6.0.1"
  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@ -28231,13 +28337,6 @@ strip-ansi@^5.1.0, strip-ansi@^5.2.0:
  dependencies:
    ansi-regex "^4.1.0"

-strip-ansi@^6.0.0, strip-ansi@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
-  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
-  dependencies:
-    ansi-regex "^5.0.1"
-
 strip-ansi@^7.0.1:
  version "7.0.1"
  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2"
@ -30673,7 +30772,7 @@ workerpool@^6.0.2, workerpool@^6.0.3, workerpool@^6.1.5, workerpool@^6.4.0:
  resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.5.1.tgz#060f73b39d0caf97c6db64da004cd01b4c099544"
  integrity sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA==

-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
  version "7.0.0"
  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@ -30691,15 +30790,6 @@ wrap-ansi@^6.0.1:
    string-width "^4.1.0"
    strip-ansi "^6.0.0"

-wrap-ansi@^7.0.0:
-  version "7.0.0"
-  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
-  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
-  dependencies:
-    ansi-styles "^4.0.0"
-    string-width "^4.1.0"
-    strip-ansi "^6.0.0"
-
 wrap-ansi@^8.0.1, wrap-ansi@^8.1.0:
  version "8.1.0"
  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"