Added job-manager & wired up shutdown and testmode

- Bottom line - we need to manage shutting down gracefully when doing long-running tasks
- To achieve that, we're going to use job queues

In this commit:
- added new @tryghost/job-manager dependency
- added a minimal job service, that handles in passing things like logging and (maybe later) config
- job service is wired up to server shutdown, so that the queue finishes before the server exits
- also added a new job endpoint to testmode so that it's easy to test job behaviour without needing to do real work

core/server/ghost-server.js

@@ -29,6 +29,9 @@ class GhostServer {
 
         // Expose config module for use externally.
         this.config = config;
+
+        // Tasks that should be run before the server exits
+        this.cleanupTasks = [];
     }
 
     /**
@@ -150,14 +153,22 @@ class GhostServer {
      * @returns {Promise} Resolves once Ghost has stopped
      */
     async stop() {
+        // If we never fully started, there's nothing to stop
         if (this.httpServer === null) {
             return;
         }
 
-        await this._stopServer();
-        events.emit('server.stop');
-        this.httpServer = null;
-        this._logStopMessages();
+        try {
+            // We stop the server first so that no new long running requests or processes can be started
+            await this._stopServer();
+            // Do all of the cleanup tasks
+            await this._cleanup();
+        } finally {
+            // Wrap up
+            events.emit('server.stop');
+            this.httpServer = null;
+            this._logStopMessages();
+        }
     }
 
     /**
@@ -168,6 +179,10 @@ class GhostServer {
         logging.info(i18n.t('notices.httpServer.cantTouchThis'));
     }
 
+    registerCleanupTask(task) {
+        this.cleanupTasks.push(task);
+    }
+
     /**
      * ### Stop Server
      * Does the work of stopping the server using stoppable
@@ -184,6 +199,19 @@ class GhostServer {
         });
     }
 
+    async _cleanup() {
+        // Wait for all cleanup tasks to finish
+        await Promise
+            .all(this.cleanupTasks.map(task => task()));
+    }
+
+    _onShutdownComplete() {
+        // Wrap up
+        events.emit('server.stop');
+        this.httpServer = null;
+        this._logStopMessages();
+    }
+
     /**
      * ### Log Start Messages
      */

core/server/index.js

@@ -76,6 +76,7 @@ function initialiseServices() {
  */
 const minimalRequiredSetupToStartGhost = (dbState) => {
     const settings = require('./services/settings');
+    const jobService = require('./services/jobs');
     const models = require('./models');
     const GhostServer = require('./ghost-server');
 
@@ -112,6 +113,10 @@ const minimalRequiredSetupToStartGhost = (dbState) => {
         .then((_ghostServer) => {
             ghostServer = _ghostServer;
 
+            ghostServer.registerCleanupTask(async () => {
+                await jobService.shutdown();
+            });
+
             // CASE: all good or db was just initialised
             if (dbState === 1 || dbState === 2) {
                 events.emit('db.ready');

core/server/services/jobs/index.js

@@ -0,0 +1 @@
+module.exports = require('./job-service');

core/server/services/jobs/job-service.js

@@ -0,0 +1,11 @@
+/**
+ * Minimal wrapper around our external lib
+ * Intended for passing any Ghost internals such as logging and config
+ */
+
+const JobManager = require('@tryghost/job-manager');
+const logging = require('../../../shared/logging');
+
+const jobManager = new JobManager(logging);
+
+module.exports = jobManager;

core/server/web/api/testmode.js

@@ -1,5 +1,6 @@
 const logging = require('../../../shared/logging');
 const express = require('../../../shared/express');
+const jobService = require('../../services/jobs');
 
 /** A bunch of helper routes for testing purposes */
 module.exports = function testRoutes() {
@@ -19,6 +20,25 @@ module.exports = function testRoutes() {
             res.sendStatus(200);
         }, timeout);
     });
+    router.get('/job/:timeout', (req, res) => {
+        if (!req.params || !req.params.timeout) {
+            return res.sendStatus(200);
+        }
+
+        const timeout = req.params.timeout * 1000;
+        logging.info('Create Slow Job with timeout of', timeout);
+        jobService.addJob(() => {
+            return new Promise((resolve) => {
+                logging.info('Start Slow Job');
+                setTimeout(() => {
+                    logging.info('End Slow Job', timeout);
+                    resolve();
+                }, timeout);
+            }, {timeout});
+        });
+
+        res.sendStatus(202);
+    });
 
     return router;
 };

package.json

@@ -48,6 +48,7 @@
     "@tryghost/errors": "0.2.3",
     "@tryghost/helpers": "1.1.29",
     "@tryghost/image-transform": "1.0.3",
+    "@tryghost/job-manager": "0.1.0",
     "@tryghost/kg-card-factory": "2.1.1",
     "@tryghost/kg-default-atoms": "2.0.1",
     "@tryghost/kg-default-cards": "2.4.1",

yarn.lock

@@ -342,6 +342,14 @@
   optionalDependencies:
     sharp "0.25.4"
 
+"@tryghost/job-manager@0.1.0":
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/@tryghost/job-manager/-/job-manager-0.1.0.tgz#988f93ce356f98dbb8b763ee9a10a7a5ca7c1df1"
+  integrity sha512-Xd7fSn9J/iz08YtcYXpd8+xgzNPVOw3GvEdp0Q9V/ve3vknVvDi2tHVthTKVRSTZhxcK8maYOAMf9Wfk8MKq8w==
+  dependencies:
+    fastq "1.8.0"
+    p-wait-for "3.1.0"
+
 "@tryghost/kg-card-factory@2.1.1":
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/@tryghost/kg-card-factory/-/kg-card-factory-2.1.1.tgz#c157da23969bec651a021b79656c249420921e89"
@@ -3223,6 +3231,13 @@ fast-levenshtein@^2.0.6, fast-levenshtein@~2.0.6:
   resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917"
   integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=
 
+fastq@1.8.0:
+  version "1.8.0"
+  resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.8.0.tgz#550e1f9f59bbc65fe185cb6a9b4d95357107f481"
+  integrity sha512-SMIZoZdLh/fgofivvIkmknUXyPnvxRE3DhtZ5Me3Mrsk5gyPL42F0xr51TdRXskBxHfMp+07bcYzfsYEsSQA9Q==
+  dependencies:
+    reusify "^1.0.4"
+
 faye-websocket@~0.10.0:
   version "0.10.0"
   resolved "https://registry.yarnpkg.com/faye-websocket/-/faye-websocket-0.10.0.tgz#4e492f8d04dfb6f89003507f6edbf2d501e7c6f4"
@@ -6626,7 +6641,7 @@ p-locate@^4.1.0:
   dependencies:
     p-limit "^2.2.0"
 
-p-timeout@^3.1.0:
+p-timeout@^3.0.0, p-timeout@^3.1.0:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe"
   integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==
@@ -6638,6 +6653,13 @@ p-try@^2.0.0:
   resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6"
   integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==
 
+p-wait-for@3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/p-wait-for/-/p-wait-for-3.1.0.tgz#9da568a2adda3ea8175a3c43f46a5317e28c0e47"
+  integrity sha512-0Uy19uhxbssHelu9ynDMcON6BmMk6pH8551CvxROhiz3Vx+yC4RqxjyIDk2V4ll0g9177RKT++PK4zcV58uJ7A==
+  dependencies:
+    p-timeout "^3.0.0"
+
 pac-proxy-agent@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/pac-proxy-agent/-/pac-proxy-agent-3.0.1.tgz#115b1e58f92576cac2eba718593ca7b0e37de2ad"
@@ -7723,6 +7745,11 @@ ret@~0.1.10:
   resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc"
   integrity sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==
 
+reusify@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76"
+  integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==
+
 rewire@5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/rewire/-/rewire-5.0.0.tgz#c4e6558206863758f6234d8f11321793ada2dbff"