From 5c4e884070ffcfce77b368b83ff3e3266784f8ba Mon Sep 17 00:00:00 2001 From: Naz Date: Mon, 23 Nov 2020 17:17:49 +1300 Subject: [PATCH] Added support for graceful job shutdown and worker threads refs #12402 - With bumped version of job-manager it offloads job procesing into separate worker thread. Having jobs run out of main Ghost process even loop allows for safe job execution, which does not block Ghost from serving requests or performing other functions without a delay - Added experimental data access to 'testmode' jobs. This should serve as an illustration of how to access data from the job layer --- core/server/ghost-server.js | 5 +- core/server/web/api/testmode/index.js | 2 +- core/server/web/api/testmode/jobs/cpu-hog.js | 15 +++ .../web/api/testmode/jobs/graceful-job.js | 48 ++++++++++ package.json | 3 +- yarn.lock | 94 ++++++++++--------- 6 files changed, 118 insertions(+), 49 deletions(-) create mode 100644 core/server/web/api/testmode/jobs/cpu-hog.js create mode 100644 core/server/web/api/testmode/jobs/graceful-job.js diff --git a/core/server/ghost-server.js b/core/server/ghost-server.js index 861c3a8d10..caac7faf4c 100644 --- a/core/server/ghost-server.js +++ b/core/server/ghost-server.js @@ -124,7 +124,10 @@ class GhostServer { }); // Output job queue length every 5 seconds - setInterval(() => logging.warn(`${jobService.queue.length()} jobs in the queue. Idle: ${jobService.queue.idle()}`), 5000); + setInterval(() => { + logging.warn(`${jobService.queue.length()} jobs in the queue. Idle: ${jobService.queue.idle()}`); + logging.warn(`${Object.keys(jobService.bree.workers).length} workers registered. Scheduled jobs: ${Object.keys(jobService.bree.intervals).length}`); + }, 5000); } return GhostServer.announceServerReadiness() diff --git a/core/server/web/api/testmode/index.js b/core/server/web/api/testmode/index.js index ae671fdfa5..0479369ddf 100644 --- a/core/server/web/api/testmode/index.js +++ b/core/server/web/api/testmode/index.js @@ -50,7 +50,7 @@ module.exports = function testRoutes() { logging.info('Achedule a Job with schedule of:', schedule, req.params.name); if (req.params.name) { - const jobPath = path.resolve(__dirname, 'jobs', req.params.name); + const jobPath = path.resolve(__dirname, 'jobs', `${req.params.name}.js`); jobService.scheduleJob(schedule, jobPath); } else { jobService.scheduleJob(schedule, () => { diff --git a/core/server/web/api/testmode/jobs/cpu-hog.js b/core/server/web/api/testmode/jobs/cpu-hog.js new file mode 100644 index 0000000000..9cf83a90d4 --- /dev/null +++ b/core/server/web/api/testmode/jobs/cpu-hog.js @@ -0,0 +1,15 @@ +const logging = require('../../../../../shared/logging'); + +(async () => { + let sum = 0; + + logging.info(`Starting CPU intensive task at ${new Date()}`); + + for (let i = 0; i < 100000000; i++) { + sum += Math.tan(Math.tan(i)); + } + + logging.info(`Calculation result: ${sum}`); + + logging.info(`Finishing CPU intensive task at ${new Date()}`); +})(); diff --git a/core/server/web/api/testmode/jobs/graceful-job.js b/core/server/web/api/testmode/jobs/graceful-job.js new file mode 100644 index 0000000000..46b90e484a --- /dev/null +++ b/core/server/web/api/testmode/jobs/graceful-job.js @@ -0,0 +1,48 @@ +const {parentPort} = require('bthreads'); +const util = require('util'); +const logging = require('../../../../../shared/logging'); +const models = require('../../../../models'); + +let shutdown = false; + +parentPort.on('message', (message) => { + logging.info(`paret message received: ${message}`); + + if (message === 'cancel') { + shutdown = true; + } +}); + +const setTimeoutPromise = util.promisify(setTimeout); +const internalContext = {context: {internal: true}}; + +(async () => { + try { + await models.init(); + + logging.info(`Fetching tags`); + const tags = await models.Tag.findPage(internalContext); + + logging.info(`Found ${tags.data.length} tags. First one: ${tags.data[0].toJSON().slug}`); + + logging.info(`Waiting 5 seconds to perform second part of the job`); + await setTimeoutPromise(5 * 1000); + + if (shutdown) { + logging.info(`Job shutting down gracefully`); + process.exit(0); + } + + logging.info(`Fetching posts`); + const posts = await models.Post.findPage(internalContext); + + logging.info(`Found ${posts.data.length} posts. First one: ${posts.data[0].toJSON().slug}`); + + logging.info('Graceful job has completed!'); + + process.exit(0); + } catch (err) { + logging.error(err); + process.exit(1); + } +})(); diff --git a/package.json b/package.json index 32a4469a76..bdd54cce14 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "@tryghost/errors": "0.2.5", "@tryghost/helpers": "1.1.34", "@tryghost/image-transform": "1.0.3", - "@tryghost/job-manager": "0.3.0", + "@tryghost/job-manager": "0.3.1", "@tryghost/kg-card-factory": "2.1.4", "@tryghost/kg-default-atoms": "2.0.2", "@tryghost/kg-default-cards": "3.0.1", @@ -77,6 +77,7 @@ "bookshelf-relations": "1.3.2", "brute-knex": "4.0.1", "bson-objectid": "1.3.1", + "bthreads": "0.5.1", "cheerio": "0.22.0", "compression": "1.7.4", "connect-slashes": "1.4.0", diff --git a/yarn.lock b/yarn.lock index d76dac4eff..790e30dca9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -23,7 +23,7 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@babel/runtime@^7.10.5", "@babel/runtime@^7.11.2": +"@babel/runtime@^7.10.5", "@babel/runtime@^7.12.5": version "7.12.5" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.12.5.tgz#410e7e487441e1b360c29be715d870d9b985882e" integrity sha512-plcc+hbExy3McchJCEQG3knOsuh3HH+Prx1P6cLIkET/0dLuQDEnrT+s27Axgc9bqfsmNUNHfscgMUdBpC9xfg== @@ -449,14 +449,14 @@ optionalDependencies: sharp "0.25.4" -"@tryghost/job-manager@0.3.0": - version "0.3.0" - resolved "https://registry.yarnpkg.com/@tryghost/job-manager/-/job-manager-0.3.0.tgz#c68fe9adfd6c73f2f418e27fce0a08efdf68e33f" - integrity sha512-EB23lziNQuoVHrDJXEgxXBmM5uxVKNYBHPWwfVcPJ2zWv8616cu1xytH9jMG9GSFbWcZDChhb6hFuoMKgiPx6g== +"@tryghost/job-manager@0.3.1": + version "0.3.1" + resolved "https://registry.yarnpkg.com/@tryghost/job-manager/-/job-manager-0.3.1.tgz#b3220b4a93e79b3254e4fbb83111afe31cb41dd7" + integrity sha512-Vtk5zR1OFEx6pRohH/GkbEIeSk12JNgOOgRWprQOluQVzMdcPTBBizSlP1713BSk6/Is7CXAIsdtLGsi5CX1ww== dependencies: "@breejs/later" "4.0.2" - bree "3.4.0" - cron-validate "1.4.0" + bree "4.0.0" + cron-validate "1.4.1" fastq "1.9.0" p-wait-for "3.1.0" @@ -1363,7 +1363,7 @@ boolbase@^1.0.0, boolbase@~1.0.0: resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" integrity sha1-aN/1++YMUes3cl6p4+0xDcwed24= -boolean@^3.0.1: +boolean@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/boolean/-/boolean-3.0.2.tgz#df1baa18b6a2b0e70840475e1d93ec8fe75b2570" integrity sha512-RwywHlpCRc3/Wh81MiCKun4ydaIFyW5Ea6JbL6sRCVx5q5irDw7pMXBUFYF/jArQ6YrG36q0kpovc9P/Kd3I4g== @@ -1399,19 +1399,19 @@ braces@~3.0.2: dependencies: fill-range "^7.0.1" -bree@3.4.0: - version "3.4.0" - resolved "https://registry.yarnpkg.com/bree/-/bree-3.4.0.tgz#1af7660df7cba7467a471052408e9211f151bd7f" - integrity sha512-jxTj5O36jUNo5QW2vn40559Atp5mZhZIPVMPv/B/lRg0UxgLGF/VtOwY7D2levyLon6ZY469AmJEfx8/sjOklg== +bree@4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/bree/-/bree-4.0.0.tgz#1ce6b74cbbfd49fb87c171001f37ddcec1cea996" + integrity sha512-z9vM8rc4KNEzn8j+XOcJDt65ah2G/zuXkEeR2ovzX9A3kaoOL/jo41YLwdTZQ542YBOHcHomzd5pc6CkHAADgQ== dependencies: - "@babel/runtime" "^7.11.2" + "@babel/runtime" "^7.12.5" "@breejs/later" "^4.0.2" - boolean "^3.0.1" + boolean "^3.0.2" bthreads "^0.5.1" combine-errors "^3.0.3" - cron-validate "^1.3.0" - debug "^4.1.1" - human-interval "^1.0.0" + cron-validate "^1.4.1" + debug "^4.3.1" + human-interval "^2.0.0" is-string-and-not-blank "^0.0.2" is-valid-path "^0.1.1" ms "^2.1.2" @@ -1458,7 +1458,7 @@ bson-objectid@1.3.1: resolved "https://registry.yarnpkg.com/bson-objectid/-/bson-objectid-1.3.1.tgz#11e4ce4c3419161fd388113781bb62c1dfbce34b" integrity sha512-eQBNQXsisEAXlwiSy8zRNZdW2xDBJaEVkTPbodYR9hGxxtE548Qq7ilYOd8WAQ86xF7NRUdiWSQ1pa/TkKiE2A== -bthreads@^0.5.1: +bthreads@0.5.1, bthreads@^0.5.1: version "0.5.1" resolved "https://registry.yarnpkg.com/bthreads/-/bthreads-0.5.1.tgz#c7a4dacc2d159c50de08b37b1e2a7da836171063" integrity sha512-nK7Jo9ll+r1FRMNPWEFRTZMQrX6HhX8JjPAofxmbTNILHqWVIJPmWzCi9JlX/K0DL5AKZTFZg2Qser5C6gVs9A== @@ -2238,12 +2238,12 @@ create-error@~0.3.1: resolved "https://registry.yarnpkg.com/create-error/-/create-error-0.3.1.tgz#69810245a629e654432bf04377360003a5351a23" integrity sha1-aYECRaYp5lRDK/BDdzYAA6U1GiM= -cron-validate@1.4.0, cron-validate@^1.3.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/cron-validate/-/cron-validate-1.4.0.tgz#aaf5c789db5226ad82b8168ec7654023984c27b3" - integrity sha512-z4Mik62Wdvgn1WC9OCGdwxTYO6TCjasTw3SAYsVq0tfArYh6Zg7bbgGVVNBiNAcnnxXsdnBj4AatkYunntaT9A== +cron-validate@1.4.1, cron-validate@^1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/cron-validate/-/cron-validate-1.4.1.tgz#d637f677a2f5fa37ed61d265a17af502b02b12d6" + integrity sha512-83VZ6/I52ybU8lwvrAgjnwR70/1pl867pgE4LbzO+ha+rZOmNMRqMm8vVG9s7a2fpnBQR6/+F7oRiGVfnC+9Ug== dependencies: - yup "0.29.3" + yup "0.30.0" cross-spawn@^5.0.1: version "5.1.0" @@ -2511,6 +2511,13 @@ debug@4.2.0: dependencies: ms "2.1.2" +debug@^4.3.1: + version "4.3.1" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" + integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ== + dependencies: + ms "2.1.2" + decamelize@^1.1.1, decamelize@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -3664,11 +3671,6 @@ flatted@^2.0.0: resolved "https://registry.yarnpkg.com/flatted/-/flatted-2.0.2.tgz#4575b21e2bcee7434aa9be662f4b7b5f9c2b5138" integrity sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA== -fn-name@~3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/fn-name/-/fn-name-3.0.0.tgz#0596707f635929634d791f452309ab41558e3c5c" - integrity sha512-eNMNr5exLoavuAMhIUVsOKF79SWd/zG104ef6sxBTSw+cZc6BXdQXDvYcGvp0VbxVVSp1XDUNoz7mg1xMtSznA== - follow-redirects@0.0.3: version "0.0.3" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-0.0.3.tgz#6ce67a24db1fe13f226c1171a72a7ef2b17b8f65" @@ -4584,10 +4586,12 @@ https-proxy-agent@^5.0.0: agent-base "6" debug "4" -human-interval@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/human-interval/-/human-interval-1.0.0.tgz#7ba00a15f3d94ab6a4c16f76060e4aa07c713019" - integrity sha512-SWPw3rD6/ocA0JnGePoXp5Zf5eILzsoL5vdWdLwtTuyrElyCpfQb0whIcxMdK/gAKNl2rFDGkPAbwI2KGZCvNA== +human-interval@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/human-interval/-/human-interval-2.0.0.tgz#a9f4cedf4544005398c39767a484fdf062f4287a" + integrity sha512-r9qKIElAPkR4+uKSY2T7ImenvKItsLhT2Rm3OmFbA0pBt44YC8mWBpSFsnAbHnVs2D47hm0GquhpCjN+yjR24g== + dependencies: + numbered "^1.1.0" iconv-lite@0.4.24, iconv-lite@^0.4.24, iconv-lite@^0.4.4, iconv-lite@~0.4.13: version "0.4.24" @@ -6812,6 +6816,11 @@ number-is-nan@^1.0.0: resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0= +numbered@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/numbered/-/numbered-1.1.0.tgz#9fcd79564c73a84b9574e8370c3d8e58fe3c133c" + integrity sha512-pv/ue2Odr7IfYOO0byC1KgBI10wo5YDauLhxY6/saNzAdAs0r1SotGCPzzCLNPL0xtrAwWRialLu23AAu9xO1g== + nwsapi@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.0.tgz#204879a9e3d068ff2a55139c2c772780681a38b7" @@ -7681,7 +7690,7 @@ propagate@^2.0.0: resolved "https://registry.yarnpkg.com/propagate/-/propagate-2.0.1.tgz#40cdedab18085c792334e64f0ac17256d38f9a45" integrity sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag== -property-expr@^2.0.2: +property-expr@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/property-expr/-/property-expr-2.0.4.tgz#37b925478e58965031bb612ec5b3260f8241e910" integrity sha512-sFPkHQjVKheDNnPvotjQmm3KD3uk1fWKUN7CrpdbwmUx3CrG3QiM8QpTSimvig5vTXmTvjz7+TDvXOI9+4rkcg== @@ -8988,11 +8997,6 @@ symbol-tree@^3.2.4: resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2" integrity sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw== -synchronous-promise@^2.0.13: - version "2.0.15" - resolved "https://registry.yarnpkg.com/synchronous-promise/-/synchronous-promise-2.0.15.tgz#07ca1822b9de0001f5ff73595f3d08c4f720eb8e" - integrity sha512-k8uzYIkIVwmT+TcglpdN50pS2y1BDcUnBPK9iJeGu0Pl1lOI8pD6wtzgw91Pjpe+RxtTncw32tLxs/R0yNL2Mg== - sywac@1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/sywac/-/sywac-1.3.0.tgz#324789bdb8bd7d0d66625c9144fce81ab5ba6f99" @@ -9966,17 +9970,15 @@ yauzl@^2.10.0: buffer-crc32 "~0.2.3" fd-slicer "~1.1.0" -yup@0.29.3: - version "0.29.3" - resolved "https://registry.yarnpkg.com/yup/-/yup-0.29.3.tgz#69a30fd3f1c19f5d9e31b1cf1c2b851ce8045fea" - integrity sha512-RNUGiZ/sQ37CkhzKFoedkeMfJM0vNQyaz+wRZJzxdKE7VfDeVKH8bb4rr7XhRLbHJz5hSjoDNwMEIaKhuMZ8gQ== +yup@0.30.0: + version "0.30.0" + resolved "https://registry.yarnpkg.com/yup/-/yup-0.30.0.tgz#427ee076abb1d7e01e747fb782801f7df252fb76" + integrity sha512-GX3vqpC9E+Ow0fmQPgqbEg7UV40XRrN1IOEgKF5v04v6T4ha2vBas/hu0thWgewk8L4wUEBLRO/EnXwYyP+p+A== dependencies: "@babel/runtime" "^7.10.5" - fn-name "~3.0.0" - lodash "^4.17.15" + lodash "^4.17.20" lodash-es "^4.17.11" - property-expr "^2.0.2" - synchronous-promise "^2.0.13" + property-expr "^2.0.4" toposort "^2.0.2" zip-stream@^1.1.0: