mirror of
https://github.com/TryGhost/Ghost.git
synced 2025-02-10 23:36:14 -05:00
Added support for graceful job shutdown and worker threads
refs #12402 - With bumped version of job-manager it offloads job procesing into separate worker thread. Having jobs run out of main Ghost process even loop allows for safe job execution, which does not block Ghost from serving requests or performing other functions without a delay - Added experimental data access to 'testmode' jobs. This should serve as an illustration of how to access data from the job layer
This commit is contained in:
Naz
parent
73911f7ba4
commit
5c4e884070
6 changed files with 118 additions and 49 deletions
|
|
@ -124,7 +124,10 @@ class GhostServer {
|
|||
});
|
||||
|
||||
// Output job queue length every 5 seconds
|
||||
setInterval(() => logging.warn(`${jobService.queue.length()} jobs in the queue. Idle: ${jobService.queue.idle()}`), 5000);
|
||||
setInterval(() => {
|
||||
logging.warn(`${jobService.queue.length()} jobs in the queue. Idle: ${jobService.queue.idle()}`);
|
||||
logging.warn(`${Object.keys(jobService.bree.workers).length} workers registered. Scheduled jobs: ${Object.keys(jobService.bree.intervals).length}`);
|
||||
}, 5000);
|
||||
}
|
||||
|
||||
return GhostServer.announceServerReadiness()
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ module.exports = function testRoutes() {
|
|||
logging.info('Achedule a Job with schedule of:', schedule, req.params.name);
|
||||
|
||||
if (req.params.name) {
|
||||
const jobPath = path.resolve(__dirname, 'jobs', req.params.name);
|
||||
const jobPath = path.resolve(__dirname, 'jobs', `${req.params.name}.js`);
|
||||
jobService.scheduleJob(schedule, jobPath);
|
||||
} else {
|
||||
jobService.scheduleJob(schedule, () => {
|
||||
|
|
|
|||
15
core/server/web/api/testmode/jobs/cpu-hog.js
Normal file
15
core/server/web/api/testmode/jobs/cpu-hog.js
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
const logging = require('../../../../../shared/logging');
|
||||
|
||||
(async () => {
|
||||
let sum = 0;
|
||||
|
||||
logging.info(`Starting CPU intensive task at ${new Date()}`);
|
||||
|
||||
for (let i = 0; i < 100000000; i++) {
|
||||
sum += Math.tan(Math.tan(i));
|
||||
}
|
||||
|
||||
logging.info(`Calculation result: ${sum}`);
|
||||
|
||||
logging.info(`Finishing CPU intensive task at ${new Date()}`);
|
||||
})();
|
||||
48
core/server/web/api/testmode/jobs/graceful-job.js
Normal file
48
core/server/web/api/testmode/jobs/graceful-job.js
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
const {parentPort} = require('bthreads');
|
||||
const util = require('util');
|
||||
const logging = require('../../../../../shared/logging');
|
||||
const models = require('../../../../models');
|
||||
|
||||
let shutdown = false;
|
||||
|
||||
parentPort.on('message', (message) => {
|
||||
logging.info(`paret message received: ${message}`);
|
||||
|
||||
if (message === 'cancel') {
|
||||
shutdown = true;
|
||||
}
|
||||
});
|
||||
|
||||
const setTimeoutPromise = util.promisify(setTimeout);
|
||||
const internalContext = {context: {internal: true}};
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
await models.init();
|
||||
|
||||
logging.info(`Fetching tags`);
|
||||
const tags = await models.Tag.findPage(internalContext);
|
||||
|
||||
logging.info(`Found ${tags.data.length} tags. First one: ${tags.data[0].toJSON().slug}`);
|
||||
|
||||
logging.info(`Waiting 5 seconds to perform second part of the job`);
|
||||
await setTimeoutPromise(5 * 1000);
|
||||
|
||||
if (shutdown) {
|
||||
logging.info(`Job shutting down gracefully`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
logging.info(`Fetching posts`);
|
||||
const posts = await models.Post.findPage(internalContext);
|
||||
|
||||
logging.info(`Found ${posts.data.length} posts. First one: ${posts.data[0].toJSON().slug}`);
|
||||
|
||||
logging.info('Graceful job has completed!');
|
||||
|
||||
process.exit(0);
|
||||
} catch (err) {
|
||||
logging.error(err);
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
|
|
@ -49,7 +49,7 @@
|
|||
"@tryghost/errors": "0.2.5",
|
||||
"@tryghost/helpers": "1.1.34",
|
||||
"@tryghost/image-transform": "1.0.3",
|
||||
"@tryghost/job-manager": "0.3.0",
|
||||
"@tryghost/job-manager": "0.3.1",
|
||||
"@tryghost/kg-card-factory": "2.1.4",
|
||||
"@tryghost/kg-default-atoms": "2.0.2",
|
||||
"@tryghost/kg-default-cards": "3.0.1",
|
||||
|
|
@ -77,6 +77,7 @@
|
|||
"bookshelf-relations": "1.3.2",
|
||||
"brute-knex": "4.0.1",
|
||||
"bson-objectid": "1.3.1",
|
||||
"bthreads": "0.5.1",
|
||||
"cheerio": "0.22.0",
|
||||
"compression": "1.7.4",
|
||||
"connect-slashes": "1.4.0",
|
||||
|
|
|
|||
94
yarn.lock
94
yarn.lock
|
|
@ -23,7 +23,7 @@
|
|||
chalk "^2.0.0"
|
||||
js-tokens "^4.0.0"
|
||||
|
||||
"@babel/runtime@^7.10.5", "@babel/runtime@^7.11.2":
|
||||
"@babel/runtime@^7.10.5", "@babel/runtime@^7.12.5":
|
||||
version "7.12.5"
|
||||
resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.12.5.tgz#410e7e487441e1b360c29be715d870d9b985882e"
|
||||
integrity sha512-plcc+hbExy3McchJCEQG3knOsuh3HH+Prx1P6cLIkET/0dLuQDEnrT+s27Axgc9bqfsmNUNHfscgMUdBpC9xfg==
|
||||
|
|
@ -449,14 +449,14 @@
|
|||
optionalDependencies:
|
||||
sharp "0.25.4"
|
||||
|
||||
"@tryghost/job-manager@0.3.0":
|
||||
version "0.3.0"
|
||||
resolved "https://registry.yarnpkg.com/@tryghost/job-manager/-/job-manager-0.3.0.tgz#c68fe9adfd6c73f2f418e27fce0a08efdf68e33f"
|
||||
integrity sha512-EB23lziNQuoVHrDJXEgxXBmM5uxVKNYBHPWwfVcPJ2zWv8616cu1xytH9jMG9GSFbWcZDChhb6hFuoMKgiPx6g==
|
||||
"@tryghost/job-manager@0.3.1":
|
||||
version "0.3.1"
|
||||
resolved "https://registry.yarnpkg.com/@tryghost/job-manager/-/job-manager-0.3.1.tgz#b3220b4a93e79b3254e4fbb83111afe31cb41dd7"
|
||||
integrity sha512-Vtk5zR1OFEx6pRohH/GkbEIeSk12JNgOOgRWprQOluQVzMdcPTBBizSlP1713BSk6/Is7CXAIsdtLGsi5CX1ww==
|
||||
dependencies:
|
||||
"@breejs/later" "4.0.2"
|
||||
bree "3.4.0"
|
||||
cron-validate "1.4.0"
|
||||
bree "4.0.0"
|
||||
cron-validate "1.4.1"
|
||||
fastq "1.9.0"
|
||||
p-wait-for "3.1.0"
|
||||
|
||||
|
|
@ -1363,7 +1363,7 @@ boolbase@^1.0.0, boolbase@~1.0.0:
|
|||
resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e"
|
||||
integrity sha1-aN/1++YMUes3cl6p4+0xDcwed24=
|
||||
|
||||
boolean@^3.0.1:
|
||||
boolean@^3.0.2:
|
||||
version "3.0.2"
|
||||
resolved "https://registry.yarnpkg.com/boolean/-/boolean-3.0.2.tgz#df1baa18b6a2b0e70840475e1d93ec8fe75b2570"
|
||||
integrity sha512-RwywHlpCRc3/Wh81MiCKun4ydaIFyW5Ea6JbL6sRCVx5q5irDw7pMXBUFYF/jArQ6YrG36q0kpovc9P/Kd3I4g==
|
||||
|
|
@ -1399,19 +1399,19 @@ braces@~3.0.2:
|
|||
dependencies:
|
||||
fill-range "^7.0.1"
|
||||
|
||||
bree@3.4.0:
|
||||
version "3.4.0"
|
||||
resolved "https://registry.yarnpkg.com/bree/-/bree-3.4.0.tgz#1af7660df7cba7467a471052408e9211f151bd7f"
|
||||
integrity sha512-jxTj5O36jUNo5QW2vn40559Atp5mZhZIPVMPv/B/lRg0UxgLGF/VtOwY7D2levyLon6ZY469AmJEfx8/sjOklg==
|
||||
bree@4.0.0:
|
||||
version "4.0.0"
|
||||
resolved "https://registry.yarnpkg.com/bree/-/bree-4.0.0.tgz#1ce6b74cbbfd49fb87c171001f37ddcec1cea996"
|
||||
integrity sha512-z9vM8rc4KNEzn8j+XOcJDt65ah2G/zuXkEeR2ovzX9A3kaoOL/jo41YLwdTZQ542YBOHcHomzd5pc6CkHAADgQ==
|
||||
dependencies:
|
||||
"@babel/runtime" "^7.11.2"
|
||||
"@babel/runtime" "^7.12.5"
|
||||
"@breejs/later" "^4.0.2"
|
||||
boolean "^3.0.1"
|
||||
boolean "^3.0.2"
|
||||
bthreads "^0.5.1"
|
||||
combine-errors "^3.0.3"
|
||||
cron-validate "^1.3.0"
|
||||
debug "^4.1.1"
|
||||
human-interval "^1.0.0"
|
||||
cron-validate "^1.4.1"
|
||||
debug "^4.3.1"
|
||||
human-interval "^2.0.0"
|
||||
is-string-and-not-blank "^0.0.2"
|
||||
is-valid-path "^0.1.1"
|
||||
ms "^2.1.2"
|
||||
|
|
@ -1458,7 +1458,7 @@ bson-objectid@1.3.1:
|
|||
resolved "https://registry.yarnpkg.com/bson-objectid/-/bson-objectid-1.3.1.tgz#11e4ce4c3419161fd388113781bb62c1dfbce34b"
|
||||
integrity sha512-eQBNQXsisEAXlwiSy8zRNZdW2xDBJaEVkTPbodYR9hGxxtE548Qq7ilYOd8WAQ86xF7NRUdiWSQ1pa/TkKiE2A==
|
||||
|
||||
bthreads@^0.5.1:
|
||||
bthreads@0.5.1, bthreads@^0.5.1:
|
||||
version "0.5.1"
|
||||
resolved "https://registry.yarnpkg.com/bthreads/-/bthreads-0.5.1.tgz#c7a4dacc2d159c50de08b37b1e2a7da836171063"
|
||||
integrity sha512-nK7Jo9ll+r1FRMNPWEFRTZMQrX6HhX8JjPAofxmbTNILHqWVIJPmWzCi9JlX/K0DL5AKZTFZg2Qser5C6gVs9A==
|
||||
|
|
@ -2238,12 +2238,12 @@ create-error@~0.3.1:
|
|||
resolved "https://registry.yarnpkg.com/create-error/-/create-error-0.3.1.tgz#69810245a629e654432bf04377360003a5351a23"
|
||||
integrity sha1-aYECRaYp5lRDK/BDdzYAA6U1GiM=
|
||||
|
||||
cron-validate@1.4.0, cron-validate@^1.3.0:
|
||||
version "1.4.0"
|
||||
resolved "https://registry.yarnpkg.com/cron-validate/-/cron-validate-1.4.0.tgz#aaf5c789db5226ad82b8168ec7654023984c27b3"
|
||||
integrity sha512-z4Mik62Wdvgn1WC9OCGdwxTYO6TCjasTw3SAYsVq0tfArYh6Zg7bbgGVVNBiNAcnnxXsdnBj4AatkYunntaT9A==
|
||||
cron-validate@1.4.1, cron-validate@^1.4.1:
|
||||
version "1.4.1"
|
||||
resolved "https://registry.yarnpkg.com/cron-validate/-/cron-validate-1.4.1.tgz#d637f677a2f5fa37ed61d265a17af502b02b12d6"
|
||||
integrity sha512-83VZ6/I52ybU8lwvrAgjnwR70/1pl867pgE4LbzO+ha+rZOmNMRqMm8vVG9s7a2fpnBQR6/+F7oRiGVfnC+9Ug==
|
||||
dependencies:
|
||||
yup "0.29.3"
|
||||
yup "0.30.0"
|
||||
|
||||
cross-spawn@^5.0.1:
|
||||
version "5.1.0"
|
||||
|
|
@ -2511,6 +2511,13 @@ debug@4.2.0:
|
|||
dependencies:
|
||||
ms "2.1.2"
|
||||
|
||||
debug@^4.3.1:
|
||||
version "4.3.1"
|
||||
resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee"
|
||||
integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==
|
||||
dependencies:
|
||||
ms "2.1.2"
|
||||
|
||||
decamelize@^1.1.1, decamelize@^1.2.0:
|
||||
version "1.2.0"
|
||||
resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
|
||||
|
|
@ -3664,11 +3671,6 @@ flatted@^2.0.0:
|
|||
resolved "https://registry.yarnpkg.com/flatted/-/flatted-2.0.2.tgz#4575b21e2bcee7434aa9be662f4b7b5f9c2b5138"
|
||||
integrity sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==
|
||||
|
||||
fn-name@~3.0.0:
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/fn-name/-/fn-name-3.0.0.tgz#0596707f635929634d791f452309ab41558e3c5c"
|
||||
integrity sha512-eNMNr5exLoavuAMhIUVsOKF79SWd/zG104ef6sxBTSw+cZc6BXdQXDvYcGvp0VbxVVSp1XDUNoz7mg1xMtSznA==
|
||||
|
||||
follow-redirects@0.0.3:
|
||||
version "0.0.3"
|
||||
resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-0.0.3.tgz#6ce67a24db1fe13f226c1171a72a7ef2b17b8f65"
|
||||
|
|
@ -4584,10 +4586,12 @@ https-proxy-agent@^5.0.0:
|
|||
agent-base "6"
|
||||
debug "4"
|
||||
|
||||
human-interval@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/human-interval/-/human-interval-1.0.0.tgz#7ba00a15f3d94ab6a4c16f76060e4aa07c713019"
|
||||
integrity sha512-SWPw3rD6/ocA0JnGePoXp5Zf5eILzsoL5vdWdLwtTuyrElyCpfQb0whIcxMdK/gAKNl2rFDGkPAbwI2KGZCvNA==
|
||||
human-interval@^2.0.0:
|
||||
version "2.0.0"
|
||||
resolved "https://registry.yarnpkg.com/human-interval/-/human-interval-2.0.0.tgz#a9f4cedf4544005398c39767a484fdf062f4287a"
|
||||
integrity sha512-r9qKIElAPkR4+uKSY2T7ImenvKItsLhT2Rm3OmFbA0pBt44YC8mWBpSFsnAbHnVs2D47hm0GquhpCjN+yjR24g==
|
||||
dependencies:
|
||||
numbered "^1.1.0"
|
||||
|
||||
iconv-lite@0.4.24, iconv-lite@^0.4.24, iconv-lite@^0.4.4, iconv-lite@~0.4.13:
|
||||
version "0.4.24"
|
||||
|
|
@ -6812,6 +6816,11 @@ number-is-nan@^1.0.0:
|
|||
resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d"
|
||||
integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=
|
||||
|
||||
numbered@^1.1.0:
|
||||
version "1.1.0"
|
||||
resolved "https://registry.yarnpkg.com/numbered/-/numbered-1.1.0.tgz#9fcd79564c73a84b9574e8370c3d8e58fe3c133c"
|
||||
integrity sha512-pv/ue2Odr7IfYOO0byC1KgBI10wo5YDauLhxY6/saNzAdAs0r1SotGCPzzCLNPL0xtrAwWRialLu23AAu9xO1g==
|
||||
|
||||
nwsapi@^2.2.0:
|
||||
version "2.2.0"
|
||||
resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.0.tgz#204879a9e3d068ff2a55139c2c772780681a38b7"
|
||||
|
|
@ -7681,7 +7690,7 @@ propagate@^2.0.0:
|
|||
resolved "https://registry.yarnpkg.com/propagate/-/propagate-2.0.1.tgz#40cdedab18085c792334e64f0ac17256d38f9a45"
|
||||
integrity sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==
|
||||
|
||||
property-expr@^2.0.2:
|
||||
property-expr@^2.0.4:
|
||||
version "2.0.4"
|
||||
resolved "https://registry.yarnpkg.com/property-expr/-/property-expr-2.0.4.tgz#37b925478e58965031bb612ec5b3260f8241e910"
|
||||
integrity sha512-sFPkHQjVKheDNnPvotjQmm3KD3uk1fWKUN7CrpdbwmUx3CrG3QiM8QpTSimvig5vTXmTvjz7+TDvXOI9+4rkcg==
|
||||
|
|
@ -8988,11 +8997,6 @@ symbol-tree@^3.2.4:
|
|||
resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2"
|
||||
integrity sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==
|
||||
|
||||
synchronous-promise@^2.0.13:
|
||||
version "2.0.15"
|
||||
resolved "https://registry.yarnpkg.com/synchronous-promise/-/synchronous-promise-2.0.15.tgz#07ca1822b9de0001f5ff73595f3d08c4f720eb8e"
|
||||
integrity sha512-k8uzYIkIVwmT+TcglpdN50pS2y1BDcUnBPK9iJeGu0Pl1lOI8pD6wtzgw91Pjpe+RxtTncw32tLxs/R0yNL2Mg==
|
||||
|
||||
sywac@1.3.0:
|
||||
version "1.3.0"
|
||||
resolved "https://registry.yarnpkg.com/sywac/-/sywac-1.3.0.tgz#324789bdb8bd7d0d66625c9144fce81ab5ba6f99"
|
||||
|
|
@ -9966,17 +9970,15 @@ yauzl@^2.10.0:
|
|||
buffer-crc32 "~0.2.3"
|
||||
fd-slicer "~1.1.0"
|
||||
|
||||
yup@0.29.3:
|
||||
version "0.29.3"
|
||||
resolved "https://registry.yarnpkg.com/yup/-/yup-0.29.3.tgz#69a30fd3f1c19f5d9e31b1cf1c2b851ce8045fea"
|
||||
integrity sha512-RNUGiZ/sQ37CkhzKFoedkeMfJM0vNQyaz+wRZJzxdKE7VfDeVKH8bb4rr7XhRLbHJz5hSjoDNwMEIaKhuMZ8gQ==
|
||||
yup@0.30.0:
|
||||
version "0.30.0"
|
||||
resolved "https://registry.yarnpkg.com/yup/-/yup-0.30.0.tgz#427ee076abb1d7e01e747fb782801f7df252fb76"
|
||||
integrity sha512-GX3vqpC9E+Ow0fmQPgqbEg7UV40XRrN1IOEgKF5v04v6T4ha2vBas/hu0thWgewk8L4wUEBLRO/EnXwYyP+p+A==
|
||||
dependencies:
|
||||
"@babel/runtime" "^7.10.5"
|
||||
fn-name "~3.0.0"
|
||||
lodash "^4.17.15"
|
||||
lodash "^4.17.20"
|
||||
lodash-es "^4.17.11"
|
||||
property-expr "^2.0.2"
|
||||
synchronous-promise "^2.0.13"
|
||||
property-expr "^2.0.4"
|
||||
toposort "^2.0.2"
|
||||
|
||||
zip-stream@^1.1.0:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue