Infrastructure/ghost
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2025-04-15 03:01:37 -05:00
Code Issues Activity

Added ability to push prometheus metrics to a pushgateway (#21526)

Browse source 
ref
https://linear.app/ghost/issue/ENG-1746/enable-ghost-to-push-metrics-to-a-pushgateway

- We'd like to use prometheus to expose metrics from Ghost, but the
"standard" approach of having prometheus scrape the `/metrics` endpoint
adds some complexity and additional challenges on Pro.
- A suggested simpler alternative is to use a pushgateway, to have Ghost
_push_ metrics to prometheus, rather than have prometheus scrape the
running instances.
- This PR introduces this functionality behind a configuration. 
- It also includes a refactor to the current metrics-server
implementation so all the related code for prometheus is colocated, and
the configuration is a bit more organized. `@tryghost/metrics-server`
has been renamed to `@tryghost/prometheus-metrics`, and it now includes
the metrics server and prometheus-client code itself (including the
pushgateway code)
- To enable the prometheus client alone, `prometheus:enabled` must be
true. This will _not_ enable the metrics server or the pushgateway — it
will essentially collect the metrics, but not do anything with them.
- To enable the metrics server, set `prometheus:metrics_server:enabled`
to true. You can also configure the host and port that the metrics
server should export the `/metrics` endpoint on in the
`prometheus:metrics_server` block.
- To enable the pushgateway, set `prometheus:pushgateway:enabled` to
true. You can also configure the pushgateway's `url`, the `interval` it
should push metrics in (in milliseconds) and the `jobName` in the
`prometheus:pushgateway` block.
This commit is contained in:
Chris Raible 2024-11-05 11:50:39 -08:00 • committed by GitHub
parent b6f1ecc149
commit 190ebcd684
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 264 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.github/scripts/docker-compose.yml vendored
View file

@ -48,5 +48,11 @@ services:
- ./grafana/datasources:/etc/grafana/provisioning/datasources
- ./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/main.yaml
- ./grafana/dashboards:/var/lib/grafana/dashboards
pushgateway:
profiles: [monitoring]
image: prom/pushgateway:v1.6.0
container_name: ghost-pushgateway
ports:
- "9091:9091"
volumes:
mysql-data:

17
ghost/core/core/boot.js
View file

@ -257,7 +257,7 @@ async function initExpressApps({frontend, backend, config}) {
* Initialize prometheus client
*/
function initPrometheusClient({config}) {
if (config.get('metrics_server:enabled')) {
if (config.get('prometheus:enabled')) {
debug('Begin: initPrometheusClient');
const prometheusClient = require('./shared/prometheus-client');
debug('End: initPrometheusClient');
@ -272,11 +272,11 @@ function initPrometheusClient({config}) {
*/
async function initMetricsServer({prometheusClient, ghostServer, config}) {
debug('Begin: initMetricsServer');
if (ghostServer && config.get('metrics_server:enabled')) {
const {MetricsServer} = require('@tryghost/metrics-server');
if (ghostServer && config.get('prometheus:metrics_server:enabled')) {
const {MetricsServer} = require('@tryghost/prometheus-metrics');
const serverConfig = {
host: config.get('metrics_server:host') || '127.0.0.1',
port: config.get('metrics_server:port') || 9416
host: config.get('prometheus:metrics_server:host') || '127.0.0.1',
port: config.get('prometheus:metrics_server:port') || 9416
};
const handler = prometheusClient.handleMetricsRequest.bind(prometheusClient);
const metricsServer = new MetricsServer({serverConfig, handler});
@ -564,6 +564,13 @@ async function bootGhost({backend = true, frontend = true, server = true} = {})
ghostServer = new GhostServer({url: config.getSiteUrl(), env: config.get('env'), serverConfig: config.get('server')});
await ghostServer.start(rootApp);
bootLogger.log('server started');
// Ensure the prometheus client is stopped when the server shuts down
ghostServer.registerCleanupTask(async () => {
if (prometheusClient) {
prometheusClient.stop();
}
});
debug('End: load server + minimal app');
}

3
ghost/core/core/shared/config/env/config.testing.json vendored
View file

@ -11,9 +11,6 @@
"server": {
"port": 2369
},
"metrics_server": {
"port": 9417
},
"logging": {
"level": "error"
},

39
ghost/core/core/shared/prometheus-client.js
View file

@ -1,36 +1,13 @@
class PrometheusClient {
constructor() {
this.client = require('prom-client');
this.prefix = 'ghost_';
this.collectDefaultMetrics();
}
const {PrometheusClient} = require('@tryghost/prometheus-metrics');
const config = require('./config');
collectDefaultMetrics() {
this.client.collectDefaultMetrics({prefix: this.prefix});
}
let prometheusClient;
async handleMetricsRequest(req, res) {
try {
res.set('Content-Type', this.getContentType());
res.end(await this.getMetrics());
} catch (err) {
res.status(500).end(err.message);
}
}
async getMetrics() {
return this.client.register.metrics();
}
getRegistry() {
return this.client.register;
}
getContentType() {
return this.getRegistry().contentType;
}
if (!prometheusClient) {
const pushgatewayConfig = config.get('prometheus:pushgateway');
const prometheusConfig = {pushgateway: pushgatewayConfig};
prometheusClient = new PrometheusClient(prometheusConfig);
prometheusClient.init();
}
// Create a singleton instance and export it as the default export
const prometheusClient = new PrometheusClient();
module.exports = prometheusClient;

3
ghost/core/package.json
View file

@ -127,7 +127,6 @@
"@tryghost/members-stripe-service": "0.0.0",
"@tryghost/mentions-email-report": "0.0.0",
"@tryghost/metrics": "1.0.34",
"@tryghost/metrics-server": "0.0.0",
"@tryghost/milestones": "0.0.0",
"@tryghost/minifier": "0.0.0",
"@tryghost/model-to-domain-event-interceptor": "0.0.0",
@ -144,6 +143,7 @@
"@tryghost/post-revisions": "0.0.0",
"@tryghost/posts-service": "0.0.0",
"@tryghost/pretty-cli": "1.2.44",
"@tryghost/prometheus-metrics": "0.0.0",
"@tryghost/promise": "0.3.12",
"@tryghost/recommendations": "0.0.0",
"@tryghost/request": "1.0.8",
@ -213,7 +213,6 @@
"node-jose": "2.2.0",
"path-match": "1.2.4",
"probe-image-size": "7.2.3",
"prom-client": "15.1.3",
"rss": "1.2.2",
"sanitize-html": "2.13.1",
"semver": "7.6.3",

9
ghost/core/test/e2e-server/metrics-server/metrics-server.test.js
View file

@ -6,7 +6,10 @@ const configUtils = require('../../utils/configUtils');
describe('Metrics Server', function () {
before(function () {
configUtils.set('metrics_server:enabled', true);
configUtils.set('prometheus:enabled', true);
configUtils.set('prometheus:metrics_server:enabled', true);
configUtils.set('prometheus:metrics_server:host', '127.0.0.1');
configUtils.set('prometheus:metrics_server:port', 9417);
});
it('should start up when Ghost boots and stop when Ghost stops', async function () {
@ -28,7 +31,7 @@ describe('Metrics Server', function () {
});
it('should not start if enabled is false', async function () {
configUtils.set('metrics_server:enabled', false);
configUtils.set('prometheus:metrics_server:enabled', false);
await testUtils.startGhost({forceStart: true});
// Requesting the metrics endpoint should throw an error
let error;
@ -45,7 +48,7 @@ describe('Metrics Server', function () {
let metricsResponse;
let metricsText;
before(async function () {
configUtils.set('metrics_server:enabled', true);
configUtils.set('prometheus:metrics_server:enabled', true);
await testUtils.startGhost({forceStart: true});
metricsResponse = await request('http://127.0.0.1:9417').get('/metrics');
metricsText = metricsResponse.text;

49
ghost/core/test/unit/shared/prometheus-client.test.js
View file

@ -1,49 +0,0 @@
const assert = require('node:assert/strict');
const prometheusClient = require('../../../core/shared/prometheus-client');
const sinon = require('sinon');
describe('PrometheusClient', function () {
describe('getMetrics', function () {
it('should return metrics', async function () {
const metrics = await prometheusClient.getMetrics();
assert.match(metrics, /^# HELP/);
});
});
describe('getContentType', function () {
it('should return the content type', function () {
assert.equal(prometheusClient.getContentType(), 'text/plain; version=0.0.4; charset=utf-8');
});
});
describe('getRegistry', function () {
it('should return the registry', function () {
assert.ok(prometheusClient.getRegistry());
});
});
describe('handleMetricsRequest', function () {
it('should return metrics', async function () {
const req = {};
const res = {
set: sinon.stub(),
end: sinon.stub()
};
await prometheusClient.handleMetricsRequest(req, res);
assert.ok(res.set.calledWith('Content-Type', prometheusClient.getContentType()));
assert.ok(res.end.calledOnce);
});
it('should return an error if getting metrics fails', async function () {
sinon.stub(prometheusClient, 'getMetrics').throws(new Error('Failed to get metrics'));
const req = {};
const res = {
set: sinon.stub(),
end: sinon.stub(),
status: sinon.stub().returnsThis()
};
await prometheusClient.handleMetricsRequest(req, res);
assert.ok(res.status.calledWith(500));
});
});
});

1
ghost/metrics-server/src/index.ts
View file

@ -1 +0,0 @@
export * from './MetricsServer';

0
ghost/metrics-server/.eslintrc.js → ghost/prometheus-metrics/.eslintrc.js
View file

0
ghost/metrics-server/README.md → ghost/prometheus-metrics/README.md
View file

7
ghost/metrics-server/package.json → ghost/prometheus-metrics/package.json
View file

@ -1,7 +1,7 @@
{
"name": "@tryghost/metrics-server",
"name": "@tryghost/prometheus-metrics",
"version": "0.0.0",
"repository": "https://github.com/TryGhost/Ghost/tree/main/packages/metrics-server",
"repository": "https://github.com/TryGhost/Ghost/tree/main/packages/prometheus-metrics",
"author": "Ghost Foundation",
"private": true,
"main": "build/index.js",
@ -11,7 +11,7 @@
"build": "yarn build:ts",
"build:ts": "tsc",
"prepare": "tsc",
"test:unit": "NODE_ENV=testing c8 --src src --all --check-coverage --100 --reporter text --reporter cobertura mocha -r ts-node/register './test/**/*.test.ts'",
"test:unit": "NODE_ENV=testing c8 --src src --all --check-coverage --90 --reporter text --reporter cobertura mocha -r ts-node/register './test/**/*.test.ts'",
"test": "yarn test:types && yarn test:unit",
"test:types": "tsc --noEmit",
"lint:code": "eslint src/ --ext .ts --cache",
@ -33,6 +33,7 @@
},
"dependencies": {
"express": "4.21.1",
"prom-client": "^15.1.3",
"stoppable": "1.1.0"
}
}

0
ghost/metrics-server/src/MetricsServer.ts → ghost/prometheus-metrics/src/MetricsServer.ts
View file

110
ghost/prometheus-metrics/src/PrometheusClient.ts Normal file
View file

@ -0,0 +1,110 @@
import {Request, Response} from 'express';
import client from 'prom-client';
type PrometheusClientConfig = {
register?: client.Registry;
pushgateway?: {
enabled: boolean;
url?: string;
interval?: number;
jobName?: string;
}
};
/**
* A client for exporting metrics to Prometheus, based on prom-client
*/
export class PrometheusClient {
/**
* Creates a new PrometheusClient instance
* @param prometheusConfig - The configuration for the PrometheusClient
*/
constructor(prometheusConfig: PrometheusClientConfig = {}) {
this.config = prometheusConfig;
this.client = client;
this.prefix = 'ghost_';
this.register = this.config.register || client.register;
}
public client;
private config: PrometheusClientConfig;
private prefix;
public register: client.Registry; // public for testing
public gateway: client.Pushgateway<client.RegistryContentType> | undefined; // public for testing
private pushInterval: ReturnType<typeof setInterval> | undefined;
/**
* Initializes the prometheus client, setting up the pushgateway if enabled
*/
init() {
this.collectDefaultMetrics();
if (this.config.pushgateway?.enabled) {
const gatewayUrl = this.config.pushgateway.url || 'http://localhost:9091';
const interval = this.config.pushgateway.interval || 5000;
this.gateway = new client.Pushgateway(gatewayUrl);
this.pushInterval = setInterval(() => {
this.pushMetrics();
}, interval);
}
}
/**
* Pushes metrics to the pushgateway, if enabled
*/
async pushMetrics() {
if (this.config.pushgateway?.enabled && this.gateway) {
const jobName = this.config.pushgateway?.jobName || 'ghost';
await this.gateway.pushAdd({jobName});
}
}
/**
* Shuts down the prometheus client cleanly
*/
stop() {
// Clear the push interval
if (this.pushInterval) {
clearInterval(this.pushInterval);
}
}
/**
* Tells prom-client to collect default metrics
* Only called once on init
*/
collectDefaultMetrics() {
this.client.collectDefaultMetrics({prefix: this.prefix, register: this.register});
}
/**
* Handles metrics requests to serve the /metrics endpoint
* @param req - The request object
* @param res - The response object
*/
async handleMetricsRequest(req: Request, res: Response) {
try {
res.set('Content-Type', this.getContentType());
res.end(await this.getMetrics());
} catch (err) {
if (err instanceof Error && err.message) {
res.status(500).end(err.message);
} else {
res.status(500).end('Unknown error');
}
}
}
/**
* Returns the metrics from the registry
*/
async getMetrics() {
return this.register.metrics();
}
/**
* Returns the content type for the metrics
*/
getContentType() {
return this.register.contentType;
}
}

2
ghost/prometheus-metrics/src/index.ts Normal file
View file

@ -0,0 +1,2 @@
export * from './MetricsServer';
export * from './PrometheusClient';

0
ghost/metrics-server/src/libraries.d.ts → ghost/prometheus-metrics/src/libraries.d.ts vendored
View file

0
ghost/metrics-server/test/.eslintrc.js → ghost/prometheus-metrics/test/.eslintrc.js
View file

0
ghost/metrics-server/test/metrics-server.test.ts → ghost/prometheus-metrics/test/metrics-server.test.ts
View file

114
ghost/prometheus-metrics/test/prometheus-client.test.ts Normal file
View file

@ -0,0 +1,114 @@
import assert from 'assert/strict';
import {PrometheusClient} from '../src';
import {Request, Response} from 'express';
import * as sinon from 'sinon';
describe('Prometheus Client', function () {
let instance: PrometheusClient;
beforeEach(function () {
sinon.restore();
});
afterEach(function () {
if (instance) {
instance.stop();
instance.register.clear();
}
});
describe('constructor', function () {
it('should create a new instance', function () {
instance = new PrometheusClient();
assert.ok(instance);
});
});
describe('init', function () {
it('should call collectDefaultMetrics', function () {
instance = new PrometheusClient();
const collectDefaultMetricsSpy = sinon.spy(instance.client, 'collectDefaultMetrics');
instance.init();
assert.ok(collectDefaultMetricsSpy.called);
});
it('should create the pushgateway client if the pushgateway is enabled', function () {
instance = new PrometheusClient({pushgateway: {enabled: true}});
instance.init();
assert.ok(instance.gateway);
});
it('should push metrics to the pushgateway if it is enabled', async function () {
const clock = sinon.useFakeTimers();
instance = new PrometheusClient({pushgateway: {enabled: true}});
const pushMetricsSpy = sinon.spy(instance, 'pushMetrics');
instance.init();
clock.tick(10000);
assert.ok(pushMetricsSpy.called);
});
});
describe('collectDefaultMetrics', function () {
it('should call collectDefaultMetrics on the client', function () {
instance = new PrometheusClient();
const collectDefaultMetricsSpy = sinon.spy(instance.client, 'collectDefaultMetrics');
instance.collectDefaultMetrics();
assert.ok(collectDefaultMetricsSpy.called);
});
});
describe('handleMetricsRequest', function () {
it('should return the metrics', async function () {
const setStub = sinon.stub();
const endStub = sinon.stub();
const req = {} as Request;
const res = {
set: setStub,
end: endStub
} as unknown as Response;
await instance.handleMetricsRequest(req, res);
assert.ok(setStub.calledWith('Content-Type', instance.getContentType()));
assert.ok(endStub.calledOnce);
});
it('should return an error if getting metrics fails', async function () {
instance = new PrometheusClient();
sinon.stub(instance, 'getMetrics').throws(new Error('Failed to get metrics'));
const statusStub = sinon.stub().returnsThis();
const endStub = sinon.stub();
const req = {} as Request;
const res = {
set: sinon.stub(),
end: endStub,
status: statusStub
} as unknown as Response;
await instance.handleMetricsRequest(req, res);
assert.ok(statusStub.calledWith(500));
assert.ok(endStub.calledOnce);
});
it('should return a generic error if the error is unknown', async function () {
instance = new PrometheusClient();
sinon.stub(instance, 'getMetrics').throws({name: 'UnknownError'});
const statusStub = sinon.stub().returnsThis();
const endStub = sinon.stub();
const req = {} as Request;
const res = {
set: sinon.stub(),
end: endStub,
status: statusStub
} as unknown as Response;
await instance.handleMetricsRequest(req, res);
assert.ok(statusStub.calledWith(500));
assert.ok(endStub.calledOnce);
});
});
describe('getMetrics', function () {
it('should return metrics', async function () {
instance = new PrometheusClient();
instance.init();
const metrics = await instance.getMetrics();
assert.match(metrics, /^# HELP/);
});
});
});

0
ghost/metrics-server/tsconfig.json → ghost/prometheus-metrics/tsconfig.json
View file

2
yarn.lock
View file

@ -26410,7 +26410,7 @@ progress@^2.0.0, progress@^2.0.1:
resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
prom-client@15.1.3:
prom-client@^15.1.3:
version "15.1.3"
resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-15.1.3.tgz#69fa8de93a88bc9783173db5f758dc1c69fa8fc2"
integrity sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==