Added utilities for creating custom prometheus metrics (#21614)

ref
https://linear.app/ghost/issue/ENG-1771/add-utility-functions-to-easily-create-custom-metrics

- Currently adding custom metrics to our prometheus client requires you
to directly access the `prometheusClient.client` to create the metrics
- This isn't super convenient, as you then have to either keep the
metric in a local variable, or manually get it from the
`prometheusClient.client.register`
- This commit exposes some utility functions for registering metrics on
the `prometheusClient` class, and for retrieving metrics that have
already been registered

ghost/prometheus-metrics/src/PrometheusClient.ts

@@ -1,5 +1,6 @@
 import {Request, Response} from 'express';
 import client from 'prom-client';
+import type {Metric, MetricObjectWithValues, MetricValue} from 'prom-client';
 import type {Knex} from 'knex';
 import logging from '@tryghost/logging';
 
@@ -112,12 +113,25 @@ export class PrometheusClient {
     }
 
     /**
-     * Returns the metrics from the registry
+     * Returns the metrics from the registry as a string
      */
-    async getMetrics() {
+    async getMetrics(): Promise<string> {
         return this.client.register.metrics();
     }
 
+    /**
+     * Returns the metrics from the registry as a JSON object
+     * 
+     * Particularly useful for testing
+     */
+    async getMetricsAsJSON(): Promise<object[]> {
+        return this.client.register.getMetricsAsJSON();
+    }
+
+    async getMetricsAsArray(): Promise<object[]> {
+        return this.client.register.getMetricsAsArray();
+    }
+
     /**
      * Returns the content type for the metrics
      */
@@ -125,6 +139,104 @@ export class PrometheusClient {
         return this.client.register.contentType;
     }
 
+    /**
+     * Returns a single metric from the registry
+     * @param name - The name of the metric
+     * @returns The metric
+     */
+    getMetric(name: string): Metric | undefined {
+        if (!name.startsWith(this.prefix)) {
+            name = `${this.prefix}${name}`;
+        }
+        return this.client.register.getSingleMetric(name);
+    }
+
+    /**
+     * Returns the metric object of a single metric, if it exists
+     * @param name - The name of the metric
+     * @returns The values of the metric
+     */
+    async getMetricObject(name: string): Promise<MetricObjectWithValues<MetricValue<string>> | undefined> {
+        const metric = this.getMetric(name);
+        if (!metric) {
+            return undefined;
+        }
+        return await metric.get();
+    }
+
+    async getMetricValues(name: string): Promise<MetricValue<string>[] | undefined> {
+        const metricObject = await this.getMetricObject(name);
+        if (!metricObject) {
+            return undefined;
+        }
+        return metricObject.values;
+    }
+
+    /**
+     * 
+     */
+
+    /**
+     * Registers a counter metric
+     * @param name - The name of the metric
+     * @param help - The help text for the metric
+     * @returns The counter metric
+     */
+    registerCounter({name, help}: {name: string, help: string}): client.Counter {
+        return new this.client.Counter({
+            name: `${this.prefix}${name}`,
+            help
+        });
+    }
+
+    /**
+     * Registers a gauge metric
+     * @param name - The name of the metric
+     * @param help - The help text for the metric
+     * @param collect - The collect function to use for the gauge
+     * @returns The gauge metric
+     */
+    registerGauge({name, help, collect}: {name: string, help: string, collect?: () => void}): client.Gauge {
+        return new this.client.Gauge({
+            name: `${this.prefix}${name}`,
+            help,
+            collect
+        });
+    }
+
+    /**
+     * Registers a summary metric
+     * @param name - The name of the metric
+     * @param help - The help text for the metric
+     * @param percentiles - The percentiles to calculate for the summary
+     * @param collect - The collect function to use for the summary
+     * @returns The summary metric
+     */
+    registerSummary({name, help, percentiles, collect}: {name: string, help: string, percentiles?: number[], collect?: () => void}): client.Summary {
+        return new this.client.Summary({
+            name: `${this.prefix}${name}`,
+            help,
+            percentiles: percentiles || [0.5, 0.9, 0.99],
+            collect
+        });
+    }
+
+    /**
+     * Registers a histogram metric
+     * @param name - The name of the metric
+     * @param help - The help text for the metric
+     * @param buckets - The buckets to calculate for the histogram
+     * @param collect - The collect function to use for the histogram
+     * @returns The histogram metric
+     */
+    registerHistogram({name, help, buckets}: {name: string, help: string, buckets: number[], collect?: () => void}): client.Histogram {
+        return new this.client.Histogram({
+            name: `${this.prefix}${name}`,
+            help,
+            buckets: buckets
+        });
+    }
+
     // Utility functions for creating custom metrics
 
     /**

ghost/prometheus-metrics/test/prometheus-client.test.ts

@@ -6,7 +6,7 @@ import type {Knex} from 'knex';
 import nock from 'nock';
 import {EventEmitter} from 'events';
 import type {EventEmitter as EventEmitterType} from 'events';
-import type {Gauge, Counter, Summary, Pushgateway, RegistryContentType} from 'prom-client';
+import type {Gauge, Counter, Summary, Pushgateway, RegistryContentType, Metric} from 'prom-client';
 
 describe('Prometheus Client', function () {
     let instance: PrometheusClient;
@@ -155,11 +155,94 @@ describe('Prometheus Client', function () {
     });
 
     describe('getMetrics', function () {
-        it('should return metrics', async function () {
+        it('should return metrics as a string', async function () {
             instance = new PrometheusClient();
             instance.init();
             const metrics = await instance.getMetrics();
-            assert.match(metrics, /^# HELP/);
+            assert.equal(typeof metrics, 'string');
+            assert.match(metrics as string, /^# HELP/);
+        });
+    });
+
+    describe('getMetricsAsJSON', function () {
+        it('should return metrics as an array of objects', async function () {
+            instance = new PrometheusClient();
+            instance.init();
+            const metrics = await instance.getMetricsAsJSON();
+            assert.equal(typeof metrics, 'object');
+            assert.ok(Array.isArray(metrics));
+            assert.ok(Object.keys(metrics[0]).includes('name'));
+        });
+    });
+
+    describe('getMetricsAsArray', function () {
+        it('should return metrics as an array', async function () {
+            instance = new PrometheusClient();
+            instance.init();
+            const metricsArray = await instance.getMetricsAsArray();
+            assert.ok(Array.isArray(metricsArray));
+            assert.ok((metricsArray[0] as Metric).get());
+        });
+    });
+
+    describe('getMetric', function () {
+        it('should return a metric from the registry by name', async function () {
+            instance = new PrometheusClient();
+            instance.init();
+            const metric = instance.getMetric('ghost_process_cpu_seconds_total');
+            assert.ok(metric);
+        });
+
+        it('should return undefined if the metric is not found', function () {
+            instance = new PrometheusClient();
+            instance.init();
+            const metric = instance.getMetric('ghost_not_a_metric');
+            assert.equal(metric, undefined);
+        });
+
+        it('should add the prefix to the metric name if it is not already present', function () {
+            instance = new PrometheusClient();
+            instance.init();
+            const metric = instance.getMetric('process_cpu_seconds_total');
+            assert.ok(metric);
+        });
+    });
+
+    describe('getMetricObject', function () {
+        it('should return the values of a metric', async function () {
+            instance = new PrometheusClient();
+            instance.init();
+            const metricObject = await instance.getMetricObject('ghost_process_cpu_seconds_total');
+            assert.ok(metricObject);
+            assert.ok(metricObject.values);
+            assert.ok(Array.isArray(metricObject.values));
+            assert.equal(metricObject.help, 'Total user and system CPU time spent in seconds.');
+            assert.equal(metricObject.type, 'counter');
+            assert.equal(metricObject.name, 'ghost_process_cpu_seconds_total');
+        });
+
+        it('should return undefined if the metric is not found', async function () {
+            instance = new PrometheusClient();
+            instance.init();
+            const metricObject = await instance.getMetricObject('ghost_not_a_metric');
+            assert.equal(metricObject, undefined);
+        });
+    });
+
+    describe('getMetricValues', function () {
+        it('should return the values of a metric', async function () {
+            instance = new PrometheusClient();
+            instance.init();
+            const metricValues = await instance.getMetricValues('ghost_process_cpu_seconds_total');
+            assert.ok(metricValues);
+            assert.ok(Array.isArray(metricValues));
+        });
+
+        it('should return undefined if the metric is not found', async function () {
+            instance = new PrometheusClient();
+            instance.init();
+            const metricValues = await instance.getMetricValues('ghost_not_a_metric');
+            assert.equal(metricValues, undefined);
         });
     });
 
@@ -339,4 +422,349 @@ describe('Prometheus Client', function () {
             ]);
         });
     });
+
+    describe('Custom Metrics', function () {
+        describe('registerCounter', function () {
+            it('should add the counter metric to the registry', function () {
+                instance = new PrometheusClient();
+                instance.init();
+                instance.registerCounter({name: 'test_counter', help: 'A test counter'});
+                const metric = instance.getMetric('ghost_test_counter');
+                assert.ok(metric);
+            });
+
+            it('should return the counter metric', function () {
+                instance = new PrometheusClient();
+                instance.init();
+                const counter = instance.registerCounter({name: 'test_counter', help: 'A test counter'});
+                const metric = instance.getMetric('ghost_test_counter');
+                assert.equal(metric, counter);
+            });
+
+            it('should increment the counter', async function () {
+                instance = new PrometheusClient();
+                instance.init();
+                const counter = instance.registerCounter({name: 'test_counter', help: 'A test counter'});
+                const metricValuesBefore = await instance.getMetricValues('ghost_test_counter');
+                assert.deepEqual(metricValuesBefore, [{value: 0, labels: {}}]);
+                counter.inc();
+                const metricValuesAfter = await instance.getMetricValues('ghost_test_counter');
+                assert.deepEqual(metricValuesAfter, [{value: 1, labels: {}}]);
+            });
+        });
+
+        describe('registerGauge', function () {
+            it('should add the gauge metric to the registry', function () {
+                instance = new PrometheusClient();
+                instance.init();
+                instance.registerGauge({name: 'test_gauge', help: 'A test gauge'});
+                const metric = instance.getMetric('ghost_test_gauge');
+                assert.ok(metric);
+            });
+
+            it('should return the gauge metric', function () {
+                instance = new PrometheusClient();
+                instance.init();
+                const gauge = instance.registerGauge({name: 'test_gauge', help: 'A test gauge'});
+                const metric = instance.getMetric('ghost_test_gauge');
+                assert.equal(metric, gauge);
+            });
+
+            it('should set the gauge value', async function () {
+                instance = new PrometheusClient();
+                instance.init();
+                const gauge = instance.registerGauge({name: 'test_gauge', help: 'A test gauge'});
+                gauge.set(10);
+                const metricValues = await instance.getMetricValues('ghost_test_gauge');
+                assert.deepEqual(metricValues, [{value: 10, labels: {}}]);
+            });
+
+            it('should increment the gauge', async function () {
+                instance = new PrometheusClient();
+                instance.init();
+                const gauge = instance.registerGauge({name: 'test_gauge', help: 'A test gauge'});
+                const metricValuesBefore = await instance.getMetricValues('ghost_test_gauge');
+                assert.deepEqual(metricValuesBefore, [{value: 0, labels: {}}]);
+                gauge.inc();
+                const metricValuesAfter = await instance.getMetricValues('ghost_test_gauge');
+                assert.deepEqual(metricValuesAfter, [{value: 1, labels: {}}]);
+            });
+
+            it('should decrement the gauge', async function () {
+                instance = new PrometheusClient();
+                instance.init();
+                const gauge = instance.registerGauge({name: 'test_gauge', help: 'A test gauge'});
+                const metricValuesBefore = await instance.getMetricValues('ghost_test_gauge');
+                assert.deepEqual(metricValuesBefore, [{value: 0, labels: {}}]);
+                gauge.dec();
+                const metricValuesAfter = await instance.getMetricValues('ghost_test_gauge');
+                assert.deepEqual(metricValuesAfter, [{value: -1, labels: {}}]);
+            });
+
+            it('should use the collect function to set the gauge value', async function () {
+                instance = new PrometheusClient();
+                instance.init();
+                instance.registerGauge({name: 'test_gauge', help: 'A test gauge', collect() {
+                    (this as unknown as Gauge).set(10); // `this` is the gauge instance
+                }});
+                const metricValues = await instance.getMetricValues('ghost_test_gauge');
+                assert.deepEqual(metricValues, [{value: 10, labels: {}}]);
+            });
+
+            it('should use an async collect function to set the gauge value', async function () {
+                instance = new PrometheusClient();
+                instance.init();
+                instance.registerGauge({name: 'test_gauge', help: 'A test gauge', async collect() {
+                    await new Promise((resolve) => {
+                        setTimeout(resolve, 10);
+                    });
+                    (this as unknown as Gauge).set(20); // `this` is the gauge instance
+                }});
+                const metricValues = await instance.getMetricValues('ghost_test_gauge');
+                assert.deepEqual(metricValues, [{value: 20, labels: {}}]);
+            });
+        });
+
+        describe('registerSummary', function () {
+            it('should add the summary metric to the registry', function () {
+                instance = new PrometheusClient();
+                instance.init();
+                instance.registerSummary({name: 'test_summary', help: 'A test summary'});
+                const metric = instance.getMetric('ghost_test_summary');
+                assert.ok(metric);
+            });
+
+            it('should return the summary metric', function () {
+                instance = new PrometheusClient();
+                instance.init();
+                const summary = instance.registerSummary({name: 'test_summary', help: 'A test summary'});
+                const metric = instance.getMetric('ghost_test_summary');
+                assert.equal(metric, summary);
+            });
+
+            it('can observe a value', async function () {
+                instance = new PrometheusClient();
+                instance.init();
+                const summary = instance.registerSummary({name: 'test_summary', help: 'A test summary'});
+                summary.observe(10);
+                const metricValues = await instance.getMetricValues('ghost_test_summary');
+                assert.deepEqual(metricValues, [
+                    {labels: {quantile: 0.5}, value: 10},
+                    {labels: {quantile: 0.9}, value: 10},
+                    {labels: {quantile: 0.99}, value: 10},
+                    {metricName: 'ghost_test_summary_sum', labels: {}, value: 10},
+                    {metricName: 'ghost_test_summary_count', labels: {}, value: 1}
+                ]);
+            });
+
+            it('can use the collect function to set the summary value', async function () {
+                instance = new PrometheusClient();
+                instance.init();
+                instance.registerSummary({name: 'test_summary', help: 'A test summary', collect() {
+                    (this as unknown as Summary).observe(20);
+                }});
+                const metricValues = await instance.getMetricValues('ghost_test_summary');
+                assert.deepEqual(metricValues, [
+                    {labels: {quantile: 0.5}, value: 20},
+                    {labels: {quantile: 0.9}, value: 20},
+                    {labels: {quantile: 0.99}, value: 20},
+                    {metricName: 'ghost_test_summary_sum', labels: {}, value: 20},
+                    {metricName: 'ghost_test_summary_count', labels: {}, value: 1}
+                ]);
+            });
+
+            it('can use an async collect function to set the summary value', async function () {
+                instance = new PrometheusClient();
+                instance.init();
+                instance.registerSummary({name: 'test_summary', help: 'A test summary', async collect() {
+                    await new Promise((resolve) => {
+                        setTimeout(resolve, 10);
+                    });
+                    (this as unknown as Summary).observe(30);
+                }});
+                const metricValues = await instance.getMetricValues('ghost_test_summary');
+                assert.deepEqual(metricValues, [
+                    {labels: {quantile: 0.5}, value: 30},
+                    {labels: {quantile: 0.9}, value: 30},
+                    {labels: {quantile: 0.99}, value: 30},
+                    {metricName: 'ghost_test_summary_sum', labels: {}, value: 30},
+                    {metricName: 'ghost_test_summary_count', labels: {}, value: 1}
+                ]);
+            });
+
+            it('can use the percentiles option to set the summary value', async function () {
+                instance = new PrometheusClient();
+                instance.init();
+                instance.registerSummary({name: 'test_summary', help: 'A test summary', percentiles: [0.1, 0.5, 0.9]});
+                const metricValues = await instance.getMetricValues('ghost_test_summary');
+                assert.deepEqual(metricValues, [
+                    {labels: {quantile: 0.1}, value: 0},
+                    {labels: {quantile: 0.5}, value: 0},
+                    {labels: {quantile: 0.9}, value: 0},
+                    {metricName: 'ghost_test_summary_sum', labels: {}, value: 0},
+                    {metricName: 'ghost_test_summary_count', labels: {}, value: 0}
+                ]);
+            });
+
+            it('can use a timer to observe the summary value', async function () {
+                instance = new PrometheusClient();
+                instance.init();
+                const summary = instance.registerSummary({name: 'test_summary', help: 'A test summary', percentiles: [0.1, 0.5, 0.9]});
+                const clock = sinon.useFakeTimers();
+                const timer = summary.startTimer();
+                clock.tick(1000);
+                timer();
+                const metricValues = await instance.getMetricValues('ghost_test_summary');
+                assert.deepEqual(metricValues, [
+                    {labels: {quantile: 0.1}, value: 1},
+                    {labels: {quantile: 0.5}, value: 1},
+                    {labels: {quantile: 0.9}, value: 1},
+                    {metricName: 'ghost_test_summary_sum', labels: {}, value: 1},
+                    {metricName: 'ghost_test_summary_count', labels: {}, value: 1}
+                ]);
+
+                clock.restore();
+            });
+        });
+
+        describe('registerHistogram', function () {
+            it('should add the histogram metric to the registry', function () {
+                instance = new PrometheusClient();
+                instance.init();
+                instance.registerHistogram({name: 'test_histogram', help: 'A test histogram', buckets: [1, 2, 3]});
+                const metric = instance.getMetric('ghost_test_histogram');
+                assert.ok(metric);
+            });
+
+            it('should return the histogram metric', function () {
+                instance = new PrometheusClient();
+                instance.init();
+                const histogram = instance.registerHistogram({name: 'test_histogram', help: 'A test histogram', buckets: [1, 2, 3]});
+                const metric = instance.getMetric('ghost_test_histogram');
+                assert.equal(metric, histogram);
+            });
+
+            it('can observe a value', async function () {
+                instance = new PrometheusClient();
+                instance.init();
+                const histogram = instance.registerHistogram({name: 'test_histogram', help: 'A test histogram', buckets: [1, 2, 3]});
+                histogram.observe(1);
+                histogram.observe(2);
+                histogram.observe(3);
+                const metricValues = await instance.getMetricValues('ghost_test_histogram');
+                assert.deepEqual(metricValues, [
+                    {
+                        exemplar: null,
+                        labels: {
+                            le: 1
+                        },
+                        metricName: 'ghost_test_histogram_bucket',
+                        value: 1
+                    },
+                    {
+                        exemplar: null,
+                        labels: {
+                            le: 2
+                        },
+                        metricName: 'ghost_test_histogram_bucket',
+                        value: 2
+                    },
+                    {
+                        exemplar: null,
+                        labels: {
+                            le: 3
+                        },
+                        metricName: 'ghost_test_histogram_bucket',
+                        value: 3
+                    },
+                    {
+                        exemplar: null,
+                        labels: {
+                            le: '+Inf'
+                        },
+                        metricName: 'ghost_test_histogram_bucket',
+                        value: 3
+                    },
+                    {
+                        exemplar: undefined,
+                        labels: {},
+                        metricName: 'ghost_test_histogram_sum',
+                        value: 6
+                    },
+                    {
+                        exemplar: undefined,
+                        labels: {},
+                        metricName: 'ghost_test_histogram_count',
+                        value: 3
+                    }
+                ]);
+            });
+
+            it('can use a timer to observe the histogram value', async function () {
+                instance = new PrometheusClient();
+                instance.init();
+                const histogram = instance.registerHistogram({name: 'test_histogram', help: 'A test histogram', buckets: [1000, 2000, 3000]});
+                const clock = sinon.useFakeTimers();
+                // Observe a value of 1 second
+                const timer1 = histogram.startTimer();
+                clock.tick(1000);
+                timer1();
+
+                // Observe a value of 2 seconds
+                const timer2 = histogram.startTimer();
+                clock.tick(2000);
+                timer2();
+
+                const metricValues = await instance.getMetricValues('ghost_test_histogram');
+                assert.deepEqual(metricValues, [
+                    {
+                        exemplar: null,
+                        labels: {
+                            le: 1000
+                        },
+                        metricName: 'ghost_test_histogram_bucket',
+                        value: 2
+                    },
+                    {
+                        exemplar: null,
+                        labels: {
+                            le: 2000
+                        },
+                        metricName: 'ghost_test_histogram_bucket',
+                        value: 2
+                    },
+                    {
+                        exemplar: null,
+                        labels: {
+                            le: 3000
+                        },
+                        metricName: 'ghost_test_histogram_bucket',
+                        value: 2
+                    },
+                    {
+                        exemplar: null,
+                        labels: {
+                            le: '+Inf'
+                        },
+                        metricName: 'ghost_test_histogram_bucket',
+                        value: 2
+                    },
+                    {
+                        exemplar: undefined,
+                        labels: {},
+                        metricName: 'ghost_test_histogram_sum',
+                        value: 3
+                    },
+                    {
+                        exemplar: undefined,
+                        labels: {},
+                        metricName: 'ghost_test_histogram_count',
+                        value: 2
+                    }
+                ]);
+
+                clock.restore();
+            });
+        });
+    });
 });