From a26f63dc112cebee766b2eb5280ed846c04da930 Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Wed, 6 Nov 2024 11:36:37 -0800 Subject: [PATCH] Configured local prometheus and pushgateway in docker-compose (#21538) ref https://linear.app/ghost/issue/ENG-1746/enable-ghost-to-push-metrics-to-a-pushgateway - Added prometheus job to scrape the pushgateway - Updated grafana dashboard to use the metrics from the pushgateway - Added some logging to prometheus client to log errors when pushing metrics to pushgateway --- .../grafana/dashboards/main-dashboard.json | 12 ++++++------ .github/scripts/prometheus/prometheus.yml | 11 +++-------- ghost/prometheus-metrics/package.json | 1 + ghost/prometheus-metrics/src/PrometheusClient.ts | 16 ++++++++++------ ghost/prometheus-metrics/src/libraries.d.ts | 3 ++- .../test/prometheus-client.test.ts | 2 +- yarn.lock | 2 +- 7 files changed, 24 insertions(+), 23 deletions(-) diff --git a/.github/scripts/grafana/dashboards/main-dashboard.json b/.github/scripts/grafana/dashboards/main-dashboard.json index a753bdc207..5ddb81dab5 100644 --- a/.github/scripts/grafana/dashboards/main-dashboard.json +++ b/.github/scripts/grafana/dashboards/main-dashboard.json @@ -312,21 +312,21 @@ "steppedLine": false, "targets": [ { - "expr": "irate(ghost_process_cpu_user_seconds_total{job=~\"$job\", instance=~\"$instance\"}[1m]) * 100", + "expr": "irate(ghost_process_cpu_user_seconds_total{job=~\"$job\"}[1m]) * 100", "interval": "", - "legendFormat": "User CPU - {{instance}}", + "legendFormat": "User CPU - {{job}}", "refId": "A" }, { - "expr": "irate(ghost_process_cpu_system_seconds_total{job=~\"$job\", instance=~\"$instance\"}[1m]) * 100", + "expr": "irate(ghost_process_cpu_system_seconds_total{job=~\"$job\"}[1m]) * 100", "interval": "", - "legendFormat": "System CPU - {{instance}}", + "legendFormat": "System CPU - {{job}}", "refId": "B" }, { - "expr": "irate(ghost_process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[1m]) * 100", + "expr": "irate(ghost_process_cpu_seconds_total{job=~\"$job\"}[1m]) * 100", "interval": "", - "legendFormat": "Total CPU - {{instance}}", + "legendFormat": "Total CPU - {{job}}", "refId": "C" } ], diff --git a/.github/scripts/prometheus/prometheus.yml b/.github/scripts/prometheus/prometheus.yml index f9852993a5..0645eae14f 100644 --- a/.github/scripts/prometheus/prometheus.yml +++ b/.github/scripts/prometheus/prometheus.yml @@ -1,11 +1,6 @@ global: scrape_interval: 15s # By default, scrape targets every 15 seconds. - # Attach these labels to any time series or alerts when communicating with - # external systems (federation, remote storage, Alertmanager). - external_labels: - monitor: 'codelab-monitor' - # A scrape configuration containing exactly one endpoint to scrape: # Here it's Prometheus itself. scrape_configs: @@ -18,14 +13,14 @@ scrape_configs: static_configs: - targets: ['localhost:9090'] - - job_name: 'ghost' + - job_name: 'pushgateway' scrape_interval: 1s static_configs: - - targets: ['host.docker.internal:9416'] + - targets: ['pushgateway:9091'] - metrics_path: '/metrics' + honor_labels: true remote_write: - url: http://grafana:3000/api/prom/push \ No newline at end of file diff --git a/ghost/prometheus-metrics/package.json b/ghost/prometheus-metrics/package.json index fcf907dfd2..7512a325de 100644 --- a/ghost/prometheus-metrics/package.json +++ b/ghost/prometheus-metrics/package.json @@ -32,6 +32,7 @@ "typescript": "5.6.2" }, "dependencies": { + "@tryghost/logging": "2.4.19", "express": "4.21.1", "prom-client": "15.1.3", "stoppable": "1.1.0" diff --git a/ghost/prometheus-metrics/src/PrometheusClient.ts b/ghost/prometheus-metrics/src/PrometheusClient.ts index 0047593ac2..c09889f600 100644 --- a/ghost/prometheus-metrics/src/PrometheusClient.ts +++ b/ghost/prometheus-metrics/src/PrometheusClient.ts @@ -1,5 +1,6 @@ import {Request, Response} from 'express'; import client from 'prom-client'; +import logging from '@tryghost/logging'; type PrometheusClientConfig = { register?: client.Registry; @@ -23,13 +24,11 @@ export class PrometheusClient { this.config = prometheusConfig; this.client = client; this.prefix = 'ghost_'; - this.register = this.config.register || client.register; } public client; private config: PrometheusClientConfig; private prefix; - public register: client.Registry; // public for testing public gateway: client.Pushgateway | undefined; // public for testing private pushInterval: ReturnType | undefined; @@ -54,7 +53,12 @@ export class PrometheusClient { async pushMetrics() { if (this.config.pushgateway?.enabled && this.gateway) { const jobName = this.config.pushgateway?.jobName || 'ghost'; - await this.gateway.pushAdd({jobName}); + try { + await this.gateway.pushAdd({jobName}); + logging.debug('Metrics pushed to pushgateway - jobName: ', jobName); + } catch (error) { + logging.error('Error pushing metrics to pushgateway - jobName: ', jobName); + } } } @@ -73,7 +77,7 @@ export class PrometheusClient { * Only called once on init */ collectDefaultMetrics() { - this.client.collectDefaultMetrics({prefix: this.prefix, register: this.register}); + this.client.collectDefaultMetrics({prefix: this.prefix}); } /** @@ -98,13 +102,13 @@ export class PrometheusClient { * Returns the metrics from the registry */ async getMetrics() { - return this.register.metrics(); + return this.client.register.metrics(); } /** * Returns the content type for the metrics */ getContentType() { - return this.register.contentType; + return this.client.register.contentType; } } diff --git a/ghost/prometheus-metrics/src/libraries.d.ts b/ghost/prometheus-metrics/src/libraries.d.ts index c99c02f0f0..98ac778dce 100644 --- a/ghost/prometheus-metrics/src/libraries.d.ts +++ b/ghost/prometheus-metrics/src/libraries.d.ts @@ -1 +1,2 @@ -declare module '@tryghost/debug'; \ No newline at end of file +declare module '@tryghost/debug'; +declare module '@tryghost/logging'; \ No newline at end of file diff --git a/ghost/prometheus-metrics/test/prometheus-client.test.ts b/ghost/prometheus-metrics/test/prometheus-client.test.ts index 360f41e1e5..65c10abaf0 100644 --- a/ghost/prometheus-metrics/test/prometheus-client.test.ts +++ b/ghost/prometheus-metrics/test/prometheus-client.test.ts @@ -12,7 +12,7 @@ describe('Prometheus Client', function () { afterEach(function () { if (instance) { instance.stop(); - instance.register.clear(); + instance.client.register.clear(); } }); diff --git a/yarn.lock b/yarn.lock index 87674de458..3b4361ed77 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7653,7 +7653,7 @@ lodash "^4.17.21" luxon "^1.26.0" -"@tryghost/logging@2.4.10", "@tryghost/logging@2.4.18", "@tryghost/logging@^2.4.7": +"@tryghost/logging@2.4.10", "@tryghost/logging@2.4.18", "@tryghost/logging@2.4.19", "@tryghost/logging@^2.4.7": version "2.4.18" resolved "https://registry.yarnpkg.com/@tryghost/logging/-/logging-2.4.18.tgz#5d7ddb2d0a66dc6834a6048ebbf48418420445d5" integrity sha512-mMJkdCFDXa0ohS0FlDTvOrJQd7VamBIqjljGYvNECdVli7BMjdUYgZyWr8bEJ/d7scsq8OE2bVVBJWLxvPxLAg==