From ae54352a29583c5b693970a9d8731416e5409999 Mon Sep 17 00:00:00 2001 From: Simon Backx Date: Thu, 31 Mar 2022 16:01:11 +0200 Subject: [PATCH] Created new stats service and stats API to get member count history (#14391) refs TryGhost/Team#1458 refs TryGhost/Team#1459 refs TryGhost/Team#1372 - Added a new stats service, which is divided into several categories. Currently only the 'members' category for member related stats. - When there are missing or corrupt members status events in the DB, the totals returned by the old member stats endpoint (`/members/stats/count`) were wrong. This is fixed in the new service by counting in reverse order and starting with the actual totals. - New Stats API, with the new `/stats/members/count-history` endpoint. - This new endpoint also returns the paid deltas -> dashboard 5.0 will show subscribed and canceled paid members for each day - Includes tests for the new stats service and endpoint --- core/server/api/canary/index.js | 4 + core/server/api/canary/stats.js | 14 ++ core/server/services/stats/index.js | 1 + .../stats/lib/members-stats-service.js | 163 +++++++++++++++ core/server/services/stats/service.js | 6 + core/server/web/api/canary/admin/routes.js | 3 + .../admin/__snapshots__/stats.test.js.snap | 38 ++++ test/e2e-api/admin/stats.test.js | 26 +++ .../stats/members-stats-service.test.js | 192 ++++++++++++++++++ 9 files changed, 447 insertions(+) create mode 100644 core/server/api/canary/stats.js create mode 100644 core/server/services/stats/index.js create mode 100644 core/server/services/stats/lib/members-stats-service.js create mode 100644 core/server/services/stats/service.js create mode 100644 test/e2e-api/admin/__snapshots__/stats.test.js.snap create mode 100644 test/e2e-api/admin/stats.test.js create mode 100644 test/unit/server/services/stats/members-stats-service.test.js diff --git a/core/server/api/canary/index.js b/core/server/api/canary/index.js index 84e76f34df..90bb200838 100644 --- a/core/server/api/canary/index.js +++ b/core/server/api/canary/index.js @@ -169,6 +169,10 @@ module.exports = { return shared.pipeline(require('./snippets'), localUtils); }, + get stats() { + return shared.pipeline(require('./stats'), localUtils); + }, + get customThemeSettings() { return shared.pipeline(require('./custom-theme-settings'), localUtils); }, diff --git a/core/server/api/canary/stats.js b/core/server/api/canary/stats.js new file mode 100644 index 0000000000..6ae1538643 --- /dev/null +++ b/core/server/api/canary/stats.js @@ -0,0 +1,14 @@ +const statsService = require('../../services/stats'); + +module.exports = { + docName: 'stats', + memberCountHistory: { + permissions: { + docName: 'members', + method: 'browse' + }, + async query() { + return await statsService.members.getCountHistory(); + } + } +}; diff --git a/core/server/services/stats/index.js b/core/server/services/stats/index.js new file mode 100644 index 0000000000..102ef66d4f --- /dev/null +++ b/core/server/services/stats/index.js @@ -0,0 +1 @@ +module.exports = require('./service'); diff --git a/core/server/services/stats/lib/members-stats-service.js b/core/server/services/stats/lib/members-stats-service.js new file mode 100644 index 0000000000..653284f9fc --- /dev/null +++ b/core/server/services/stats/lib/members-stats-service.js @@ -0,0 +1,163 @@ +const {DateTime} = require('luxon'); + +class MembersStatsService { + constructor({db}) { + this.db = db; + } + + /** + * Get the current total members grouped by status + * @returns {Promise} + */ + async getCount() { + const knex = this.db.knex; + const rows = await knex('members') + .select('status') + .select(knex.raw('COUNT(id) AS total')) + .groupBy('status'); + + const paidEvent = rows.find(c => c.status === 'paid'); + const freeEvent = rows.find(c => c.status === 'free'); + const compedEvent = rows.find(c => c.status === 'comped'); + + return { + paid: paidEvent ? paidEvent.total : 0, + free: freeEvent ? freeEvent.total : 0, + comped: compedEvent ? compedEvent.total : 0 + }; + } + + /** + * Get the member deltas by status for all days (from new to old) + * @returns {Promise} The deltas of paid, free and comped users per day, sorted from new to old + */ + async fetchAllStatusDeltas() { + const knex = this.db.knex; + const rows = await knex('members_status_events') + .select(knex.raw('DATE(created_at) as date')) + .select(knex.raw(`SUM( + CASE WHEN to_status='paid' THEN 1 + ELSE 0 END + ) as paid_subscribed`)) + .select(knex.raw(`SUM( + CASE WHEN from_status='paid' THEN 1 + ELSE 0 END + ) as paid_canceled`)) + .select(knex.raw(`SUM( + CASE WHEN to_status='comped' THEN 1 + WHEN from_status='comped' THEN -1 + ELSE 0 END + ) as comped_delta`)) + .select(knex.raw(`SUM( + CASE WHEN to_status='free' THEN 1 + WHEN from_status='free' THEN -1 + ELSE 0 END + ) as free_delta`)) + .groupByRaw('DATE(created_at)') + .orderByRaw('DATE(created_at) DESC'); + return rows; + } + + /** + * Returns a list of the total members by status for each day, including the paid deltas paid_subscribed and paid_canceled + * @returns {Promise} + */ + async getCountHistory() { + const rows = await this.fetchAllStatusDeltas(); + + // Fetch current total amounts and start counting from there + let {paid, free, comped} = await this.getCount(); + + // Get today in UTC (default timezone for Luxon) + const today = DateTime.local().toISODate(); + + const cumulativeResults = []; + for (const row of rows) { + // Convert JSDates to YYYY-MM-DD (in UTC) + const date = DateTime.fromJSDate(row.date).toISODate(); + if (date > today) { + // Skip results that are in the future (fix for invalid events) + continue; + } + cumulativeResults.unshift({ + date, + paid, + free, + comped, + + // Deltas + paid_subscribed: row.paid_subscribed, + paid_canceled: row.paid_canceled + }); + + // Update current counts + paid = Math.max(0, paid - row.paid_subscribed + row.paid_canceled); + free = Math.max(0, free - row.free_delta); + comped = Math.max(0, comped - row.comped_delta); + } + + // Always make sure we have at least one result + if (cumulativeResults.length === 0) { + cumulativeResults.push({ + date: today, + paid, + free, + comped, + + // Deltas + paid_subscribed: 0, + paid_canceled: 0 + }); + } + + return { + data: cumulativeResults, + meta: { + pagination: { + page: 1, + limit: 'all', + pages: 1, + total: cumulativeResults.length, + next: null, + prev: null + } + } + }; + } +} + +module.exports = MembersStatsService; + +/** + * @typedef MemberStatusDelta + * @type {Object} + * @property {Date} date + * @property {number} paid_subscribed Paid members that subscribed on this day + * @property {number} paid_canceled Paid members that canceled on this day + * @property {number} comped_delta Total net comped members on this day + * @property {number} free_delta Total net members on this day + */ + +/** + * @typedef TotalMembersByStatus + * @type {Object} + * @property {number} paid Total paid members + * @property {number} free Total free members + * @property {number} comped Total comped members + */ + +/** + * @typedef {Object} TotalMembersByStatusItem + * @property {string} date In YYYY-MM-DD format + * @property {number} paid Total paid members + * @property {number} free Total free members + * @property {number} comped Total comped members + * @property {number} paid_subscribed Paid members that subscribed on this day + * @property {number} paid_canceled Paid members that canceled on this day + */ + +/** + * @typedef {Object} CountHistory + * @property {TotalMembersByStatusItem[]} data List of the total members by status for each day, including the paid deltas paid_subscribed and paid_canceled + * @property {Object} meta + */ diff --git a/core/server/services/stats/service.js b/core/server/services/stats/service.js new file mode 100644 index 0000000000..280bb775e6 --- /dev/null +++ b/core/server/services/stats/service.js @@ -0,0 +1,6 @@ +const db = require('../../data/db'); +const MemberStatsService = require('./lib/members-stats-service'); + +module.exports = { + members: new MemberStatsService({db}) +}; diff --git a/core/server/web/api/canary/admin/routes.js b/core/server/web/api/canary/admin/routes.js index 162a3f2632..48c550d44f 100644 --- a/core/server/web/api/canary/admin/routes.js +++ b/core/server/web/api/canary/admin/routes.js @@ -137,6 +137,9 @@ module.exports = function apiRoutes() { router.get('/members/:id/signin_urls', mw.authAdminApi, http(api.memberSigninUrls.read)); + // ## Stats + router.get('/stats/members/count-history', mw.authAdminApi, http(api.stats.memberCountHistory)); + // ## Labels router.get('/labels', mw.authAdminApi, http(api.labels.browse)); router.get('/labels/:id', mw.authAdminApi, http(api.labels.read)); diff --git a/test/e2e-api/admin/__snapshots__/stats.test.js.snap b/test/e2e-api/admin/__snapshots__/stats.test.js.snap new file mode 100644 index 0000000000..f8edc8c8fb --- /dev/null +++ b/test/e2e-api/admin/__snapshots__/stats.test.js.snap @@ -0,0 +1,38 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Stats API Can fetch member count history 1: [body] 1`] = ` +Object { + "meta": Object { + "pagination": Object { + "limit": "all", + "next": null, + "page": 1, + "pages": 1, + "prev": null, + "total": 1, + }, + }, + "stats": Array [ + Object { + "comped": 0, + "date": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}/, + "free": 3, + "paid": 5, + "paid_canceled": 0, + "paid_subscribed": 0, + }, + ], +} +`; + +exports[`Stats API Can fetch member count history 2: [headers] 1`] = ` +Object { + "access-control-allow-origin": "http://127.0.0.1:2369", + "cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0", + "content-length": "191", + "content-type": "application/json; charset=utf-8", + "etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/, + "vary": "Origin, Accept-Encoding", + "x-powered-by": "Express", +} +`; diff --git a/test/e2e-api/admin/stats.test.js b/test/e2e-api/admin/stats.test.js new file mode 100644 index 0000000000..0c15d4263a --- /dev/null +++ b/test/e2e-api/admin/stats.test.js @@ -0,0 +1,26 @@ +const {agentProvider, fixtureManager, matchers} = require('../../utils/e2e-framework'); +const {anyEtag, anyISODate} = matchers; + +let agent; + +describe('Stats API', function () { + before(async function () { + agent = await agentProvider.getAdminAPIAgent(); + await fixtureManager.init('members'); + await agent.loginAsOwner(); + }); + + it('Can fetch member count history', async function () { + await agent + .get(`/stats/members/count-history`) + .expectStatus(200) + .matchBodySnapshot({ + stats: [{ + date: anyISODate + }] + }) + .matchHeaderSnapshot({ + etag: anyEtag + }); + }); +}); diff --git a/test/unit/server/services/stats/members-stats-service.test.js b/test/unit/server/services/stats/members-stats-service.test.js new file mode 100644 index 0000000000..2941e8f168 --- /dev/null +++ b/test/unit/server/services/stats/members-stats-service.test.js @@ -0,0 +1,192 @@ +const MembersStatsService = require('../../../../../core/server/services/stats/lib/members-stats-service'); +const {DateTime} = require('luxon'); +const sinon = require('sinon'); +require('should'); + +describe('MembersStatsService', function () { + describe('getCountHistory', function () { + let membersStatsService; + let fakeStatuses; + let fakeTotal; + + /** + * @type {MembersStatsService.TotalMembersByStatus} + */ + const currentCounts = {paid: 0, free: 0, comped: 0}; + /** + * @type {MembersStatsService.MemberStatusDelta[]} + */ + const events = []; + const today = '2000-01-10'; + const tomorrow = '2000-01-11'; + const yesterday = '2000-01-09'; + const todayDate = DateTime.fromISO(today).toJSDate(); + const tomorrowDate = DateTime.fromISO(tomorrow).toJSDate(); + const yesterdayDate = DateTime.fromISO(yesterday).toJSDate(); + + before(function () { + sinon.useFakeTimers(todayDate.getTime()); + membersStatsService = new MembersStatsService({db: null}); + fakeTotal = sinon.stub(membersStatsService, 'getCount').resolves(currentCounts); + fakeStatuses = sinon.stub(membersStatsService, 'fetchAllStatusDeltas').resolves(events); + }); + + afterEach(function () { + fakeStatuses.resetHistory(); + fakeTotal.resetHistory(); + }); + + it('Always returns at least one value', async function () { + // No status events + events.splice(0, events.length); + currentCounts.paid = 1; + currentCounts.free = 2; + currentCounts.comped = 3; + + const {data: results} = await membersStatsService.getCountHistory(); + results.length.should.eql(1); + results[0].should.eql({ + date: today, + paid: 1, + free: 2, + comped: 3, + paid_subscribed: 0, + paid_canceled: 0 + }); + + fakeStatuses.calledOnce.should.eql(true); + fakeTotal.calledOnce.should.eql(true); + }); + + it('Passes paid_subscribers and paid_canceled', async function () { + // Update faked status events + events.splice(0, events.length, { + date: todayDate, + paid_subscribed: 4, + paid_canceled: 3, + free_delta: 2, + comped_delta: 3 + }); + + // Update current faked counts + currentCounts.paid = 1; + currentCounts.free = 2; + currentCounts.comped = 3; + + const {data: results} = await membersStatsService.getCountHistory(); + results.length.should.eql(1); + results[0].should.eql({ + date: today, + paid: 1, + free: 2, + comped: 3, + paid_subscribed: 4, + paid_canceled: 3 + }); + + fakeStatuses.calledOnce.should.eql(true); + fakeTotal.calledOnce.should.eql(true); + }); + + it('Correctly resolves deltas', async function () { + // Update faked status events + events.splice(0, events.length, + { + date: todayDate, + paid_subscribed: 4, + paid_canceled: 3, + free_delta: 2, + comped_delta: 3 + }, + { + date: yesterdayDate, + paid_subscribed: 2, + paid_canceled: 1, + free_delta: 0, + comped_delta: 0 + } + ); + + // Update current faked counts + currentCounts.paid = 2; + currentCounts.free = 3; + currentCounts.comped = 4; + + const {data: results} = await membersStatsService.getCountHistory(); + results.should.eql([ + { + date: yesterday, + paid: 1, + free: 1, + comped: 1, + paid_subscribed: 2, + paid_canceled: 1 + }, + { + date: today, + paid: 2, + free: 3, + comped: 4, + paid_subscribed: 4, + paid_canceled: 3 + } + ]); + fakeStatuses.calledOnce.should.eql(true); + fakeTotal.calledOnce.should.eql(true); + }); + + it('Ignores events in the future', async function () { + // Update faked status events + events.splice(0, events.length, + { + date: tomorrowDate, + paid_subscribed: 10, + paid_canceled: 5, + free_delta: 8, + comped_delta: 9 + }, + { + date: todayDate, + paid_subscribed: 4, + paid_canceled: 3, + free_delta: 2, + comped_delta: 3 + }, + { + date: yesterdayDate, + paid_subscribed: 0, + paid_canceled: 0, + free_delta: 0, + comped_delta: 0 + } + ); + + // Update current faked counts + currentCounts.paid = 1; + currentCounts.free = 2; + currentCounts.comped = 3; + + const {data: results} = await membersStatsService.getCountHistory(); + results.should.eql([ + { + date: yesterday, + paid: 0, + free: 0, + comped: 0, + paid_subscribed: 0, + paid_canceled: 0 + }, + { + date: today, + paid: 1, + free: 2, + comped: 3, + paid_subscribed: 4, + paid_canceled: 3 + } + ]); + fakeStatuses.calledOnce.should.eql(true); + fakeTotal.calledOnce.should.eql(true); + }); + }); +});