Infrastructure/ghost
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2025-01-06 22:40:14 -05:00
Code Issues Activity

Created new stats service and stats API to get member count history (#14391)

Browse source 
refs TryGhost/Team#1458
refs TryGhost/Team#1459
refs TryGhost/Team#1372

- Added a new stats service, which is divided into several categories. Currently only the 'members' category for member related stats.
- When there are missing or corrupt members status events in the DB, the totals returned by the old member stats endpoint (`/members/stats/count`) were wrong. This is fixed in the new service by counting in reverse order and starting with the actual totals.
- New Stats API, with the new `/stats/members/count-history` endpoint.
- This new endpoint also returns the paid deltas -> dashboard 5.0 will show subscribed and canceled paid members for each day
- Includes tests for the new stats service and endpoint
This commit is contained in:
Simon Backx 2022-03-31 16:01:11 +02:00 committed by GitHub
parent 9e4401d9f6
commit ae54352a29
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 447 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
core/server/api/canary/index.js
View file

@ -169,6 +169,10 @@ module.exports = {
return shared.pipeline(require('./snippets'), localUtils);
},
get stats() {
return shared.pipeline(require('./stats'), localUtils);
},
get customThemeSettings() {
return shared.pipeline(require('./custom-theme-settings'), localUtils);
},

14
core/server/api/canary/stats.js Normal file
View file

@ -0,0 +1,14 @@
const statsService = require('../../services/stats');
module.exports = {
docName: 'stats',
memberCountHistory: {
permissions: {
docName: 'members',
method: 'browse'
},
async query() {
return await statsService.members.getCountHistory();
}
}
};

1
core/server/services/stats/index.js Normal file
View file

@ -0,0 +1 @@
module.exports = require('./service');

163
core/server/services/stats/lib/members-stats-service.js Normal file
View file

@ -0,0 +1,163 @@
const {DateTime} = require('luxon');
class MembersStatsService {
constructor({db}) {
this.db = db;
}
/**
* Get the current total members grouped by status
* @returns {Promise<TotalMembersByStatus>}
*/
async getCount() {
const knex = this.db.knex;
const rows = await knex('members')
.select('status')
.select(knex.raw('COUNT(id) AS total'))
.groupBy('status');
const paidEvent = rows.find(c => c.status === 'paid');
const freeEvent = rows.find(c => c.status === 'free');
const compedEvent = rows.find(c => c.status === 'comped');
return {
paid: paidEvent ? paidEvent.total : 0,
free: freeEvent ? freeEvent.total : 0,
comped: compedEvent ? compedEvent.total : 0
};
}
/**
* Get the member deltas by status for all days (from new to old)
* @returns {Promise<MemberStatusDelta[]>} The deltas of paid, free and comped users per day, sorted from new to old
*/
async fetchAllStatusDeltas() {
const knex = this.db.knex;
const rows = await knex('members_status_events')
.select(knex.raw('DATE(created_at) as date'))
.select(knex.raw(`SUM(
CASE WHEN to_status='paid' THEN 1
ELSE 0 END
) as paid_subscribed`))
.select(knex.raw(`SUM(
CASE WHEN from_status='paid' THEN 1
ELSE 0 END
) as paid_canceled`))
.select(knex.raw(`SUM(
CASE WHEN to_status='comped' THEN 1
WHEN from_status='comped' THEN -1
ELSE 0 END
) as comped_delta`))
.select(knex.raw(`SUM(
CASE WHEN to_status='free' THEN 1
WHEN from_status='free' THEN -1
ELSE 0 END
) as free_delta`))
.groupByRaw('DATE(created_at)')
.orderByRaw('DATE(created_at) DESC');
return rows;
}
/**
* Returns a list of the total members by status for each day, including the paid deltas paid_subscribed and paid_canceled
* @returns {Promise<CountHistory>}
*/
async getCountHistory() {
const rows = await this.fetchAllStatusDeltas();
// Fetch current total amounts and start counting from there
let {paid, free, comped} = await this.getCount();
// Get today in UTC (default timezone for Luxon)
const today = DateTime.local().toISODate();
const cumulativeResults = [];
for (const row of rows) {
// Convert JSDates to YYYY-MM-DD (in UTC)
const date = DateTime.fromJSDate(row.date).toISODate();
if (date > today) {
// Skip results that are in the future (fix for invalid events)
continue;
}
cumulativeResults.unshift({
date,
paid,
free,
comped,
// Deltas
paid_subscribed: row.paid_subscribed,
paid_canceled: row.paid_canceled
});
// Update current counts
paid = Math.max(0, paid - row.paid_subscribed + row.paid_canceled);
free = Math.max(0, free - row.free_delta);
comped = Math.max(0, comped - row.comped_delta);
}
// Always make sure we have at least one result
if (cumulativeResults.length === 0) {
cumulativeResults.push({
date: today,
paid,
free,
comped,
// Deltas
paid_subscribed: 0,
paid_canceled: 0
});
}
return {
data: cumulativeResults,
meta: {
pagination: {
page: 1,
limit: 'all',
pages: 1,
total: cumulativeResults.length,
next: null,
prev: null
}
}
};
}
}
module.exports = MembersStatsService;
/**
* @typedef MemberStatusDelta
* @type {Object}
* @property {Date} date
* @property {number} paid_subscribed Paid members that subscribed on this day
* @property {number} paid_canceled Paid members that canceled on this day
* @property {number} comped_delta Total net comped members on this day
* @property {number} free_delta Total net members on this day
*/
/**
* @typedef TotalMembersByStatus
* @type {Object}
* @property {number} paid Total paid members
* @property {number} free Total free members
* @property {number} comped Total comped members
*/
/**
* @typedef {Object} TotalMembersByStatusItem
* @property {string} date In YYYY-MM-DD format
* @property {number} paid Total paid members
* @property {number} free Total free members
* @property {number} comped Total comped members
* @property {number} paid_subscribed Paid members that subscribed on this day
* @property {number} paid_canceled Paid members that canceled on this day
*/
/**
* @typedef {Object} CountHistory
* @property {TotalMembersByStatusItem[]} data List of the total members by status for each day, including the paid deltas paid_subscribed and paid_canceled
* @property {Object} meta
*/

6
core/server/services/stats/service.js Normal file
View file

@ -0,0 +1,6 @@
const db = require('../../data/db');
const MemberStatsService = require('./lib/members-stats-service');
module.exports = {
members: new MemberStatsService({db})
};

3
core/server/web/api/canary/admin/routes.js
View file

@ -137,6 +137,9 @@ module.exports = function apiRoutes() {
router.get('/members/:id/signin_urls', mw.authAdminApi, http(api.memberSigninUrls.read));
// ## Stats
router.get('/stats/members/count-history', mw.authAdminApi, http(api.stats.memberCountHistory));
// ## Labels
router.get('/labels', mw.authAdminApi, http(api.labels.browse));
router.get('/labels/:id', mw.authAdminApi, http(api.labels.read));

38
test/e2e-api/admin/__snapshots__/stats.test.js.snap Normal file
View file

@ -0,0 +1,38 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`Stats API Can fetch member count history 1: [body] 1`] = `
Object {
"meta": Object {
"pagination": Object {
"limit": "all",
"next": null,
"page": 1,
"pages": 1,
"prev": null,
"total": 1,
},
},
"stats": Array [
Object {
"comped": 0,
"date": StringMatching /\\\\d\\{4\\}-\\\\d\\{2\\}-\\\\d\\{2\\}/,
"free": 3,
"paid": 5,
"paid_canceled": 0,
"paid_subscribed": 0,
},
],
}
`;
exports[`Stats API Can fetch member count history 2: [headers] 1`] = `
Object {
"access-control-allow-origin": "http://127.0.0.1:2369",
"cache-control": "no-cache, private, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0",
"content-length": "191",
"content-type": "application/json; charset=utf-8",
"etag": StringMatching /\\(\\?:W\\\\/\\)\\?"\\(\\?:\\[ !#-\\\\x7E\\\\x80-\\\\xFF\\]\\*\\|\\\\r\\\\n\\[\\\\t \\]\\|\\\\\\\\\\.\\)\\*"/,
"vary": "Origin, Accept-Encoding",
"x-powered-by": "Express",
}
`;

26
test/e2e-api/admin/stats.test.js Normal file
View file

@ -0,0 +1,26 @@
const {agentProvider, fixtureManager, matchers} = require('../../utils/e2e-framework');
const {anyEtag, anyISODate} = matchers;
let agent;
describe('Stats API', function () {
before(async function () {
agent = await agentProvider.getAdminAPIAgent();
await fixtureManager.init('members');
await agent.loginAsOwner();
});
it('Can fetch member count history', async function () {
await agent
.get(`/stats/members/count-history`)
.expectStatus(200)
.matchBodySnapshot({
stats: [{
date: anyISODate
}]
})
.matchHeaderSnapshot({
etag: anyEtag
});
});
});

192
test/unit/server/services/stats/members-stats-service.test.js Normal file
View file

@ -0,0 +1,192 @@
const MembersStatsService = require('../../../../../core/server/services/stats/lib/members-stats-service');
const {DateTime} = require('luxon');
const sinon = require('sinon');
require('should');
describe('MembersStatsService', function () {
describe('getCountHistory', function () {
let membersStatsService;
let fakeStatuses;
let fakeTotal;
/**
* @type {MembersStatsService.TotalMembersByStatus}
*/
const currentCounts = {paid: 0, free: 0, comped: 0};
/**
* @type {MembersStatsService.MemberStatusDelta[]}
*/
const events = [];
const today = '2000-01-10';
const tomorrow = '2000-01-11';
const yesterday = '2000-01-09';
const todayDate = DateTime.fromISO(today).toJSDate();
const tomorrowDate = DateTime.fromISO(tomorrow).toJSDate();
const yesterdayDate = DateTime.fromISO(yesterday).toJSDate();
before(function () {
sinon.useFakeTimers(todayDate.getTime());
membersStatsService = new MembersStatsService({db: null});
fakeTotal = sinon.stub(membersStatsService, 'getCount').resolves(currentCounts);
fakeStatuses = sinon.stub(membersStatsService, 'fetchAllStatusDeltas').resolves(events);
});
afterEach(function () {
fakeStatuses.resetHistory();
fakeTotal.resetHistory();
});
it('Always returns at least one value', async function () {
// No status events
events.splice(0, events.length);
currentCounts.paid = 1;
currentCounts.free = 2;
currentCounts.comped = 3;
const {data: results} = await membersStatsService.getCountHistory();
results.length.should.eql(1);
results[0].should.eql({
date: today,
paid: 1,
free: 2,
comped: 3,
paid_subscribed: 0,
paid_canceled: 0
});
fakeStatuses.calledOnce.should.eql(true);
fakeTotal.calledOnce.should.eql(true);
});
it('Passes paid_subscribers and paid_canceled', async function () {
// Update faked status events
events.splice(0, events.length, {
date: todayDate,
paid_subscribed: 4,
paid_canceled: 3,
free_delta: 2,
comped_delta: 3
});
// Update current faked counts
currentCounts.paid = 1;
currentCounts.free = 2;
currentCounts.comped = 3;
const {data: results} = await membersStatsService.getCountHistory();
results.length.should.eql(1);
results[0].should.eql({
date: today,
paid: 1,
free: 2,
comped: 3,
paid_subscribed: 4,
paid_canceled: 3
});
fakeStatuses.calledOnce.should.eql(true);
fakeTotal.calledOnce.should.eql(true);
});
it('Correctly resolves deltas', async function () {
// Update faked status events
events.splice(0, events.length,
{
date: todayDate,
paid_subscribed: 4,
paid_canceled: 3,
free_delta: 2,
comped_delta: 3
},
{
date: yesterdayDate,
paid_subscribed: 2,
paid_canceled: 1,
free_delta: 0,
comped_delta: 0
}
);
// Update current faked counts
currentCounts.paid = 2;
currentCounts.free = 3;
currentCounts.comped = 4;
const {data: results} = await membersStatsService.getCountHistory();
results.should.eql([
{
date: yesterday,
paid: 1,
free: 1,
comped: 1,
paid_subscribed: 2,
paid_canceled: 1
},
{
date: today,
paid: 2,
free: 3,
comped: 4,
paid_subscribed: 4,
paid_canceled: 3
}
]);
fakeStatuses.calledOnce.should.eql(true);
fakeTotal.calledOnce.should.eql(true);
});
it('Ignores events in the future', async function () {
// Update faked status events
events.splice(0, events.length,
{
date: tomorrowDate,
paid_subscribed: 10,
paid_canceled: 5,
free_delta: 8,
comped_delta: 9
},
{
date: todayDate,
paid_subscribed: 4,
paid_canceled: 3,
free_delta: 2,
comped_delta: 3
},
{
date: yesterdayDate,
paid_subscribed: 0,
paid_canceled: 0,
free_delta: 0,
comped_delta: 0
}
);
// Update current faked counts
currentCounts.paid = 1;
currentCounts.free = 2;
currentCounts.comped = 3;
const {data: results} = await membersStatsService.getCountHistory();
results.should.eql([
{
date: yesterday,
paid: 0,
free: 0,
comped: 0,
paid_subscribed: 0,
paid_canceled: 0
},
{
date: today,
paid: 1,
free: 2,
comped: 3,
paid_subscribed: 4,
paid_canceled: 3
}
]);
fakeStatuses.calledOnce.should.eql(true);
fakeTotal.calledOnce.should.eql(true);
});
});
});