From f4cb5c57c6484b19b73ff385ebd2f1b56a5bc914 Mon Sep 17 00:00:00 2001 From: Fabien 'egg' O'Carroll Date: Tue, 16 Feb 2021 10:38:36 +0000 Subject: [PATCH] Updated `members_status_events` table (#12647) refs https://github.com/TryGhost/Ghost/issues/12602 * Updated members_status_events table By replacing the `status` column with a `from_status` and `to_status` column, we are able to track the changes between multiple statuses easier, and accumulate the data. e.g. the delta of paid members in a given time range is the sum of the `to_status` columns set to 'paid' minus the sum of the `from_status` columns set to 'paid' within that time range * Updated MEGA to handle addition of 'comped' status With the addition of the 'comped' status, we need to ensure that MEGA will still send emails to the correct recipients. I've opted to use an "inverse" filter, as that is the intention of the free/paid split in MEGA - as far as MEGA is concerned, "free" is the opposite of "paid" * Updated customQuery for MemberStatusEvent With the `status` column replaced with `from_status` and `to_status` this allows us to fix and update the customQuery to correctly accumulate the data into deltas over time, broken down by day. * Populated members_status_events table As the table will be used to generate deltas, we need to backfill the data so that existing sites will be able to sum up the deltas and calculate correct data. The assumptions used in backfilling is that a Member's current status, is their only status. --- .../4.0/10-add-members-status-events-table.js | 3 +- ...17-populate-members-status-events-table.js | 38 +++++++++++++++++++ core/server/data/schema/schema.js | 9 ++++- core/server/models/member-status-event.js | 18 +++++++-- core/server/services/mega/mega.js | 4 +- test/unit/data/schema/integrity_spec.js | 2 +- 6 files changed, 65 insertions(+), 9 deletions(-) create mode 100644 core/server/data/migrations/versions/4.0/17-populate-members-status-events-table.js diff --git a/core/server/data/migrations/versions/4.0/10-add-members-status-events-table.js b/core/server/data/migrations/versions/4.0/10-add-members-status-events-table.js index 6b754d5ee9..6a359345aa 100644 --- a/core/server/data/migrations/versions/4.0/10-add-members-status-events-table.js +++ b/core/server/data/migrations/versions/4.0/10-add-members-status-events-table.js @@ -3,6 +3,7 @@ const {addTable} = require('../../utils'); module.exports = addTable('members_status_events', { id: {type: 'string', maxlength: 24, nullable: false, primary: true}, member_id: {type: 'string', maxlength: 24, nullable: false, references: 'members.id', cascadeDelete: true}, - status: {type: 'string', maxlength: 50, nullable: false}, + from_status: {type: 'string', maxlength: 50, nullable: true}, + to_status: {type: 'string', maxlength: 50, nullable: true}, created_at: {type: 'dateTime', nullable: false} }); diff --git a/core/server/data/migrations/versions/4.0/17-populate-members-status-events-table.js b/core/server/data/migrations/versions/4.0/17-populate-members-status-events-table.js new file mode 100644 index 0000000000..5e361f3537 --- /dev/null +++ b/core/server/data/migrations/versions/4.0/17-populate-members-status-events-table.js @@ -0,0 +1,38 @@ +const {chunk} = require('lodash'); +const ObjectID = require('bson-objectid'); +const logging = require('../../../../../shared/logging'); +const {createTransactionalMigration} = require('../../utils'); + +module.exports = createTransactionalMigration( + async function up(knex) { + logging.info('Populating members_status_events from members table'); + await knex('members_status_events').del(); + const allMembers = await knex.select( + 'id as member_id', + 'status as to_status', + 'created_at' + ).from('members'); + + const membersStatusEvents = allMembers.map((event) => { + return { + ...event, + id: ObjectID.generate(), + from_status: null + }; + }); + + // SQLite3 supports 999 variables max, each row uses 5 variables so ⌊999/5⌋ = 199 + const chunkSize = 199; + + const eventChunks = chunk(membersStatusEvents, chunkSize); + + for (const events of eventChunks) { + await knex.insert(events).into('members_status_events'); + } + }, + async function down(knex) { + logging.info('Deleting all members_status_events'); + return knex('members_status_events').del(); + } +); + diff --git a/core/server/data/schema/schema.js b/core/server/data/schema/schema.js index f8640bf513..98d1648a20 100644 --- a/core/server/data/schema/schema.js +++ b/core/server/data/schema/schema.js @@ -379,8 +379,13 @@ module.exports = { members_status_events: { id: {type: 'string', maxlength: 24, nullable: false, primary: true}, member_id: {type: 'string', maxlength: 24, nullable: false, references: 'members.id', cascadeDelete: true}, - status: { - type: 'string', maxlength: 50, nullable: false, validations: { + from_status: { + type: 'string', maxlength: 50, nullable: true, validations: { + isIn: [['free', 'paid', 'comped']] + } + }, + to_status: { + type: 'string', maxlength: 50, nullable: true, validations: { isIn: [['free', 'paid', 'comped']] } }, diff --git a/core/server/models/member-status-event.js b/core/server/models/member-status-event.js index 25617031a0..83af8add29 100644 --- a/core/server/models/member-status-event.js +++ b/core/server/models/member-status-event.js @@ -11,9 +11,21 @@ const MemberStatusEvent = ghostBookshelf.Model.extend({ const knex = ghostBookshelf.knex; return qb.clear('select') .select(knex.raw('DATE(created_at) as date')) - .select(knex.raw(`SUM(CASE WHEN status='paid' THEN 1 ELSE 0 END) as paid_delta`)) - .select(knex.raw(`SUM(CASE WHEN status='comped' THEN 1 ELSE 0 END) as comped_delta`)) - .select(knex.raw(`SUM(CASE WHEN status='free' THEN 1 ELSE 0 END) as free_delta`)) + .select(knex.raw(`SUM( + CASE WHEN to_status='paid' THEN 1 + CASE WHEN from_status='paid' THEN -1 + ELSE 0 END + ) as paid_delta`)) + .select(knex.raw(`SUM( + CASE WHEN to_status='comped' THEN 1 + CASE WHEN from_status='comped' THEN -1 + ELSE 0 END + ) as comped_delta`)) + .select(knex.raw(`SUM( + CASE WHEN to_status='free' THEN 1 + CASE WHEN from_status='free' THEN -1 + ELSE 0 END + ) as free_delta`)) .groupByRaw('DATE(created_at)') .orderByRaw('DATE(created_at)'); } diff --git a/core/server/services/mega/mega.js b/core/server/services/mega/mega.js index ac96da8f13..3ad98f152e 100644 --- a/core/server/services/mega/mega.js +++ b/core/server/services/mega/mega.js @@ -98,7 +98,7 @@ const addEmail = async (postModel, options) => { switch (emailRecipientFilter) { case 'paid': - filterOptions.filter = 'subscribed:true+status:paid'; + filterOptions.filter = 'subscribed:true+status:-free'; break; case 'free': filterOptions.filter = 'subscribed:true+status:free'; @@ -295,7 +295,7 @@ async function getEmailMemberRows({emailModel, options}) { switch (recipientFilter) { case 'paid': - filterOptions.filter = 'subscribed:true+status:paid'; + filterOptions.filter = 'subscribed:true+status:-free'; break; case 'free': filterOptions.filter = 'subscribed:true+status:free'; diff --git a/test/unit/data/schema/integrity_spec.js b/test/unit/data/schema/integrity_spec.js index 7a4b5cbd05..470101107d 100644 --- a/test/unit/data/schema/integrity_spec.js +++ b/test/unit/data/schema/integrity_spec.js @@ -32,7 +32,7 @@ const defaultSettings = require('../../../../core/server/data/schema/default-set */ describe('DB version integrity', function () { // Only these variables should need updating - const currentSchemaHash = '3f88f2c34001cc34d74d945dbf4f0bb5'; + const currentSchemaHash = 'd23279e16028161ab337000744480111'; const currentFixturesHash = '370d0da0ab7c45050b2ff30bce8896ba'; const currentSettingsHash = '24453dc02be9df7284acf1748862a545'; const currentRoutesHash = '3d180d52c663d173a6be791ef411ed01';