Added migrations for email analytics (#12387)

no issue

- cleans up unused tables `emails.{meta,stats}`
- adds timestamp columns `email_recipients.{delivered_at,opened_at,failed_at}` that can be used for event timelines and basic stats aggregation
  - indexed because we want to sort by these columns to find the "latest event" when limiting Mailgun events API requests
- adds aggregated stats columns `emails.{delivered_count,opened_count,failed_count}`
- adds a composite index on `email_recipients.[email_id,member_email]` to dramatically speed up `email_recipient` update queries when processing events
  - modifies the db initialisation to support an `'@@INDEXES@@'` key in table schema definition for composite indexes

core/server/data/migrations/versions/3.39/05-remove-unused-columns-on-emails.js

@@ -0,0 +1,14 @@
+const {createDropColumnMigration, combineNonTransactionalMigrations} = require('../../utils');
+
+module.exports = combineNonTransactionalMigrations(
+    createDropColumnMigration('emails', 'meta', {
+        type: 'text',
+        length: 65535,
+        nullable: true
+    }),
+    createDropColumnMigration('emails', 'stats', {
+        type: 'text',
+        length: 65535,
+        nullable: true
+    })
+);

core/server/data/migrations/versions/3.39/06-add-email-recipient-index.js

@@ -0,0 +1,57 @@
+const logging = require('../../../../../shared/logging');
+const {createNonTransactionalMigration} = require('../../utils');
+
+module.exports = createNonTransactionalMigration(
+    async function up(knex) {
+        let hasIndex = false;
+
+        if (knex.client.config.client === 'sqlite3') {
+            const result = await knex.raw(`select * from sqlite_master where type = 'index' and tbl_name = 'email_recipients' and name = 'email_recipients_email_id_member_email_index'`);
+            hasIndex = result.length !== 0;
+        } else {
+            const result = await knex.raw(`show index from email_recipients where Key_name = 'email_recipients_email_id_member_email_index'`);
+            hasIndex = result[0].length !== 0;
+        }
+
+        if (hasIndex) {
+            logging.info('Skipping creation of composite index on email_recipients for [email_id, member_email] - already exists');
+            return;
+        }
+
+        logging.info('Creating composite index on email_recipients for [email_id, member_email]');
+        await knex.schema.table('email_recipients', (table) => {
+            table.index(['email_id', 'member_email']);
+        });
+    },
+
+    async function down(knex) {
+        let missingIndex = false;
+
+        if (knex.client.config.client === 'sqlite3') {
+            const result = await knex.raw(`select * from sqlite_master where type = 'index' and tbl_name = 'email_recipients' and name = 'email_recipients_email_id_member_email_index'`);
+            missingIndex = result.length === 0;
+        } else {
+            const result = await knex.raw(`show index from email_recipients where Key_name = 'email_recipients_email_id_member_email_index'`);
+            missingIndex = result[0].length === 0;
+        }
+
+        if (missingIndex) {
+            logging.info('Skipping drop of composite index on email_recipients for [email_id, member_email] - does not exist');
+            return;
+        }
+
+        logging.info('Dropping composite index on email_recipients for [email_id, member_email]');
+
+        if (knex.client.config.client === 'mysql') {
+            await knex.schema.table('email_recipients', (table) => {
+                table.dropForeign('email_id');
+                table.dropIndex(['email_id', 'member_email']);
+                table.foreign('email_id').references('emails.id');
+            });
+        } else {
+            await knex.schema.table('email_recipients', (table) => {
+                table.dropIndex(['email_id', 'member_email']);
+            });
+        }
+    }
+);

core/server/data/migrations/versions/3.39/07-add-email-recipients-event-timestamps.js

@@ -0,0 +1,19 @@
+const {createAddColumnMigration, combineNonTransactionalMigrations} = require('../../utils');
+
+module.exports = combineNonTransactionalMigrations(
+    createAddColumnMigration('email_recipients', 'delivered_at', {
+        type: 'dateTime',
+        nullable: true,
+        index: true
+    }),
+    createAddColumnMigration('email_recipients', 'opened_at', {
+        type: 'dateTime',
+        nullable: true,
+        index: true
+    }),
+    createAddColumnMigration('email_recipients', 'failed_at', {
+        type: 'dateTime',
+        nullable: true,
+        index: true
+    })
+);

core/server/data/migrations/versions/3.39/08-add-email-stats-columns.js

@@ -0,0 +1,22 @@
+const {createAddColumnMigration, combineNonTransactionalMigrations} = require('../../utils');
+
+module.exports = combineNonTransactionalMigrations(
+    createAddColumnMigration('emails', 'delivered_count', {
+        type: 'integer',
+        unsigned: true,
+        nullable: false,
+        defaultTo: 0
+    }),
+    createAddColumnMigration('emails', 'opened_count', {
+        type: 'integer',
+        unsigned: true,
+        nullable: false,
+        defaultTo: 0
+    }),
+    createAddColumnMigration('emails', 'failed_count', {
+        type: 'integer',
+        unsigned: true,
+        nullable: false,
+        defaultTo: 0
+    })
+);

core/server/data/schema/commands.js

@@ -87,10 +87,21 @@ function createTable(table, transaction) {
             }
 
             return (transaction || db.knex).schema.createTable(table, function (t) {
+                let tableIndexes = [];
+
                 const columnKeys = _.keys(schema[table]);
                 _.each(columnKeys, function (column) {
+                    if (column === '@@INDEXES@@') {
+                        tableIndexes = schema[table]['@@INDEXES@@'];
+                        return;
+                    }
+
                     return addTableColumn(table, t, column);
                 });
+
+                _.each(tableIndexes, function (index) {
+                    t.index(index);
+                });
             });
         });
 }

core/server/data/schema/schema.js

@@ -463,9 +463,10 @@ module.exports = {
         },
         error: {type: 'string', maxlength: 2000, nullable: true},
         error_data: {type: 'text', maxlength: 1000000000, fieldtype: 'long', nullable: true},
-        meta: {type: 'text', maxlength: 65535, nullable: true},
-        stats: {type: 'text', maxlength: 65535, nullable: true},
         email_count: {type: 'integer', nullable: false, unsigned: true, defaultTo: 0},
+        delivered_count: {type: 'integer', nullable: false, unsigned: true, defaultTo: 0},
+        opened_count: {type: 'integer', nullable: false, unsigned: true, defaultTo: 0},
+        failed_count: {type: 'integer', nullable: false, unsigned: true, defaultTo: 0},
         subject: {type: 'string', maxlength: 300, nullable: true},
         from: {type: 'string', maxlength: 2000, nullable: true},
         reply_to: {type: 'string', maxlength: 2000, nullable: true},
@@ -498,9 +499,15 @@ module.exports = {
         member_id: {type: 'string', maxlength: 24, nullable: false, index: true},
         batch_id: {type: 'string', maxlength: 24, nullable: false, references: 'email_batches.id'},
         processed_at: {type: 'dateTime', nullable: true},
+        delivered_at: {type: 'dateTime', nullable: true, index: true},
+        opened_at: {type: 'dateTime', nullable: true, index: true},
+        failed_at: {type: 'dateTime', nullable: true, index: true},
         member_uuid: {type: 'string', maxlength: 36, nullable: false},
         member_email: {type: 'string', maxlength: 191, nullable: false},
-        member_name: {type: 'string', maxlength: 191, nullable: true}
+        member_name: {type: 'string', maxlength: 191, nullable: true},
+        '@@INDEXES@@': [
+            ['email_id', 'member_email']
+        ]
     },
     tokens: {
         id: {type: 'string', maxlength: 24, nullable: false, primary: true},

core/server/models/email.js

@@ -9,15 +9,10 @@ const Email = ghostBookshelf.Model.extend({
             uuid: uuid.v4(),
             status: 'pending',
             recipient_filter: 'paid',
-            stats: JSON.stringify({
-                delivered: 0,
-                failed: 0,
-                opened: 0,
-                clicked: 0,
-                unsubscribed: 0,
-                complaints: 0
-            }),
-            track_opens: false
+            track_opens: false,
+            delivered_count: 0,
+            opened_count: 0,
+            failed_count: 0
         };
     },
 

test/api-acceptance/admin/emails_spec.js

@@ -33,15 +33,6 @@ describe('Email API', function () {
                 should.exist(jsonResponse.emails);
                 jsonResponse.emails.should.have.length(1);
                 localUtils.API.checkResponse(jsonResponse.emails[0], 'email');
-
-                const stats = JSON.parse(jsonResponse.emails[0].stats);
-
-                should.exist(stats.delivered);
-                should.exist(stats.failed);
-                should.exist(stats.opened);
-                should.exist(stats.clicked);
-                should.exist(stats.unsubscribed);
-                should.exist(stats.complaints);
             });
     });
 });

test/unit/data/schema/integrity_spec.js

@@ -32,7 +32,7 @@ const defaultSettings = require('../../../../core/server/data/schema/default-set
  */
 describe('DB version integrity', function () {
     // Only these variables should need updating
-    const currentSchemaHash = 'ce6d902fd2b0e3921ffb76d61a35e285';
+    const currentSchemaHash = 'bd3ddfbe8e2ec223d8a57c9ebbe7b2ba';
     const currentFixturesHash = 'd46d696c94d03e41a5903500547fea77';
     const currentSettingsHash = 'd3821715e4b34d92d6ba6ed0d4918f5c';
     const currentRoutesHash = '3d180d52c663d173a6be791ef411ed01';