Added indexes to members_created_events and members_subscription_created_events (#18805)

fixes https://github.com/TryGhost/Product/issues/4085

Increases the performance for the post analytics export by adding new
indexes. These indexes are used when counting the amount of (paid)
subscribers that were attributed to a given post. With the indexes, the
time required to export 700 posts with 300k members decreases from 40s
to 0.6s.

Tests show that adding these indexes should be very fast (< 1 s) if the
tables contain up to 300k rows.

ghost/core/core/server/data/migrations/utils/schema.js

@@ -96,6 +96,22 @@ function createSetNullableMigration(table, column, options = {}) {
     );
 }
 
+/**
+ * @param {string} table
+ * @param {string[]|string} columns One or multiple columns (in case the index should be for multiple columns)
+ * @returns {Migration}
+ */
+function createAddIndexMigration(table, columns) {
+    return createTransactionalMigration(
+        async function up(knex) {
+            await commands.addIndex(table, columns, knex);
+        },
+        async function down(knex) {
+            await commands.dropIndex(table, columns, knex);
+        }
+    );
+}
+
 /**
  * @param {string} table
  * @param {string} from
@@ -163,7 +179,8 @@ module.exports = {
     createDropColumnMigration,
     createSetNullableMigration,
     createDropNullableMigration,
-    createRenameColumnMigration
+    createRenameColumnMigration,
+    createAddIndexMigration
 };
 
 /**

ghost/core/core/server/data/migrations/versions/5.72/2023-10-31-11-06-00-members-created-attribution-id-index.js

@@ -0,0 +1,3 @@
+const {createAddIndexMigration} = require('../../utils');
+
+module.exports = createAddIndexMigration('members_created_events', ['attribution_id']);

ghost/core/core/server/data/migrations/versions/5.72/2023-10-31-11-06-01-members-subscription-created-attribution-id-index.js

@@ -0,0 +1,3 @@
+const {createAddIndexMigration} = require('../../utils');
+
+module.exports = createAddIndexMigration('members_subscription_created_events', ['attribution_id']);

ghost/core/core/server/data/schema/commands.js

@@ -175,6 +175,60 @@ async function renameColumn(tableName, from, to, transaction = db.knex) {
     });
 }
 
+/**
+ * Adds an non-unique index to a table over the given columns.
+ *
+ * @param {string} tableName - name of the table to add indexes to
+ * @param {string|string[]} columns - column(s) to add indexes for
+ * @param {import('knex').Knex} [transaction] - connection object containing knex reference
+ */
+async function addIndex(tableName, columns, transaction = db.knex) {
+    try {
+        logging.info(`Adding index for '${columns}' in table '${tableName}'`);
+
+        return await transaction.schema.table(tableName, function (table) {
+            table.index(columns);
+        });
+    } catch (err) {
+        if (err.code === 'SQLITE_ERROR') {
+            logging.warn(`Index for '${columns}' already exists for table '${tableName}'`);
+            return;
+        }
+        if (err.code === 'ER_DUP_KEYNAME') {
+            logging.warn(`Index for '${columns}' already exists for table '${tableName}'`);
+            return;
+        }
+        throw err;
+    }
+}
+
+/**
+ * Drops a non-unique index from a table over the given columns.
+ *
+ * @param {string} tableName - name of the table to remove indexes from
+ * @param {string|string[]} columns - column(s) to remove indexes for
+ * @param {import('knex').Knex} [transaction] - connection object containing knex reference
+ */
+async function dropIndex(tableName, columns, transaction = db.knex) {
+    try {
+        logging.info(`Dropping index for '${columns}' in table '${tableName}'`);
+
+        return await transaction.schema.table(tableName, function (table) {
+            table.dropIndex(columns);
+        });
+    } catch (err) {
+        if (err.code === 'SQLITE_ERROR') {
+            logging.warn(`Constraint for '${columns}' does not exist for table '${tableName}'`);
+            return;
+        }
+        if (err.code === 'ER_CANT_DROP_FIELD_OR_KEY') {
+            logging.warn(`Constraint for '${columns}' does not exist for table '${tableName}'`);
+            return;
+        }
+        throw err;
+    }
+}
+
 /**
  * Adds an unique index to a table over the given columns.
  *
@@ -535,6 +589,8 @@ module.exports = {
     getIndexes,
     addUnique,
     dropUnique,
+    addIndex,
+    dropIndex,
     addPrimaryKey,
     addForeign,
     dropForeign,

ghost/core/core/server/data/schema/schema.js

@@ -523,7 +523,7 @@ module.exports = {
         id: {type: 'string', maxlength: 24, nullable: false, primary: true},
         created_at: {type: 'dateTime', nullable: false},
         member_id: {type: 'string', maxlength: 24, nullable: false, references: 'members.id', cascadeDelete: true},
-        attribution_id: {type: 'string', maxlength: 24, nullable: true},
+        attribution_id: {type: 'string', maxlength: 24, nullable: true, index: true},
         attribution_type: {
             type: 'string', maxlength: 50, nullable: true, validations: {
                 isIn: [['url', 'post', 'page', 'author', 'tag']]
@@ -709,7 +709,7 @@ module.exports = {
         created_at: {type: 'dateTime', nullable: false},
         member_id: {type: 'string', maxlength: 24, nullable: false, references: 'members.id', cascadeDelete: true},
         subscription_id: {type: 'string', maxlength: 24, nullable: false, references: 'members_stripe_customers_subscriptions.id', cascadeDelete: true},
-        attribution_id: {type: 'string', maxlength: 24, nullable: true},
+        attribution_id: {type: 'string', maxlength: 24, nullable: true, index: true},
         attribution_type: {
             type: 'string', maxlength: 50, nullable: true, validations: {
                 isIn: [['url', 'post', 'page', 'author', 'tag']]

ghost/core/test/unit/server/data/schema/integrity.test.js

@@ -35,7 +35,7 @@ const validateRouteSettings = require('../../../../../core/server/services/route
  */
 describe('DB version integrity', function () {
     // Only these variables should need updating
-    const currentSchemaHash = '013334f4f51aae785b9afd345861c06a';
+    const currentSchemaHash = '1c1f476e830ce01a0167229697bd4523';
     const currentFixturesHash = '4db87173699ad9c9d8a67ccab96dfd2d';
     const currentSettingsHash = '3128d4ec667a50049486b0c21f04be07';
     const currentRoutesHash = '3d180d52c663d173a6be791ef411ed01';