From d5a97318452c3554bf9a2ee86c8317eeb284da58 Mon Sep 17 00:00:00 2001 From: Kevin Ansfield Date: Wed, 3 Apr 2024 17:52:52 +0100 Subject: [PATCH] Fixed email_recipients indexes to match query usage (#19918) closes https://linear.app/tryghost/issue/ENG-791/migration-to-fix-email-recipients-indexes Our indexes over single columns (`delivered_at`, `opened_at`, `failed_at`) were ineffective because the only time we query those is alongside `email_id` meaning we were frequently performing full table scans on very large tables during our email analytics jobs. - added migration to add new indexes covering `email_id` and the respective columns - added migration to drop the old indexes that weren't being used in any query plans Local runtime with ~2M email_recipient rows: - before: 1.7s - after: 99ms Explain output... before: ``` +----+-------------+------------------+------------+-------+----------------------------------------------------------------------------------+----------------------------------------------+---------+-------+--------+----------+------------------------------------+ | id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra | +----+-------------+------------------+------------+-------+----------------------------------------------------------------------------------+----------------------------------------------+---------+-------+--------+----------+------------------------------------+ | 1 | UPDATE | emails | NULL | index | NULL | PRIMARY | 98 | NULL | 1 | 100.00 | Using where | | 4 | SUBQUERY | email_recipients | NULL | range | email_recipients_email_id_member_email_index,email_recipients_failed_at_index | email_recipients_failed_at_index | 6 | NULL | 2343 | 7.76 | Using index condition; Using where | | 3 | SUBQUERY | email_recipients | NULL | ref | email_recipients_email_id_member_email_index,email_recipients_opened_at_index | email_recipients_email_id_member_email_index | 98 | const | 159126 | 50.00 | Using where | | 2 | SUBQUERY | email_recipients | NULL | ref | email_recipients_email_id_member_email_index,email_recipients_delivered_at_index | email_recipients_email_id_member_email_index | 98 | const | 159126 | 50.00 | Using where | +----+-------------+------------------+------------+-------+----------------------------------------------------------------------------------+----------------------------------------------+---------+-------+--------+----------+------------------------------------+ ``` after: ``` +----+-------------+------------------+------------+-------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+---------+------+--------+----------+--------------------------+ | id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra | +----+-------------+------------------+------------+-------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+---------+------+--------+----------+--------------------------+ | 1 | UPDATE | emails | NULL | index | NULL | PRIMARY | 98 | NULL | 1 | 100.00 | Using where; | | 4 | SUBQUERY | email_recipients | NULL | range | email_recipients_email_id_member_email_index,email_recipients_email_id_delivered_at_index,email_recipients_email_id_opened_at_index,email_recipients_email_id_failed_at_index | email_recipients_email_id_failed_at_index | 104 | NULL | 60 | 100.00 | Using where; Using index | | 3 | SUBQUERY | email_recipients | NULL | range | email_recipients_email_id_member_email_index,email_recipients_email_id_delivered_at_index,email_recipients_email_id_opened_at_index,email_recipients_email_id_failed_at_index | email_recipients_email_id_opened_at_index | 104 | NULL | 119496 | 100.00 | Using where; Using index | | 2 | SUBQUERY | email_recipients | NULL | range | email_recipients_email_id_member_email_index,email_recipients_email_id_delivered_at_index,email_recipients_email_id_opened_at_index,email_recipients_email_id_failed_at_index | email_recipients_email_id_delivered_at_index | 104 | NULL | 146030 | 100.00 | Using where; Using index | +----+-------------+------------------+------------+-------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+---------+------+--------+----------+--------------------------+ ``` --- ...-10-add-email-recipients-email-id-indexes.js | 17 +++++++++++++++++ ...rop-email-recipients-non-email-id-indexes.js | 17 +++++++++++++++++ ghost/core/core/server/data/schema/schema.js | 11 +++++++---- .../unit/server/data/schema/integrity.test.js | 2 +- 4 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 ghost/core/core/server/data/migrations/versions/5.82/2024-03-25-16-46-10-add-email-recipients-email-id-indexes.js create mode 100644 ghost/core/core/server/data/migrations/versions/5.82/2024-03-25-16-51-29-drop-email-recipients-non-email-id-indexes.js diff --git a/ghost/core/core/server/data/migrations/versions/5.82/2024-03-25-16-46-10-add-email-recipients-email-id-indexes.js b/ghost/core/core/server/data/migrations/versions/5.82/2024-03-25-16-46-10-add-email-recipients-email-id-indexes.js new file mode 100644 index 0000000000..5f97c8f719 --- /dev/null +++ b/ghost/core/core/server/data/migrations/versions/5.82/2024-03-25-16-46-10-add-email-recipients-email-id-indexes.js @@ -0,0 +1,17 @@ +// For information on writing migrations, see https://www.notion.so/ghost/Database-migrations-eb5b78c435d741d2b34a582d57c24253 + +const {createNonTransactionalMigration} = require('../../utils'); +const commands = require('../../../schema/commands'); + +module.exports = createNonTransactionalMigration( + async function up(knex) { + await commands.addIndex('email_recipients', ['email_id', 'delivered_at'], knex); + await commands.addIndex('email_recipients', ['email_id', 'opened_at'], knex); + await commands.addIndex('email_recipients', ['email_id', 'failed_at'], knex); + }, + async function down(knex) { + await commands.dropIndex('email_recipients', ['email_id', 'delivered_at'], knex); + await commands.dropIndex('email_recipients', ['email_id', 'opened_at'], knex); + await commands.dropIndex('email_recipients', ['email_id', 'failed_at'], knex); + } +); diff --git a/ghost/core/core/server/data/migrations/versions/5.82/2024-03-25-16-51-29-drop-email-recipients-non-email-id-indexes.js b/ghost/core/core/server/data/migrations/versions/5.82/2024-03-25-16-51-29-drop-email-recipients-non-email-id-indexes.js new file mode 100644 index 0000000000..7b6d5a0717 --- /dev/null +++ b/ghost/core/core/server/data/migrations/versions/5.82/2024-03-25-16-51-29-drop-email-recipients-non-email-id-indexes.js @@ -0,0 +1,17 @@ +// For information on writing migrations, see https://www.notion.so/ghost/Database-migrations-eb5b78c435d741d2b34a582d57c24253 + +const {createNonTransactionalMigration} = require('../../utils'); +const commands = require('../../../schema/commands'); + +module.exports = createNonTransactionalMigration( + async function up(knex) { + await commands.dropIndex('email_recipients', ['delivered_at'], knex); + await commands.dropIndex('email_recipients', ['opened_at'], knex); + await commands.dropIndex('email_recipients', ['failed_at'], knex); + }, + async function down(knex) { + await commands.addIndex('email_recipients', ['delivered_at'], knex); + await commands.addIndex('email_recipients', ['opened_at'], knex); + await commands.addIndex('email_recipients', ['failed_at'], knex); + } +); diff --git a/ghost/core/core/server/data/schema/schema.js b/ghost/core/core/server/data/schema/schema.js index 338318f0b9..21e0ed51f6 100644 --- a/ghost/core/core/server/data/schema/schema.js +++ b/ghost/core/core/server/data/schema/schema.js @@ -865,14 +865,17 @@ module.exports = { member_id: {type: 'string', maxlength: 24, nullable: false, index: true}, batch_id: {type: 'string', maxlength: 24, nullable: false, references: 'email_batches.id'}, processed_at: {type: 'dateTime', nullable: true}, - delivered_at: {type: 'dateTime', nullable: true, index: true}, - opened_at: {type: 'dateTime', nullable: true, index: true}, - failed_at: {type: 'dateTime', nullable: true, index: true}, + delivered_at: {type: 'dateTime', nullable: true}, + opened_at: {type: 'dateTime', nullable: true}, + failed_at: {type: 'dateTime', nullable: true}, member_uuid: {type: 'string', maxlength: 36, nullable: false}, member_email: {type: 'string', maxlength: 191, nullable: false}, member_name: {type: 'string', maxlength: 191, nullable: true}, '@@INDEXES@@': [ - ['email_id', 'member_email'] + ['email_id', 'member_email'], + ['email_id', 'delivered_at'], + ['email_id', 'opened_at'], + ['email_id', 'failed_at'] ] }, email_recipient_failures: { diff --git a/ghost/core/test/unit/server/data/schema/integrity.test.js b/ghost/core/test/unit/server/data/schema/integrity.test.js index 6e5aa22ec0..ccb1443be2 100644 --- a/ghost/core/test/unit/server/data/schema/integrity.test.js +++ b/ghost/core/test/unit/server/data/schema/integrity.test.js @@ -35,7 +35,7 @@ const validateRouteSettings = require('../../../../../core/server/services/route */ describe('DB version integrity', function () { // Only these variables should need updating - const currentSchemaHash = '34a9fa4dc1223ef6c45f8ed991d25de5'; + const currentSchemaHash = 'ccf3893bc3f8930f0d1188e646abda6d'; const currentFixturesHash = 'a489d615989eab1023d4b8af0ecee7fd'; const currentSettingsHash = '5c957ceb48c4878767d7d3db484c592d'; const currentRoutesHash = '3d180d52c663d173a6be791ef411ed01';