Infrastructure/ghost
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2025-04-01 02:41:39 -05:00
Code Issues Activity

Fixed email_recipients indexes to match query usage (#19918)

Browse source 
closes https://linear.app/tryghost/issue/ENG-791/migration-to-fix-email-recipients-indexes

Our indexes over single columns (`delivered_at`, `opened_at`, `failed_at`) were ineffective because the only time we query those is alongside `email_id` meaning we were frequently performing full table scans on very large tables during our email analytics jobs.

- added migration to add new indexes covering `email_id` and the respective columns
- added migration to drop the old indexes that weren't being used in any query plans

Local runtime with ~2M email_recipient rows:
- before: 1.7s
- after: 99ms

Explain output...

before:
```
+----+-------------+------------------+------------+-------+----------------------------------------------------------------------------------+----------------------------------------------+---------+-------+--------+----------+------------------------------------+
| id | select_type | table            | partitions | type  | possible_keys                                                                    | key                                          | key_len | ref   | rows   | filtered | Extra                              |
+----+-------------+------------------+------------+-------+----------------------------------------------------------------------------------+----------------------------------------------+---------+-------+--------+----------+------------------------------------+
|  1 | UPDATE      | emails           | NULL       | index | NULL                                                                             | PRIMARY                                      | 98      | NULL  |      1 |   100.00 | Using where                        |
|  4 | SUBQUERY    | email_recipients | NULL       | range | email_recipients_email_id_member_email_index,email_recipients_failed_at_index    | email_recipients_failed_at_index             | 6       | NULL  |   2343 |     7.76 | Using index condition; Using where |
|  3 | SUBQUERY    | email_recipients | NULL       | ref   | email_recipients_email_id_member_email_index,email_recipients_opened_at_index    | email_recipients_email_id_member_email_index | 98      | const | 159126 |    50.00 | Using where                        |
|  2 | SUBQUERY    | email_recipients | NULL       | ref   | email_recipients_email_id_member_email_index,email_recipients_delivered_at_index | email_recipients_email_id_member_email_index | 98      | const | 159126 |    50.00 | Using where                        |
+----+-------------+------------------+------------+-------+----------------------------------------------------------------------------------+----------------------------------------------+---------+-------+--------+----------+------------------------------------+
```

after:
```
+----+-------------+------------------+------------+-------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+---------+------+--------+----------+--------------------------+
| id | select_type | table            | partitions | type  | possible_keys                                                                                                                                                                 | key                                          | key_len | ref  | rows   | filtered | Extra                    |
+----+-------------+------------------+------------+-------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+---------+------+--------+----------+--------------------------+
|  1 | UPDATE      | emails           | NULL       | index | NULL                                                                                                                                                                          | PRIMARY                                      | 98      | NULL |      1 |   100.00 | Using where;             |
|  4 | SUBQUERY    | email_recipients | NULL       | range | email_recipients_email_id_member_email_index,email_recipients_email_id_delivered_at_index,email_recipients_email_id_opened_at_index,email_recipients_email_id_failed_at_index | email_recipients_email_id_failed_at_index    | 104     | NULL |     60 |   100.00 | Using where; Using index |
|  3 | SUBQUERY    | email_recipients | NULL       | range | email_recipients_email_id_member_email_index,email_recipients_email_id_delivered_at_index,email_recipients_email_id_opened_at_index,email_recipients_email_id_failed_at_index | email_recipients_email_id_opened_at_index    | 104     | NULL | 119496 |   100.00 | Using where; Using index |
|  2 | SUBQUERY    | email_recipients | NULL       | range | email_recipients_email_id_member_email_index,email_recipients_email_id_delivered_at_index,email_recipients_email_id_opened_at_index,email_recipients_email_id_failed_at_index | email_recipients_email_id_delivered_at_index | 104     | NULL | 146030 |   100.00 | Using where; Using index |
+----+-------------+------------------+------------+-------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+---------+------+--------+----------+--------------------------+
```
This commit is contained in:
Kevin Ansfield 2024-04-03 17:52:52 +01:00 • committed by GitHub
parent 2a119cc4a4
commit d5a9731845
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 42 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
ghost/core/core/server/data/migrations/versions/5.82/2024-03-25-16-46-10-add-email-recipients-email-id-indexes.js Normal file
View file

@ -0,0 +1,17 @@
// For information on writing migrations, see https://www.notion.so/ghost/Database-migrations-eb5b78c435d741d2b34a582d57c24253
const {createNonTransactionalMigration} = require('../../utils');
const commands = require('../../../schema/commands');
module.exports = createNonTransactionalMigration(
async function up(knex) {
await commands.addIndex('email_recipients', ['email_id', 'delivered_at'], knex);
await commands.addIndex('email_recipients', ['email_id', 'opened_at'], knex);
await commands.addIndex('email_recipients', ['email_id', 'failed_at'], knex);
},
async function down(knex) {
await commands.dropIndex('email_recipients', ['email_id', 'delivered_at'], knex);
await commands.dropIndex('email_recipients', ['email_id', 'opened_at'], knex);
await commands.dropIndex('email_recipients', ['email_id', 'failed_at'], knex);
}
);

17
ghost/core/core/server/data/migrations/versions/5.82/2024-03-25-16-51-29-drop-email-recipients-non-email-id-indexes.js Normal file
View file

@ -0,0 +1,17 @@
// For information on writing migrations, see https://www.notion.so/ghost/Database-migrations-eb5b78c435d741d2b34a582d57c24253
const {createNonTransactionalMigration} = require('../../utils');
const commands = require('../../../schema/commands');
module.exports = createNonTransactionalMigration(
async function up(knex) {
await commands.dropIndex('email_recipients', ['delivered_at'], knex);
await commands.dropIndex('email_recipients', ['opened_at'], knex);
await commands.dropIndex('email_recipients', ['failed_at'], knex);
},
async function down(knex) {
await commands.addIndex('email_recipients', ['delivered_at'], knex);
await commands.addIndex('email_recipients', ['opened_at'], knex);
await commands.addIndex('email_recipients', ['failed_at'], knex);
}
);

11
ghost/core/core/server/data/schema/schema.js
View file

@ -865,14 +865,17 @@ module.exports = {
member_id: {type: 'string', maxlength: 24, nullable: false, index: true},
batch_id: {type: 'string', maxlength: 24, nullable: false, references: 'email_batches.id'},
processed_at: {type: 'dateTime', nullable: true},
delivered_at: {type: 'dateTime', nullable: true, index: true},
opened_at: {type: 'dateTime', nullable: true, index: true},
failed_at: {type: 'dateTime', nullable: true, index: true},
delivered_at: {type: 'dateTime', nullable: true},
opened_at: {type: 'dateTime', nullable: true},
failed_at: {type: 'dateTime', nullable: true},
member_uuid: {type: 'string', maxlength: 36, nullable: false},
member_email: {type: 'string', maxlength: 191, nullable: false},
member_name: {type: 'string', maxlength: 191, nullable: true},
'@@INDEXES@@': [
['email_id', 'member_email']
['email_id', 'member_email'],
['email_id', 'delivered_at'],
['email_id', 'opened_at'],
['email_id', 'failed_at']
]
},
email_recipient_failures: {

2
ghost/core/test/unit/server/data/schema/integrity.test.js
View file

@ -35,7 +35,7 @@ const validateRouteSettings = require('../../../../../core/server/services/route
*/
describe('DB version integrity', function () {
// Only these variables should need updating
const currentSchemaHash = '34a9fa4dc1223ef6c45f8ed991d25de5';
const currentSchemaHash = 'ccf3893bc3f8930f0d1188e646abda6d';
const currentFixturesHash = 'a489d615989eab1023d4b8af0ecee7fd';
const currentSettingsHash = '5c957ceb48c4878767d7d3db484c592d';
const currentRoutesHash = '3d180d52c663d173a6be791ef411ed01';