From 76c1b60a4dee65da556ea8b31e189fce5c60ed09 Mon Sep 17 00:00:00 2001 From: Kevin Ansfield Date: Mon, 14 Sep 2020 12:21:58 +0100 Subject: [PATCH] Added schema+migration for email_{batches,recipients} tables (#12192) no issue We want to store a list of recipients for each bulk email so that we have a consistent set of data that background processing/sending jobs can work from without worrying about moving large data sets around or member data changing mid-send. - `email_batches` table acts as a join table with status for email<->email_recipient - stores a provider-specific ID that we get back when submitting a batch for sending to the bulk email provider - `status` allows for batch-specific status updates and picking up where we left off when submitting batches if needed - explicitly tying a list of email recipients to a batch allows for partial retries - `email_recipients` table acts as a join table for email<->member - `member_id` does not have a foreign key constraint because members can be deleted but does have an index so that we can efficiently query which emails a member has received - stores static copies of the member info present at the time of sending an email for consistency in background jobs and auditing/historical data --- .../3.33/01-add-email-recipients-tables.js | 34 +++++++++++++++++++ core/server/data/schema/schema.js | 24 +++++++++++++ test/api-acceptance/admin/db_spec.js | 2 +- test/unit/data/schema/integrity_spec.js | 2 +- 4 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 core/server/data/migrations/versions/3.33/01-add-email-recipients-tables.js diff --git a/core/server/data/migrations/versions/3.33/01-add-email-recipients-tables.js b/core/server/data/migrations/versions/3.33/01-add-email-recipients-tables.js new file mode 100644 index 0000000000..38cbc3f272 --- /dev/null +++ b/core/server/data/migrations/versions/3.33/01-add-email-recipients-tables.js @@ -0,0 +1,34 @@ +const logging = require('../../../../../shared/logging'); +const commands = require('../../../schema').commands; + +module.exports = { + async up({connection}) { + // table creation order is important because of foreign key constraints, + // email_recipients references email_batches so email_batches has to exist when creating + return Promise.each(['email_batches', 'email_recipients'], async (table) => { + const tableExists = await connection.schema.hasTable(table); + + if (tableExists) { + return logging.warn(`Skipping add table "${table}" - already exists`); + } + + logging.info(`Adding table: ${table}`); + return commands.createTable(table, connection); + }); + }, + + async down({connection}) { + // table deletion order is important because of foreign key constraints, + // email_recipients references email_batches so it has to be deleted first to not break constraints + return Promise.each(['email_recipients', 'email_batches'], async (table) => { + const tableExists = await connection.schema.hasTable(table); + + if (!tableExists) { + return logging.warn(`Skipping drop table "${table}" - does not exist`); + } + + logging.info(`Dropping table: ${table}`); + return commands.deleteTable(table, connection); + }); + } +}; diff --git a/core/server/data/schema/schema.js b/core/server/data/schema/schema.js index 072cbf362c..a970a5d70a 100644 --- a/core/server/data/schema/schema.js +++ b/core/server/data/schema/schema.js @@ -459,5 +459,29 @@ module.exports = { created_by: {type: 'string', maxlength: 24, nullable: false}, updated_at: {type: 'dateTime', nullable: true}, updated_by: {type: 'string', maxlength: 24, nullable: true} + }, + email_batches: { + id: {type: 'string', maxlength: 24, nullable: false, primary: true}, + email_id: {type: 'string', maxlength: 24, nullable: false, references: 'emails.id'}, + provider_id: {type: 'string', maxlength: 255, nullable: true}, + status: { + type: 'string', + maxlength: 50, + nullable: false, + defaultTo: 'pending', + validations: {isIn: [['pending', 'submitting', 'submitted', 'failed']]} + }, + created_at: {type: 'dateTime', nullable: false}, + updated_at: {type: 'dateTime', nullable: false} + }, + email_recipients: { + id: {type: 'string', maxlength: 24, nullable: false, primary: true}, + email_id: {type: 'string', maxlength: 24, nullable: false, references: 'emails.id'}, + member_id: {type: 'string', maxlength: 24, nullable: false, index: true}, + batch_id: {type: 'string', maxlength: 24, nullable: false, references: 'email_batches.id'}, + processed_at: {type: 'dateTime', nullable: true}, + member_uuid: {type: 'string', maxlength: 36, nullable: false}, + member_email: {type: 'string', maxlength: 191, nullable: false}, + member_name: {type: 'string', maxlength: 191, nullable: true} } }; diff --git a/test/api-acceptance/admin/db_spec.js b/test/api-acceptance/admin/db_spec.js index 5220416c37..8bec2042da 100644 --- a/test/api-acceptance/admin/db_spec.js +++ b/test/api-acceptance/admin/db_spec.js @@ -55,7 +55,7 @@ describe('DB API', function () { const jsonResponse = res.body; should.exist(jsonResponse.db); jsonResponse.db.should.have.length(1); - Object.keys(jsonResponse.db[0].data).length.should.eql(30); + Object.keys(jsonResponse.db[0].data).length.should.eql(32); }); }); diff --git a/test/unit/data/schema/integrity_spec.js b/test/unit/data/schema/integrity_spec.js index 3cc6640bd7..bea1c695b9 100644 --- a/test/unit/data/schema/integrity_spec.js +++ b/test/unit/data/schema/integrity_spec.js @@ -32,7 +32,7 @@ const defaultSettings = require('../../../../core/server/data/schema/default-set */ describe('DB version integrity', function () { // Only these variables should need updating - const currentSchemaHash = '42a966364eb4b5851e807133374821da'; + const currentSchemaHash = 'c2b2de0157edddb68791dde49391d4e5'; const currentFixturesHash = '29148c40dfaf4f828c5fca95666f6545'; const currentSettingsHash = 'c8daa2c9632bb75f9d60655de09ae3bd'; const currentRoutesHash = '3d180d52c663d173a6be791ef411ed01';