Added schema+migration for email_{batches,recipients} tables (#12192)

no issue

We want to store a list of recipients for each bulk email so that we have a consistent set of data that background processing/sending jobs can work from without worrying about moving large data sets around or member data changing mid-send.

- `email_batches` table acts as a join table with status for email<->email_recipient
  - stores a provider-specific ID that we get back when submitting a batch for sending to the bulk email provider
  - `status` allows for batch-specific status updates and picking up where we left off when submitting batches if needed
  - explicitly tying a list of email recipients to a batch allows for partial retries
- `email_recipients` table acts as a join table for email<->member
  - `member_id` does not have a foreign key constraint because members can be deleted but does have an index so that we can efficiently query which emails a member has received
  - stores static copies of the member info present at the time of sending an email for consistency in background jobs and auditing/historical data

core/server/data/migrations/versions/3.33/01-add-email-recipients-tables.js

@@ -0,0 +1,34 @@
+const logging = require('../../../../../shared/logging');
+const commands = require('../../../schema').commands;
+
+module.exports = {
+    async up({connection}) {
+        // table creation order is important because of foreign key constraints,
+        // email_recipients references email_batches so email_batches has to exist when creating
+        return Promise.each(['email_batches', 'email_recipients'], async (table) => {
+            const tableExists = await connection.schema.hasTable(table);
+
+            if (tableExists) {
+                return logging.warn(`Skipping add table "${table}" - already exists`);
+            }
+
+            logging.info(`Adding table: ${table}`);
+            return commands.createTable(table, connection);
+        });
+    },
+
+    async down({connection}) {
+        // table deletion order is important because of foreign key constraints,
+        // email_recipients references email_batches so it has to be deleted first to not break constraints
+        return Promise.each(['email_recipients', 'email_batches'], async (table) => {
+            const tableExists = await connection.schema.hasTable(table);
+
+            if (!tableExists) {
+                return logging.warn(`Skipping drop table "${table}" - does not exist`);
+            }
+
+            logging.info(`Dropping table: ${table}`);
+            return commands.deleteTable(table, connection);
+        });
+    }
+};

core/server/data/schema/schema.js

@@ -459,5 +459,29 @@ module.exports = {
         created_by: {type: 'string', maxlength: 24, nullable: false},
         updated_at: {type: 'dateTime', nullable: true},
         updated_by: {type: 'string', maxlength: 24, nullable: true}
+    },
+    email_batches: {
+        id: {type: 'string', maxlength: 24, nullable: false, primary: true},
+        email_id: {type: 'string', maxlength: 24, nullable: false, references: 'emails.id'},
+        provider_id: {type: 'string', maxlength: 255, nullable: true},
+        status: {
+            type: 'string',
+            maxlength: 50,
+            nullable: false,
+            defaultTo: 'pending',
+            validations: {isIn: [['pending', 'submitting', 'submitted', 'failed']]}
+        },
+        created_at: {type: 'dateTime', nullable: false},
+        updated_at: {type: 'dateTime', nullable: false}
+    },
+    email_recipients: {
+        id: {type: 'string', maxlength: 24, nullable: false, primary: true},
+        email_id: {type: 'string', maxlength: 24, nullable: false, references: 'emails.id'},
+        member_id: {type: 'string', maxlength: 24, nullable: false, index: true},
+        batch_id: {type: 'string', maxlength: 24, nullable: false, references: 'email_batches.id'},
+        processed_at: {type: 'dateTime', nullable: true},
+        member_uuid: {type: 'string', maxlength: 36, nullable: false},
+        member_email: {type: 'string', maxlength: 191, nullable: false},
+        member_name: {type: 'string', maxlength: 191, nullable: true}
     }
 };

test/api-acceptance/admin/db_spec.js

@@ -55,7 +55,7 @@ describe('DB API', function () {
                 const jsonResponse = res.body;
                 should.exist(jsonResponse.db);
                 jsonResponse.db.should.have.length(1);
-                Object.keys(jsonResponse.db[0].data).length.should.eql(30);
+                Object.keys(jsonResponse.db[0].data).length.should.eql(32);
             });
     });
 

test/unit/data/schema/integrity_spec.js

@@ -32,7 +32,7 @@ const defaultSettings = require('../../../../core/server/data/schema/default-set
  */
 describe('DB version integrity', function () {
     // Only these variables should need updating
-    const currentSchemaHash = '42a966364eb4b5851e807133374821da';
+    const currentSchemaHash = 'c2b2de0157edddb68791dde49391d4e5';
     const currentFixturesHash = '29148c40dfaf4f828c5fca95666f6545';
     const currentSettingsHash = 'c8daa2c9632bb75f9d60655de09ae3bd';
     const currentRoutesHash = '3d180d52c663d173a6be791ef411ed01';