Added post analytics to data generator

no issue

ghost/data-generator/lib/data-generator.js

@@ -26,7 +26,11 @@ const {
     MembersSubscriptionCreatedEventsImporter,
     MembersStripeCustomersImporter,
     MembersStripeCustomersSubscriptionsImporter,
-    MembersPaidSubscriptionEventsImporter
+    MembersPaidSubscriptionEventsImporter,
+    EmailBatchesImporter,
+    EmailRecipientsImporter,
+    RedirectsImporter,
+    MembersClickEventsImporter
 } = tables;
 const path = require('path');
 const fs = require('fs/promises');
@@ -261,7 +265,7 @@ class DataGenerator {
         });
 
         const membersImporter = new MembersImporter(transaction);
-        const members = await membersImporter.import({amount: this.modelQuantities.members, rows: ['status', 'created_at', 'name', 'email']});
+        const members = await membersImporter.import({amount: this.modelQuantities.members, rows: ['status', 'created_at', 'name', 'email', 'uuid']});
 
         const postsAuthorsImporter = new PostsAuthorsImporter(transaction, {
             users
@@ -343,7 +347,31 @@ class DataGenerator {
         await mentionsImporter.importForEach(posts, {amount: 4});
 
         const emailsImporter = new EmailsImporter(transaction, {newsletters, members, membersSubscribeEvents});
-        await emailsImporter.importForEach(posts, {amount: 1});
+        const emails = await emailsImporter.importForEach(posts, {
+            amount: 1,
+            rows: ['created_at', 'email_count', 'delivered_count', 'opened_count', 'failed_count', 'newsletter_id', 'post_id']
+        });
+
+        const emailBatchesImporter = new EmailBatchesImporter(transaction);
+        const emailBatches = await emailBatchesImporter.importForEach(emails, {
+            amount: 1,
+            rows: ['email_id', 'updated_at']
+        });
+
+        const emailRecipientsImporter = new EmailRecipientsImporter(transaction, {emailBatches, members, membersSubscribeEvents});
+        const emailRecipients = await emailRecipientsImporter.importForEach(emails, {
+            amount: this.modelQuantities.members,
+            rows: ['opened_at', 'email_id', 'member_id']
+        });
+
+        const redirectsImporter = new RedirectsImporter(transaction);
+        const redirects = await redirectsImporter.importForEach(posts, {
+            amount: 10,
+            rows: ['post_id']
+        });
+
+        const membersClickEventsImporter = new MembersClickEventsImporter(transaction, {redirects, emails});
+        await membersClickEventsImporter.importForEach(emailRecipients, {amount: 2});
 
         // TODO: Email clicks - redirect, members_click_events (relies on emails)
 

ghost/data-generator/lib/tables/email-batches.js

@@ -0,0 +1,32 @@
+const TableImporter = require('./base');
+const {faker} = require('@faker-js/faker');
+const dateToDatabaseString = require('../utils/database-date');
+
+class EmailBatchesImporter extends TableImporter {
+    static table = 'email_batches';
+
+    constructor(knex) {
+        super(EmailBatchesImporter.table, knex);
+    }
+
+    setImportOptions({model}) {
+        this.model = model;
+    }
+
+    generate() {
+        const emailSentDate = new Date(this.model.created_at);
+        const latestUpdatedDate = new Date(this.model.created_at);
+        latestUpdatedDate.setHours(latestUpdatedDate.getHours() + 1);
+
+        return {
+            id: faker.database.mongodbObjectId(),
+            email_id: this.model.id,
+            provider_id: `${new Date().toISOString().split('.')[0].replace(/[^0-9]/g, '')}.${faker.datatype.hexadecimal({length: 16, prefix: '', case: 'lower'})}@m.example.com`,
+            status: 'submitted', // TODO: introduce failures
+            created_at: this.model.created_at,
+            updated_at: dateToDatabaseString(faker.date.between(emailSentDate, latestUpdatedDate))
+        };
+    }
+}
+
+module.exports = EmailBatchesImporter;

ghost/data-generator/lib/tables/email-recipients.js

@@ -0,0 +1,96 @@
+const TableImporter = require('./base');
+const {faker} = require('@faker-js/faker');
+const generateEvents = require('../utils/event-generator');
+const dateToDatabaseString = require('../utils/database-date');
+
+const emailStatus = {
+    delivered: Symbol(),
+    opened: Symbol(),
+    failed: Symbol(),
+    none: Symbol()
+};
+
+class EmailRecipientsImporter extends TableImporter {
+    static table = 'email_recipients';
+
+    constructor(knex, {emailBatches, members, membersSubscribeEvents}) {
+        super(EmailRecipientsImporter.table, knex);
+        this.emailBatches = emailBatches;
+        this.members = members;
+        this.membersSubscribeEvents = membersSubscribeEvents;
+    }
+
+    setImportOptions({model}) {
+        this.model = model;
+        this.batch = this.emailBatches.find(batch => batch.email_id === model.id);
+        // Shallow clone members list so we can shuffle and modify it
+        const earliestOpenTime = new Date(this.batch.updated_at);
+        const latestOpenTime = new Date(this.batch.updated_at);
+        latestOpenTime.setDate(latestOpenTime.getDate() + 14);
+        const currentTime = new Date();
+        this.membersList = this.membersSubscribeEvents
+            .filter(entry => entry.newsletter_id === this.model.newsletter_id)
+            .filter(entry => new Date(entry.created_at) < earliestOpenTime)
+            .map(memberSubscribeEvent => memberSubscribeEvent.member_id);
+        this.events = generateEvents({
+            shape: 'ease-out',
+            trend: 'negative',
+            total: this.membersList.length,
+            startTime: earliestOpenTime,
+            endTime: currentTime < latestOpenTime ? currentTime : latestOpenTime
+        });
+        this.emailMeta = {
+            emailCount: this.model.email_count,
+            deliveredCount: this.model.delivered_count,
+            openedCount: this.model.opened_count,
+            failedCount: this.model.failed_count
+        };
+    }
+
+    generate() {
+        if (this.emailMeta.emailCount <= 0) {
+            return;
+        }
+        this.emailMeta.emailCount -= 1;
+
+        const timestamp = this.events.shift();
+        if (!timestamp) {
+            return;
+        }
+
+        const memberIdIndex = faker.datatype.number({
+            min: 0,
+            max: this.membersList.length - 1
+        });
+        const [memberId] = this.membersList.splice(memberIdIndex, 1);
+        const member = this.members.find(m => m.id === memberId);
+
+        let status = emailStatus.none;
+        if (this.emailMeta.failedCount > 0) {
+            status = emailStatus.failed;
+            this.emailMeta.failedCount -= 1;
+        } else if (this.emailMeta.openedCount > 0) {
+            status = emailStatus.opened;
+            this.emailMeta.deliveredCount -= 1;
+        } else if (this.emailMeta.deliveredCount > 0) {
+            status = emailStatus.delivered;
+            this.emailMeta.deliveredCount -= 1;
+        }
+
+        return {
+            id: faker.database.mongodbObjectId(),
+            email_id: this.model.id,
+            batch_id: this.batch.id,
+            member_id: member.id,
+            processed_at: this.batch.updated_at,
+            delivered_at: status === emailStatus.opened ? dateToDatabaseString(faker.date.between(new Date(this.batch.updated_at), timestamp)) : status === emailStatus.delivered ? dateToDatabaseString(timestamp) : null,
+            opened_at: status === emailStatus.opened ? dateToDatabaseString(timestamp) : null,
+            failed_at: status === emailStatus.failed ? dateToDatabaseString(timestamp) : null,
+            member_uuid: member.uuid,
+            member_email: member.email,
+            member_name: member.name
+        };
+    }
+}
+
+module.exports = EmailRecipientsImporter;

ghost/data-generator/lib/tables/index.js

@@ -25,5 +25,9 @@ module.exports = {
     MembersStripeCustomersSubscriptionsImporter: require('./members-stripe-customers-subscriptions'),
     MembersPaidSubscriptionEventsImporter: require('./members-paid-subscription-events'),
     MembersSubscriptionCreatedEventsImporter: require('./members-subscription-created-events'),
-    MembersSubscribeEventsImporter: require('./members-subscribe-events')
+    MembersSubscribeEventsImporter: require('./members-subscribe-events'),
+    EmailBatchesImporter: require('./email-batches'),
+    EmailRecipientsImporter: require('./email-recipients'),
+    RedirectsImporter: require('./redirects'),
+    MembersClickEventsImporter: require('./members-click-events')
 };

ghost/data-generator/lib/tables/members-click-events.js

@@ -0,0 +1,48 @@
+const TableImporter = require('./base');
+const {faker} = require('@faker-js/faker');
+const {luck} = require('../utils/random');
+const dateToDatabaseString = require('../utils/database-date');
+
+class MembersClickEventsImporter extends TableImporter {
+    static table = 'members_click_events';
+
+    constructor(knex, {redirects, emails}) {
+        super(MembersClickEventsImporter.table, knex);
+
+        this.redirects = redirects;
+        this.emails = emails;
+    }
+
+    setImportOptions({model, amount}) {
+        this.model = model;
+        this.amount = model.opened_at === null ? 0 : luck(40) ? faker.datatype.number({
+            min: 0,
+            max: amount
+        }) : 0;
+        const email = this.emails.find(e => e.id === this.model.email_id);
+        this.redirectList = this.redirects.filter(redirect => redirect.post_id === email.post_id);
+    }
+
+    generate() {
+        if (this.amount <= 0) {
+            return;
+        }
+        this.amount -= 1;
+
+        const openedAt = new Date(this.model.opened_at);
+        const laterOn = new Date(this.model.opened_at);
+        laterOn.setMinutes(laterOn.getMinutes() + 15);
+
+        return {
+            id: faker.database.mongodbObjectId(),
+            member_id: this.model.member_id,
+            redirect_id: this.redirectList[faker.datatype.number({
+                min: 0,
+                max: this.redirectList.length - 1
+            })].id,
+            created_at: dateToDatabaseString(faker.date.between(openedAt, laterOn))
+        };
+    }
+}
+
+module.exports = MembersClickEventsImporter;

ghost/data-generator/lib/tables/redirects.js

@@ -0,0 +1,35 @@
+const TableImporter = require('./base');
+const {faker} = require('@faker-js/faker');
+
+class RedirectsImporter extends TableImporter {
+    static table = 'redirects';
+
+    constructor(knex) {
+        super(RedirectsImporter.table, knex);
+    }
+
+    setImportOptions({model, amount}) {
+        this.model = model;
+        this.amount = faker.datatype.number({
+            min: 1,
+            max: amount
+        });
+    }
+
+    generate() {
+        if (this.amount <= 0) {
+            return;
+        }
+        this.amount -= 1;
+        return {
+            id: faker.database.mongodbObjectId(),
+            from: `/r/${faker.datatype.hexadecimal({length: 32, prefix: '', case: 'lower'})}`,
+            to: `https://${faker.internet.url()}/${faker.helpers.slugify(`${faker.word.adjective()} ${faker.word.noun()}`).toLowerCase()}`,
+            post_id: this.model.id,
+            created_at: this.model.published_at,
+            updated_at: this.model.published_at
+        };
+    }
+}
+
+module.exports = RedirectsImporter;