From 567eb6325f6b22cc257d5f856556b48004775677 Mon Sep 17 00:00:00 2001
From: Kevin Ansfield <kevin@ghost.org>
Date: Tue, 8 Dec 2020 12:43:10 +0000
Subject: [PATCH] Added members.email_open_rate aggregation to email analytics
 (#12458)

refs https://github.com/TryGhost/Ghost/issues/12421
requires https://github.com/TryGhost/Ghost/pull/12457

- updates stats aggregator to calculate and store an open rate for each member
  - uses two queries because I couldn't find a reasonable approach to perform the update in a single query as per the email aggregation
  - benchmarked locally at <1sec/1000members
  - will not store an open rate unless the number of tracked emails sent to a member is above a certain threshold (defaults to 5) to avoid new members being heavily weighted
- fixes typo in EmailAnalytics that was stopping member stats from being aggregated
---
 .../email-analytics/email-analytics.js        |  2 +-
 .../email-analytics/jobs/fetch-all.js         |  2 +-
 .../email-analytics/jobs/fetch-latest.js      |  2 +-
 .../email-analytics/lib/stats-aggregator.js   | 22 ++++++++++++++++---
 4 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/core/server/services/email-analytics/email-analytics.js b/core/server/services/email-analytics/email-analytics.js
index 5fed5351d1..666f3d122c 100644
--- a/core/server/services/email-analytics/email-analytics.js
+++ b/core/server/services/email-analytics/email-analytics.js
@@ -81,7 +81,7 @@ class EmailAnalyticsService {
             await this.aggregateEmailStats(emailId);
         }
         for (const memberId of memberIds) {
-            await this.aggregateEmailStats(memberId);
+            await this.aggregateMemberStats(memberId);
         }
     }
 
diff --git a/core/server/services/email-analytics/jobs/fetch-all.js b/core/server/services/email-analytics/jobs/fetch-all.js
index 6ad4dde640..55e6959757 100644
--- a/core/server/services/email-analytics/jobs/fetch-all.js
+++ b/core/server/services/email-analytics/jobs/fetch-all.js
@@ -50,7 +50,7 @@ if (parentPort) {
         const aggregateEndDate = new Date();
         debug(`Finished aggregating email analytics in ${aggregateEndDate - aggregateStartDate}ms`);
 
-        logging.info(`Fetched ${eventStats.totalEvents} events and aggregated stats for ${eventStats.emailIds.length} emails in ${aggregateEndDate - fetchStartDate}ms`);
+        logging.info(`Fetched ${eventStats.totalEvents} events and aggregated stats for ${eventStats.emailIds.length} emails and ${eventStats.memberIds.length} members in ${aggregateEndDate - fetchStartDate}ms`);
 
         if (parentPort) {
             parentPort.postMessage('done');
diff --git a/core/server/services/email-analytics/jobs/fetch-latest.js b/core/server/services/email-analytics/jobs/fetch-latest.js
index 1216197910..13642eef64 100644
--- a/core/server/services/email-analytics/jobs/fetch-latest.js
+++ b/core/server/services/email-analytics/jobs/fetch-latest.js
@@ -51,7 +51,7 @@ if (parentPort) {
         const aggregateEndDate = new Date();
         debug(`Finished aggregating email analytics in ${aggregateEndDate - aggregateStartDate}ms`);
 
-        logging.info(`Fetched ${eventStats.totalEvents} events and aggregated stats for ${eventStats.emailIds.length} emails in ${aggregateEndDate - fetchStartDate}ms`);
+        logging.info(`Fetched ${eventStats.totalEvents} events and aggregated stats for ${eventStats.emailIds.length} emails and ${eventStats.memberIds.length} members in ${aggregateEndDate - fetchStartDate}ms`);
 
         if (parentPort) {
             parentPort.postMessage('done');
diff --git a/core/server/services/email-analytics/lib/stats-aggregator.js b/core/server/services/email-analytics/lib/stats-aggregator.js
index 4a0d4355f7..46dd62b518 100644
--- a/core/server/services/email-analytics/lib/stats-aggregator.js
+++ b/core/server/services/email-analytics/lib/stats-aggregator.js
@@ -1,5 +1,6 @@
 class EmailAnalyticsStatsAggregator {
-    constructor({logging, db}) {
+    constructor({options, logging, db}) {
+        this.options = Object.assign({openRateEmailThreshold: 5}, options);
         this.logging = logging || console;
         this.db = db;
     }
@@ -12,8 +13,23 @@ class EmailAnalyticsStatsAggregator {
         }).where('id', emailId);
     }
 
-    async aggregateMember(/*memberId*/) {
-        // TODO: decide on aggregation algorithm when only certain emails have open tracking
+    async aggregateMember(memberId) {
+        const {trackedEmailCount} = await this.db.knex('email_recipients')
+            .select(this.db.knex.raw('COUNT(email_recipients.id) as trackedEmailCount'))
+            .leftJoin('emails', 'email_recipients.email_id', 'emails.id')
+            .where('email_recipients.member_id', memberId)
+            .where('emails.track_opens', true)
+            .first() || {};
+
+        if (trackedEmailCount >= this.options.openRateEmailThreshold) {
+            await this.db.knex('members')
+                .update({
+                    email_open_rate: this.db.knex.raw(`(
+                        (SELECT COUNT(id) FROM email_recipients WHERE member_id = ? AND opened_at IS NOT NULL) * 1.0 / ? * 100)
+                    `, [memberId, trackedEmailCount])
+                })
+                .where('id', memberId);
+        }
     }
 }