🎨 Optimised SQL query for exporting members (#22017)

ref
https://linear.app/ghost/issue/ONC-699/lever-member-export-unresponsive

- Split large SQL queries into smaller, focused queries to improve
performance and reduce database load.
- Shifted aggregation logic from database to in-memory processing for
improved query efficiency and faster execution.
- Added temp logging to identify performance bottlenecks and measure
execution time for each step in production environment as things are
pretty fast in local setup and staging.
- No updates in the test, this API already has snapshot tests and unit tests

ghost/core/core/server/services/members/exporter/query.js

@@ -1,9 +1,11 @@
 const models = require('../../../models');
 const {knex} = require('../../../data/db');
 const moment = require('moment');
+const logging = require('@tryghost/logging');
 
 module.exports = async function (options) {
     const hasFilter = options.limit !== 'all' || options.filter || options.search;
+    const start = Date.now();
 
     let ids = null;
     if (hasFilter) {
@@ -31,58 +33,118 @@ module.exports = async function (options) {
         */
     }
 
-    const allProducts = await models.Product.fetchAll();
+    const startFetchingProducts = Date.now();
-    const allLabels = await models.Label.fetchAll();
 
-    let query = knex('members')
+    const allProducts = await knex('products').select('id', 'name').then(rows => rows.reduce((acc, product) => {
+        acc[product.id] = product.name;
+        return acc; 
+    }, {})
+    );
+
+    const allLabels = await knex('labels').select('id', 'name').then(rows => rows.reduce((acc, label) => {
+        acc[label.id] = label.name;
+        return acc;
+    }, {})
+    );
+
+    logging.info('[MembersExporter] Fetched products and labels in ' + (Date.now() - startFetchingProducts) + 'ms');
+
+    const startFetchingMembers = Date.now();
+
+    const members = await knex('members')
         .select('id', 'email', 'name', 'note', 'status', 'created_at')
-        .select(knex.raw(`
+        .modify((query) => {
-            (CASE WHEN EXISTS (SELECT 1 FROM members_newsletters n WHERE n.member_id = members.id)
-                    THEN TRUE ELSE FALSE
-            END) as subscribed
-        `))
-        .select(knex.raw(`
-            (SELECT GROUP_CONCAT(product_id) FROM members_products f WHERE f.member_id = members.id) as tiers
-        `))
-        .select(knex.raw(`
-            (SELECT GROUP_CONCAT(label_id) FROM members_labels f WHERE f.member_id = members.id) as labels
-        `))
-        .select(knex.raw(`
-            (SELECT customer_id FROM members_stripe_customers f WHERE f.member_id = members.id limit 1) as stripe_customer_id
-        `));
-
             if (hasFilter) {
-        query = query.whereIn('id', ids);
+                query.whereIn('id', ids);
             }
+        });
 
-    const rows = await query;
+    logging.info('[MembersExporter] Fetched members in ' + (Date.now() - startFetchingMembers) + 'ms');
-    for (const row of rows) {
+
-        const tierIds = row.tiers ? row.tiers.split(',') : [];
+    const startFetchingTiers = Date.now();
-        const tiers = tierIds.map((id) => {
+
-            const tier = allProducts.find(p => p.id === id);
+    const tiers = await knex('members_products')
+        .select('member_id', knex.raw('GROUP_CONCAT(product_id) as tiers'))
+        .groupBy('member_id')
+        .modify((query) => {
+            if (hasFilter) {
+                query.whereIn('member_id', ids);
+            }
+        });
+
+    logging.info('[MembersExporter] Fetched tiers in ' + (Date.now() - startFetchingTiers) + 'ms');
+
+    const startFetchingLabels = Date.now();
+
+    const labels = await knex('members_labels')
+        .select('member_id', knex.raw('GROUP_CONCAT(label_id) as labels'))
+        .groupBy('member_id')
+        .modify((query) => {
+            if (hasFilter) {
+                query.whereIn('member_id', ids);
+            }
+        });
+
+    logging.info('[MembersExporter] Fetched labels in ' + (Date.now() - startFetchingLabels) + 'ms');
+
+    const startFetchingStripeCustomers = Date.now();
+
+    const stripeCustomers = await knex('members_stripe_customers')
+        .select('member_id', knex.raw('MIN(customer_id) as stripe_customer_id'))
+        .groupBy('member_id')
+        .modify((query) => {
+            if (hasFilter) {
+                query.whereIn('member_id', ids);
+            }
+        });
+
+    logging.info('[MembersExporter] Fetched stripe customers in ' + (Date.now() - startFetchingStripeCustomers) + 'ms');
+
+    const startFetchingSubscriptions = Date.now();
+
+    const subscriptions = await knex('members_newsletters')
+        .distinct('member_id')
+        .modify((query) => {
+            if (hasFilter) {
+                query.whereIn('member_id', ids);
+            }
+        });
+
+    logging.info('[MembersExporter] Fetched subscriptions in ' + (Date.now() - startFetchingSubscriptions) + 'ms');
+
+    const startInMemoryProcessing = Date.now();
+
+    const tiersMap = new Map(tiers.map(row => [row.member_id, row.tiers]));
+    const labelsMap = new Map(labels.map(row => [row.member_id, row.labels]));
+    const stripeCustomerMap = new Map(stripeCustomers.map(row => [row.member_id, row.stripe_customer_id]));
+    const subscribedSet = new Set(subscriptions.map(row => row.member_id));
+
+    for (const row of members) {
+        const tierIds = tiersMap.get(row.id) ? tiersMap.get(row.id).split(',') : [];
+        const tierDetails = tierIds.map((id) => {
             return {
-                name: tier.get('name')
+                name: allProducts[id]
             };
         });
-        row.tiers = tiers;
+        row.tiers = tierDetails;
 
-        const labelIds = row.labels ? row.labels.split(',') : [];
+        const labelIds = labelsMap.get(row.id) ? labelsMap.get(row.id).split(',') : [];
-        const labels = labelIds.map((id) => {
+        const labelDetails = labelIds.map((id) => {
-            const label = allLabels.find(l => l.id === id);
             return {
-                name: label.get('name')
+                name: allLabels[id]
             };
         });
-        row.labels = labels;
+        row.labels = labelDetails;
+
+        row.subscribed = subscribedSet.has(row.id);
+        row.comped = row.status === 'comped';
+        row.stripe_customer_id = stripeCustomerMap.get(row.id) || null;
+        row.created_at = moment(row.created_at).toISOString();
     }
 
-    for (const member of rows) {
+    logging.info('[MembersExporter] In memory processing finished in ' + (Date.now() - startInMemoryProcessing) + 'ms');
-        // Note: we don't modify the array or change/duplicate objects
-        // to increase performance
-        member.subscribed = !!member.subscribed;
-        member.comped = member.status === 'comped';
-        member.created_at = moment(member.created_at).toISOString();
-    }
 
-    return rows;
+    logging.info('[MembersExporter] Total time taken for member export: ' + (Date.now() - start) / 1000 + 's');
+
+    return members;
 };