mirror of
https://github.com/TryGhost/Ghost.git
synced 2025-02-10 23:36:14 -05:00
🎨 Optimised SQL query for exporting members (#22017)
ref https://linear.app/ghost/issue/ONC-699/lever-member-export-unresponsive - Split large SQL queries into smaller, focused queries to improve performance and reduce database load. - Shifted aggregation logic from database to in-memory processing for improved query efficiency and faster execution. - Added temp logging to identify performance bottlenecks and measure execution time for each step in production environment as things are pretty fast in local setup and staging. - No updates in the test, this API already has snapshot tests and unit tests
This commit is contained in:
Princi Vershwal
GitHub
parent
2b335e8c37
commit
a983bf0791
1 changed files with 102 additions and 40 deletions
|
|
@ -1,9 +1,11 @@
|
|||
const models = require('../../../models');
|
||||
const {knex} = require('../../../data/db');
|
||||
const moment = require('moment');
|
||||
const logging = require('@tryghost/logging');
|
||||
|
||||
module.exports = async function (options) {
|
||||
const hasFilter = options.limit !== 'all' || options.filter || options.search;
|
||||
const start = Date.now();
|
||||
|
||||
let ids = null;
|
||||
if (hasFilter) {
|
||||
|
|
@ -31,58 +33,118 @@ module.exports = async function (options) {
|
|||
*/
|
||||
}
|
||||
|
||||
const allProducts = await models.Product.fetchAll();
|
||||
const allLabels = await models.Label.fetchAll();
|
||||
const startFetchingProducts = Date.now();
|
||||
|
||||
let query = knex('members')
|
||||
const allProducts = await knex('products').select('id', 'name').then(rows => rows.reduce((acc, product) => {
|
||||
acc[product.id] = product.name;
|
||||
return acc;
|
||||
}, {})
|
||||
);
|
||||
|
||||
const allLabels = await knex('labels').select('id', 'name').then(rows => rows.reduce((acc, label) => {
|
||||
acc[label.id] = label.name;
|
||||
return acc;
|
||||
}, {})
|
||||
);
|
||||
|
||||
logging.info('[MembersExporter] Fetched products and labels in ' + (Date.now() - startFetchingProducts) + 'ms');
|
||||
|
||||
const startFetchingMembers = Date.now();
|
||||
|
||||
const members = await knex('members')
|
||||
.select('id', 'email', 'name', 'note', 'status', 'created_at')
|
||||
.select(knex.raw(`
|
||||
(CASE WHEN EXISTS (SELECT 1 FROM members_newsletters n WHERE n.member_id = members.id)
|
||||
THEN TRUE ELSE FALSE
|
||||
END) as subscribed
|
||||
`))
|
||||
.select(knex.raw(`
|
||||
(SELECT GROUP_CONCAT(product_id) FROM members_products f WHERE f.member_id = members.id) as tiers
|
||||
`))
|
||||
.select(knex.raw(`
|
||||
(SELECT GROUP_CONCAT(label_id) FROM members_labels f WHERE f.member_id = members.id) as labels
|
||||
`))
|
||||
.select(knex.raw(`
|
||||
(SELECT customer_id FROM members_stripe_customers f WHERE f.member_id = members.id limit 1) as stripe_customer_id
|
||||
`));
|
||||
.modify((query) => {
|
||||
if (hasFilter) {
|
||||
query.whereIn('id', ids);
|
||||
}
|
||||
});
|
||||
|
||||
if (hasFilter) {
|
||||
query = query.whereIn('id', ids);
|
||||
}
|
||||
logging.info('[MembersExporter] Fetched members in ' + (Date.now() - startFetchingMembers) + 'ms');
|
||||
|
||||
const rows = await query;
|
||||
for (const row of rows) {
|
||||
const tierIds = row.tiers ? row.tiers.split(',') : [];
|
||||
const tiers = tierIds.map((id) => {
|
||||
const tier = allProducts.find(p => p.id === id);
|
||||
const startFetchingTiers = Date.now();
|
||||
|
||||
const tiers = await knex('members_products')
|
||||
.select('member_id', knex.raw('GROUP_CONCAT(product_id) as tiers'))
|
||||
.groupBy('member_id')
|
||||
.modify((query) => {
|
||||
if (hasFilter) {
|
||||
query.whereIn('member_id', ids);
|
||||
}
|
||||
});
|
||||
|
||||
logging.info('[MembersExporter] Fetched tiers in ' + (Date.now() - startFetchingTiers) + 'ms');
|
||||
|
||||
const startFetchingLabels = Date.now();
|
||||
|
||||
const labels = await knex('members_labels')
|
||||
.select('member_id', knex.raw('GROUP_CONCAT(label_id) as labels'))
|
||||
.groupBy('member_id')
|
||||
.modify((query) => {
|
||||
if (hasFilter) {
|
||||
query.whereIn('member_id', ids);
|
||||
}
|
||||
});
|
||||
|
||||
logging.info('[MembersExporter] Fetched labels in ' + (Date.now() - startFetchingLabels) + 'ms');
|
||||
|
||||
const startFetchingStripeCustomers = Date.now();
|
||||
|
||||
const stripeCustomers = await knex('members_stripe_customers')
|
||||
.select('member_id', knex.raw('MIN(customer_id) as stripe_customer_id'))
|
||||
.groupBy('member_id')
|
||||
.modify((query) => {
|
||||
if (hasFilter) {
|
||||
query.whereIn('member_id', ids);
|
||||
}
|
||||
});
|
||||
|
||||
logging.info('[MembersExporter] Fetched stripe customers in ' + (Date.now() - startFetchingStripeCustomers) + 'ms');
|
||||
|
||||
const startFetchingSubscriptions = Date.now();
|
||||
|
||||
const subscriptions = await knex('members_newsletters')
|
||||
.distinct('member_id')
|
||||
.modify((query) => {
|
||||
if (hasFilter) {
|
||||
query.whereIn('member_id', ids);
|
||||
}
|
||||
});
|
||||
|
||||
logging.info('[MembersExporter] Fetched subscriptions in ' + (Date.now() - startFetchingSubscriptions) + 'ms');
|
||||
|
||||
const startInMemoryProcessing = Date.now();
|
||||
|
||||
const tiersMap = new Map(tiers.map(row => [row.member_id, row.tiers]));
|
||||
const labelsMap = new Map(labels.map(row => [row.member_id, row.labels]));
|
||||
const stripeCustomerMap = new Map(stripeCustomers.map(row => [row.member_id, row.stripe_customer_id]));
|
||||
const subscribedSet = new Set(subscriptions.map(row => row.member_id));
|
||||
|
||||
for (const row of members) {
|
||||
const tierIds = tiersMap.get(row.id) ? tiersMap.get(row.id).split(',') : [];
|
||||
const tierDetails = tierIds.map((id) => {
|
||||
return {
|
||||
name: tier.get('name')
|
||||
name: allProducts[id]
|
||||
};
|
||||
});
|
||||
row.tiers = tiers;
|
||||
row.tiers = tierDetails;
|
||||
|
||||
const labelIds = row.labels ? row.labels.split(',') : [];
|
||||
const labels = labelIds.map((id) => {
|
||||
const label = allLabels.find(l => l.id === id);
|
||||
const labelIds = labelsMap.get(row.id) ? labelsMap.get(row.id).split(',') : [];
|
||||
const labelDetails = labelIds.map((id) => {
|
||||
return {
|
||||
name: label.get('name')
|
||||
name: allLabels[id]
|
||||
};
|
||||
});
|
||||
row.labels = labels;
|
||||
row.labels = labelDetails;
|
||||
|
||||
row.subscribed = subscribedSet.has(row.id);
|
||||
row.comped = row.status === 'comped';
|
||||
row.stripe_customer_id = stripeCustomerMap.get(row.id) || null;
|
||||
row.created_at = moment(row.created_at).toISOString();
|
||||
}
|
||||
|
||||
for (const member of rows) {
|
||||
// Note: we don't modify the array or change/duplicate objects
|
||||
// to increase performance
|
||||
member.subscribed = !!member.subscribed;
|
||||
member.comped = member.status === 'comped';
|
||||
member.created_at = moment(member.created_at).toISOString();
|
||||
}
|
||||
logging.info('[MembersExporter] In memory processing finished in ' + (Date.now() - startInMemoryProcessing) + 'ms');
|
||||
|
||||
return rows;
|
||||
logging.info('[MembersExporter] Total time taken for member export: ' + (Date.now() - start) / 1000 + 's');
|
||||
|
||||
return members;
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue