Infrastructure/ghost
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2025-02-10 23:36:14 -05:00
Code Issues Activity

Improved error handling for batch inserted member records (#12146)

Browse source 
no issue

- When batch insert fails handling should be more granular and aim to retry and insert as many records from the batch as possible.  
- Added retry logic for failed member's batch inserts. It's a sequential insert for each record in the batch. This implementation was chosen to keep it as simple as possible
- Added filtering of "toCreate" records when member fails to insert. We should not try inserting related members_labels/members_stripe_customers/members_stripe_customer_subscriptions records because they would definitely fail insertion without associated member record
This commit is contained in:
naz 2020-08-20 17:41:47 +12:00
parent 0756d26d22
commit 29f9d60d3b
1 changed files with 26 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
ghost/members-importer/lib/index.js
View file

@ -16,7 +16,7 @@ const doImport = async ({members, allLabelModels, importSetLabels, createdBy}) =
const deleteMembers = createDeleter('members');
const insertLabelAssociations = createInserter('members_labels');
const {
let {
invalidMembers,
membersToInsert,
stripeCustomersToFetch,
@ -24,21 +24,36 @@ const doImport = async ({members, allLabelModels, importSetLabels, createdBy}) =
labelAssociationsToInsert
} = getMemberData({members, allLabelModels, importSetLabels, createdBy});
// NOTE: member insertion has to happen before the rest of insertions to handle validation
// errors - remove failed members from label/stripe sets
const insertedMembers = await models.Member.bulkAdd(membersToInsert).then((insertResult) => {
if (insertResult.unsuccessfulIds.length) {
labelAssociationsToInsert = labelAssociationsToInsert
.filter(la => !insertResult.unsuccessfulIds.includes(la.member_id));
stripeCustomersToFetch = stripeCustomersToFetch
.filter(sc => !insertResult.unsuccessfulIds.includes(sc.member_id));
stripeCustomersToCreate = stripeCustomersToCreate
.filter(sc => !insertResult.unsuccessfulIds.includes(sc.member_id));
}
return insertResult;
});
const fetchedStripeCustomersPromise = fetchStripeCustomers(stripeCustomersToFetch);
const createdStripeCustomersPromise = createStripeCustomers(stripeCustomersToCreate);
const insertedMembersPromise = models.Member.bulkAdd(membersToInsert);
const insertedLabelsPromise = insertedMembersPromise
.then(() => insertLabelAssociations(labelAssociationsToInsert));
const insertedLabelsPromise = insertLabelAssociations(labelAssociationsToInsert);
const insertedCustomersPromise = Promise.all([
fetchedStripeCustomersPromise,
createdStripeCustomersPromise,
insertedMembersPromise
createdStripeCustomersPromise
]).then(
([fetchedStripeCustomers, createdStripeCustomers]) => models.MemberStripeCustomer.bulkAdd(
([fetchedStripeCustomers, createdStripeCustomers]) => {
return models.MemberStripeCustomer.bulkAdd(
fetchedStripeCustomers.customersToInsert.concat(createdStripeCustomers.customersToInsert)
)
);
}
);
const insertedSubscriptionsPromise = Promise.all([
@ -53,8 +68,7 @@ const doImport = async ({members, allLabelModels, importSetLabels, createdBy}) =
const deletedMembersPromise = Promise.all([
fetchedStripeCustomersPromise,
createdStripeCustomersPromise,
insertedMembersPromise
createdStripeCustomersPromise
]).then(
([fetchedStripeCustomers, createdStripeCustomers]) => deleteMembers(
fetchedStripeCustomers.membersToDelete.concat(createdStripeCustomers.membersToDelete)
@ -64,7 +78,6 @@ const doImport = async ({members, allLabelModels, importSetLabels, createdBy}) =
// This looks sequential, but at the point insertedCustomersPromise has resolved so have all the others
const insertedSubscriptions = await insertedSubscriptionsPromise;
const insertedCustomers = await insertedCustomersPromise;
const insertedMembers = await insertedMembersPromise;
const deletedMembers = await deletedMembersPromise;
const fetchedCustomers = await fetchedStripeCustomersPromise;
const insertedLabels = await insertedLabelsPromise;