0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2025-04-08 02:52:39 -05:00

Updated canary members controller to use new Importer

no-issue

This completely replaces the old import functionality with the new
importer!
This commit is contained in:
Fabien O'Carroll 2020-12-09 16:15:53 +00:00 committed by Fabien 'egg' O'Carroll
parent ff12d7a89a
commit 32fe260763
5 changed files with 92 additions and 321 deletions

View file

@ -3,134 +3,16 @@
const Promise = require('bluebird');
const moment = require('moment-timezone');
const errors = require('@tryghost/errors');
const GhostMailer = require('../../services/mail').GhostMailer;
const config = require('../../../shared/config');
const models = require('../../models');
const membersService = require('../../services/members');
const jobsService = require('../../services/jobs');
const settingsCache = require('../../services/settings/cache');
const {i18n} = require('../../lib/common');
const logging = require('../../../shared/logging');
const db = require('../../data/db');
const _ = require('lodash');
/** NOTE: this method should not exist at all and needs to be cleaned up
it was created due to a bug in how CSV is currently created for exports
Export bug was fixed in 3.6 but method exists to handle older csv exports with undefined
**/
const cleanupUndefined = (obj) => {
for (let key in obj) {
if (obj[key] === 'undefined') {
delete obj[key];
}
}
};
const sanitizeInput = async (members) => {
const validationErrors = [];
let invalidCount = 0;
const jsonSchema = require('./utils/validators/utils/json-schema');
let invalidValidationCount = 0;
try {
await jsonSchema.validate({
docName: 'members',
method: 'upload'
}, {
data: members
});
} catch (error) {
if (error.errorDetails && error.errorDetails.length) {
const jsonPointerIndexRegex = /\[(?<index>\d+)\]/;
let invalidRecordIndexes = error.errorDetails.map((errorDetail) => {
if (errorDetail.dataPath) {
const key = errorDetail.dataPath.split('.').pop();
const [, index] = errorDetail.dataPath.match(jsonPointerIndexRegex);
validationErrors.push(new errors.ValidationError({
message: i18n.t('notices.data.validation.index.schemaValidationFailed', {
key
}),
context: `${key} ${errorDetail.message}`,
errorDetails: `${errorDetail.dataPath} with value ${members[index][key]}`
}));
return Number(index);
}
});
invalidRecordIndexes = _.uniq(invalidRecordIndexes);
invalidRecordIndexes = invalidRecordIndexes.filter(index => (index !== undefined));
invalidRecordIndexes.forEach((index) => {
members[index] = undefined;
});
members = members.filter(record => (record !== undefined));
invalidValidationCount += invalidRecordIndexes.length;
}
}
invalidCount += invalidValidationCount;
const stripeIsConnected = membersService.config.isStripeConnected();
const hasStripeConnectedMembers = members.find(member => (member.stripe_customer_id || member.comped));
if (!stripeIsConnected && hasStripeConnectedMembers) {
let nonFilteredMembersCount = members.length;
members = members.filter(member => !(member.stripe_customer_id || member.comped));
const stripeConnectedMembers = (nonFilteredMembersCount - members.length);
if (stripeConnectedMembers) {
invalidCount += stripeConnectedMembers;
validationErrors.push(new errors.ValidationError({
message: i18n.t('errors.api.members.stripeNotConnected.message'),
context: i18n.t('errors.api.members.stripeNotConnected.context'),
help: i18n.t('errors.api.members.stripeNotConnected.help')
}));
}
}
const customersMap = members.reduce((acc, member) => {
if (member.stripe_customer_id && member.stripe_customer_id !== 'undefined') {
if (acc[member.stripe_customer_id]) {
acc[member.stripe_customer_id] += 1;
} else {
acc[member.stripe_customer_id] = 1;
}
}
return acc;
}, {});
const toRemove = [];
for (const key in customersMap) {
if (customersMap[key] > 1) {
toRemove.push(key);
}
}
let sanitized = members.filter((member) => {
return !(toRemove.includes(member.stripe_customer_id));
});
const duplicateStripeCustomersCount = (members.length - sanitized.length);
if (duplicateStripeCustomersCount) {
validationErrors.push(new errors.ValidationError({
message: i18n.t('errors.api.members.duplicateStripeCustomerIds.message'),
context: i18n.t('errors.api.members.duplicateStripeCustomerIds.context'),
help: i18n.t('errors.api.members.duplicateStripeCustomerIds.help')
}));
}
invalidCount += duplicateStripeCustomersCount;
return {
sanitized,
invalidCount,
validationErrors,
duplicateStripeCustomersCount
};
};
const ghostMailer = new GhostMailer();
module.exports = {
docName: 'members',
@ -430,128 +312,69 @@ module.exports = {
},
importCSV: {
statusCode: 201,
statusCode(result) {
if (result && result.meta && result.meta.stats && result.meta.stats.imported !== null) {
return 201;
} else {
return 202;
}
},
permissions: {
method: 'add'
},
async query(frame) {
let imported = {
count: 0
const siteTimezone = settingsCache.get('timezone');
const importLabel = {
name: `Import ${moment().tz(siteTimezone).format('YYYY-MM-DD HH:mm')}`
};
let invalid = {
count: 0,
errors: []
};
let duplicateStripeCustomerIdCount = 0;
let {importSetLabels, importLabel} = await memberLabelsImporter.handleAllLabels(
frame.data.labels,
frame.data.members,
settingsCache.get('timezone'),
frame.options
);
return Promise.resolve().then(async () => {
const {sanitized, invalidCount, validationErrors, duplicateStripeCustomersCount} = await sanitizeInput(frame.data.members);
invalid.count += invalidCount;
duplicateStripeCustomerIdCount = duplicateStripeCustomersCount;
if (validationErrors.length) {
invalid.errors.push(...validationErrors);
}
return Promise.map(sanitized, ((entry) => {
const api = require('./index');
entry.labels = (entry.labels && entry.labels.split(',')) || [];
const entryLabels = memberLabelsImporter.serializeMemberLabels(entry.labels);
const mergedLabels = _.unionBy(entryLabels, importSetLabels, 'name');
cleanupUndefined(entry);
let subscribed;
if (_.isUndefined(entry.subscribed_to_emails)) {
subscribed = entry.subscribed_to_emails;
} else {
subscribed = (String(entry.subscribed_to_emails).toLowerCase() !== 'false');
}
return Promise.resolve(api.members.add.query({
data: {
members: [{
email: entry.email,
name: entry.name,
note: entry.note,
subscribed: subscribed,
stripe_customer_id: entry.stripe_customer_id,
comped: (String(entry.complimentary_plan).toLocaleLowerCase() === 'true'),
labels: mergedLabels,
created_at: entry.created_at === '' ? undefined : entry.created_at
}]
},
options: {
context: frame.options.context,
options: {send_email: false}
}
})).reflect();
}), {concurrency: 10})
.each((inspection) => {
if (inspection.isFulfilled()) {
imported.count = imported.count + 1;
} else {
const error = inspection.reason();
// NOTE: if the error happens as a result of pure API call it doesn't get logged anywhere
// for this reason we have to make sure any unexpected errors are logged here
if (Array.isArray(error)) {
logging.error(error[0]);
invalid.errors.push(...error);
} else {
logging.error(error);
invalid.errors.push(error);
}
invalid.count = invalid.count + 1;
}
});
}).then(async () => {
// NOTE: grouping by context because messages can contain unique data like "customer_id"
const groupedErrors = _.groupBy(invalid.errors, 'context');
const uniqueErrors = _.uniqBy(invalid.errors, 'context');
const outputErrors = uniqueErrors.map((error) => {
let errorGroup = groupedErrors[error.context];
let errorCount = errorGroup.length;
if (error.message === i18n.t('errors.api.members.duplicateStripeCustomerIds.message')) {
errorCount = duplicateStripeCustomerIdCount;
}
// NOTE: filtering only essential error information, so API doesn't leak more error details than it should
return {
message: error.message,
context: error.context,
help: error.help,
count: errorCount
};
});
invalid.errors = outputErrors;
if (imported.count === 0 && importLabel && importLabel.generated) {
await models.Label.destroy(Object.assign({}, {id: importLabel.id}, frame.options));
importLabel = null;
}
const globalLabels = [importLabel].concat(frame.data.labels);
const pathToCSV = frame.file.path;
const headerMapping = frame.data.mapping;
const job = await membersService.importer.prepare(pathToCSV, headerMapping, globalLabels, {
createdBy: frame.user.id
});
if (job.batches <= 500 && !job.metadata.hasStripeData) {
const result = await membersService.importer.perform(job.id);
const importLabelModel = result.imported ? await models.Label.findOne(importLabel) : null;
return {
meta: {
stats: {
imported,
invalid
imported: result.imported,
invalid: result.errors
},
import_label: importLabel
import_label: importLabelModel
}
};
});
} else {
const emailRecipient = frame.user.get('email');
jobsService.addJob(async () => {
const result = await membersService.importer.perform(job.id);
const importLabelModel = result.imported ? await models.Label.findOne(importLabel) : null;
const emailContent = membersService.importer.generateCompletionEmail(result, {
emailRecipient,
importLabel: importLabelModel ? importLabelModel.toJSON() : null
});
const errorCSV = membersService.importer.generateErrorCSV(result);
await ghostMailer.send({
to: emailRecipient,
subject: importLabel.name,
html: emailContent,
forceTextContent: true,
attachments: [{
filename: `${importLabel.name} - Errors.csv`,
contents: errorCSV,
contentType: 'text/csv',
contentDisposition: 'attachment'
}]
});
});
return {};
}
}
},

View file

@ -1,6 +1,5 @@
const _ = require('lodash');
const debug = require('ghost-ignition').debug('api:canary:utils:serializers:input:members');
const {parse} = require('@tryghost/members-csv');
function defaultRelations(frame) {
if (frame.options.withRelated) {
@ -47,6 +46,17 @@ module.exports = {
async importCSV(apiConfig, frame) {
debug('importCSV');
frame.data.members = await parse(frame.file.path, frame.data.mapping);
if (!frame.data.labels) {
frame.data.labels = [];
return;
}
if (typeof frame.data.labels === 'string') {
frame.data.labels = [{name: frame.data.labels}];
return;
}
if (Array.isArray(frame.data.labels)) {
frame.data.labels = frame.data.labels.map(name => ({name}));
return;
}
}
};

View file

@ -315,8 +315,8 @@ describe('Members API', function () {
should.exist(jsonResponse.meta);
should.exist(jsonResponse.meta.stats);
jsonResponse.meta.stats.imported.count.should.equal(2);
jsonResponse.meta.stats.invalid.count.should.equal(0);
jsonResponse.meta.stats.imported.should.equal(2);
jsonResponse.meta.stats.invalid.length.should.equal(0);
jsonResponse.meta.import_label.name.should.match(/^Import \d{4}-\d{2}-\d{2} \d{2}:\d{2}$/);
const importLabel = jsonResponse.meta.import_label;

View file

@ -371,9 +371,9 @@ describe('Members API', function () {
should.exist(jsonResponse.meta.stats);
should.exist(jsonResponse.meta.import_label);
jsonResponse.meta.import_label.slug.should.equal('global-label-1');
jsonResponse.meta.stats.imported.count.should.equal(2);
jsonResponse.meta.stats.invalid.count.should.equal(0);
jsonResponse.meta.import_label.slug.should.match(/^import-/);
jsonResponse.meta.stats.imported.should.equal(2);
jsonResponse.meta.stats.invalid.length.should.equal(0);
importLabel = jsonResponse.meta.import_label.slug;
return request
@ -401,26 +401,28 @@ describe('Members API', function () {
importedMember1.stripe.subscriptions.length.should.equal(0);
// check label order
// 1 unique global + 1 record labels
importedMember1.labels.length.should.equal(2);
importedMember1.labels[0].slug.should.equal('label');
importedMember1.labels[1].slug.should.equal('global-label-1');
// 1 unique global + 1 record labels + 1 auto generated label
importedMember1.labels.length.should.equal(3);
should.exist(importedMember1.labels.find(({slug}) => slug === 'label'));
should.exist(importedMember1.labels.find(({slug}) => slug === 'global-label-1'));
should.exist(importedMember1.labels.find(({slug}) => slug.match(/^import-/)));
const importedMember2 = jsonResponse.members.find(m => m.email === 'member+labels_2@example.com');
should.exist(importedMember2);
// 1 unique global + 2 record labels
importedMember2.labels.length.should.equal(3);
importedMember2.labels[0].slug.should.equal('another-label');
importedMember2.labels[1].slug.should.equal('and-one-more');
importedMember2.labels[2].slug.should.equal('global-label-1');
importedMember2.labels.length.should.equal(4);
should.exist(importedMember2.labels.find(({slug}) => slug === 'another-label'));
should.exist(importedMember2.labels.find(({slug}) => slug === 'and-one-more'));
should.exist(importedMember2.labels.find(({slug}) => slug === 'global-label-1'));
should.exist(importedMember2.labels.find(({slug}) => slug.match(/^import-/)));
});
});
it('Can import CSV with mapped fields', function () {
return request
.post(localUtils.API.getApiQuery(`members/upload/`))
.field('mapping[email]', 'correo_electrpnico')
.field('mapping[name]', 'nombre')
.field('mapping[correo_electrpnico]', 'email')
.field('mapping[nombre]', 'name')
.attach('membersfile', path.join(__dirname, '/../../../../utils/fixtures/csv/members-with-mappings.csv'))
.set('Origin', config.get('url'))
.expect('Content-Type', /json/)
@ -434,8 +436,8 @@ describe('Members API', function () {
should.exist(jsonResponse.meta);
should.exist(jsonResponse.meta.stats);
jsonResponse.meta.stats.imported.count.should.equal(1);
jsonResponse.meta.stats.invalid.count.should.equal(0);
jsonResponse.meta.stats.imported.should.equal(1);
jsonResponse.meta.stats.invalid.length.should.equal(0);
should.exist(jsonResponse.meta.import_label);
jsonResponse.meta.import_label.slug.should.match(/^import-/);
@ -484,8 +486,8 @@ describe('Members API', function () {
should.exist(jsonResponse.meta);
should.exist(jsonResponse.meta.stats);
jsonResponse.meta.stats.imported.count.should.equal(2);
jsonResponse.meta.stats.invalid.count.should.equal(0);
jsonResponse.meta.stats.imported.should.equal(2);
jsonResponse.meta.stats.invalid.length.should.equal(0);
})
.then(() => {
return request
@ -516,29 +518,20 @@ describe('Members API', function () {
});
});
it('Fails to import members with stripe_customer_id', function () {
it('Runs imports with stripe_customer_id as background job', function () {
return request
.post(localUtils.API.getApiQuery(`members/upload/`))
.attach('membersfile', path.join(__dirname, '/../../../../utils/fixtures/csv/members-with-stripe-ids.csv'))
.set('Origin', config.get('url'))
.expect('Content-Type', /json/)
.expect('Cache-Control', testUtils.cacheRules.private)
.expect(201)
.expect(202)
.then((res) => {
should.not.exist(res.headers['x-cache-invalidate']);
const jsonResponse = res.body;
should.exist(jsonResponse);
should.exist(jsonResponse.meta);
should.exist(jsonResponse.meta.stats);
jsonResponse.meta.stats.imported.count.should.equal(0);
jsonResponse.meta.stats.invalid.count.should.equal(2);
should.equal(jsonResponse.meta.stats.invalid.errors.length, 1);
jsonResponse.meta.stats.invalid.errors[0].message.should.match(/Missing Stripe connection/);
should.not.exist(jsonResponse.meta.import_label);
should.not.exist(jsonResponse.meta);
});
});
@ -559,64 +552,10 @@ describe('Members API', function () {
should.exist(jsonResponse.meta);
should.exist(jsonResponse.meta.stats);
jsonResponse.meta.stats.imported.count.should.equal(0);
jsonResponse.meta.stats.invalid.count.should.equal(3);
jsonResponse.meta.stats.imported.should.equal(1);
jsonResponse.meta.stats.invalid.length.should.equal(1);
const validationErrors = jsonResponse.meta.stats.invalid.errors;
should.equal(validationErrors.length, 4);
const nameValidationErrors = validationErrors.find(
obj => obj.message === 'Validation failed for \'name\'.'
);
should.exist(nameValidationErrors);
nameValidationErrors.count.should.equal(1);
const emailValidationErrors = validationErrors.find(
obj => obj.message === 'Validation (isEmail) failed for email'
);
should.exist(emailValidationErrors);
emailValidationErrors.count.should.equal(1);
const createdAtValidationErrors = validationErrors.find(
obj => obj.message === 'Validation failed for \'created_at\'.'
);
should.exist(createdAtValidationErrors);
createdAtValidationErrors.count.should.equal(1);
const compedPlanValidationErrors = validationErrors.find(
obj => obj.message === 'Validation failed for \'complimentary_plan\'.'
);
should.exist(compedPlanValidationErrors);
compedPlanValidationErrors.count.should.equal(1);
should.exist(jsonResponse.meta.import_label);
jsonResponse.meta.import_label.slug.should.equal('new-global-label');
});
});
it('Fails to import member duplicate emails', function () {
return request
.post(localUtils.API.getApiQuery(`members/upload/`))
.attach('membersfile', path.join(__dirname, '/../../../../utils/fixtures/csv/members-duplicate-emails.csv'))
.set('Origin', config.get('url'))
.expect('Content-Type', /json/)
.expect('Cache-Control', testUtils.cacheRules.private)
.expect(201)
.then((res) => {
should.not.exist(res.headers['x-cache-invalidate']);
const jsonResponse = res.body;
should.exist(jsonResponse);
should.exist(jsonResponse.meta);
should.exist(jsonResponse.meta.stats);
jsonResponse.meta.stats.imported.count.should.equal(1);
jsonResponse.meta.stats.invalid.count.should.equal(1);
should.equal(jsonResponse.meta.stats.invalid.errors.length, 1);
jsonResponse.meta.stats.invalid.errors[0].message.should.equal('Member already exists');
jsonResponse.meta.stats.invalid.errors[0].count.should.equal(1);
jsonResponse.meta.stats.invalid[0].error.should.match(/Validation \(isEmail\) failed for email/);
should.exist(jsonResponse.meta.import_label);
jsonResponse.meta.import_label.slug.should.match(/^import-/);

View file

@ -1,4 +1,3 @@
email,name,note,subscribed_to_emails,complimentary_plan,stripe_customer_id,created_at,labels
valid@email.com,",name starting with coma",,not_boolean,false,,not_a_date,labels
valid2@email.com,"good name",,true,not_boolean,,2019-10-30T14:52:08.000Z,more-labels
invalid_email_value,"good name",,true,false,,2019-10-30T14:52:08.000Z,more-labels

1 email name note subscribed_to_emails complimentary_plan stripe_customer_id created_at labels
valid@email.com ,name starting with coma not_boolean false not_a_date labels
2 valid2@email.com good name true not_boolean 2019-10-30T14:52:08.000Z more-labels
3 invalid_email_value good name true false 2019-10-30T14:52:08.000Z more-labels