✨ Added new fields to members CSV import (#11539)

no issue

- New fields that are accepted through members CSV import endpoint are:
  - `subscribed_to_emails` - corresponds to `subscribed` flag in API
  - `stripe_customer_id` - links existing Stripe customer to created member
  - `complimentary_plan` - flag controlling "Complimentary" plan subscription creation for imported member

- Noteworthy exception in field naming - `subscribed_to_emails` that corresponds to `subscribed` API flag present on members resources. It's a special case of CSV format, where users can be less technical it's more explicit to what the flag does (also the same naming is applied in the Admin UI)

- Failing to either link Stripe customer or assign "Complimentary" subscription to imported member behaves in a transaction-like manner - imported record is not created in the database. This is needed to be able to retry imports when it fails for reasons like connectivity failure with Stripe or Stripe miss-configuration.

- To avoid conflicts with linking same Stripe customer to multiple members there is a special handling for duplicate `stripe_customer_id` fields. Records with duplicates are removed from imported set.

core/server/api/canary/members.js

@@ -18,6 +18,42 @@ const decorateWithSubscriptions = async function (member) {
     });
 };
 
+const cleanupUndefined = (obj) => {
+    for (let key in obj) {
+        if (obj[key] === 'undefined') {
+            delete obj[key];
+        }
+    }
+};
+
+// NOTE: this method can be removed once unique constraints are introduced ref.: https://github.com/TryGhost/Ghost/blob/e277c6b/core/server/data/schema/schema.js#L339
+const sanitizeInput = (members) => {
+    const customersMap = members.reduce((acc, member) => {
+        if (member.stripe_customer_id) {
+            if (acc[member.stripe_customer_id]) {
+                acc[member.stripe_customer_id] += 1;
+            } else {
+                acc[member.stripe_customer_id] = 1;
+            }
+        }
+
+        return acc;
+    }, {});
+
+    const toRemove = [];
+    for (const key in customersMap) {
+        if (customersMap[key] > 1) {
+            toRemove.push(key);
+        }
+    }
+
+    let sanitized = members.filter((member) => {
+        return !(toRemove.includes(member.stripe_customer_id));
+    });
+
+    return sanitized;
+};
+
 const listMembers = async function (options) {
     const res = (await models.Member.findPage(options));
     const memberModels = res.data.map(model => model.toJSON(options));
@@ -92,21 +128,43 @@ const members = {
         },
         permissions: true,
         async query(frame) {
+            let model;
+
             try {
-                const model = await models.Member.add(frame.data.members[0], frame.options);
+                model = await models.Member.add(frame.data.members[0], frame.options);
+
+                const member = model.toJSON(frame.options);
+
+                if (frame.data.members[0].stripe_customer_id) {
+                    await membersService.api.members.linkStripeCustomer(frame.data.members[0].stripe_customer_id, member);
+                }
+
+                if (frame.data.members[0].comped) {
+                    await membersService.api.members.setComplimentarySubscription(member);
+                }
 
                 if (frame.options.send_email) {
                     await membersService.api.sendEmailWithMagicLink(model.get('email'), frame.options.email_type);
                 }
 
-                const member = model.toJSON(frame.options);
-
                 return decorateWithSubscriptions(member);
             } catch (error) {
                 if (error.code && error.message.toLowerCase().indexOf('unique') !== -1) {
                     throw new common.errors.ValidationError({message: common.i18n.t('errors.api.members.memberAlreadyExists')});
                 }
 
+                // NOTE: failed to link Stripe customer/plan/subscription
+                if (model && error.message && (error.message.indexOf('customer') || error.message.indexOf('plan') || error.message.indexOf('subscription'))) {
+                    const api = require('./index');
+
+                    await api.members.destroy.query({
+                        options: {
+                            context: frame.options.context,
+                            id: model.id
+                        }
+                    });
+                }
+
                 throw error;
             }
         }
@@ -222,24 +280,51 @@ const members = {
             method: 'add'
         },
         async query(frame) {
-            let filePath = frame.file.path,
+            let filePath = frame.file.path;
-                fulfilled = 0,
+            let fulfilled = 0;
-                invalid = 0,
+            let invalid = 0;
-                duplicates = 0;
+            let duplicates = 0;
+
+            const columnsToExtract = [{
+                name: 'email',
+                lookup: /^email/i
+            }, {
+                name: 'name',
+                lookup: /name/i
+            }, {
+                name: 'note',
+                lookup: /note/i
+            }, {
+                name: 'subscribed_to_emails',
+                lookup: /subscribed_to_emails/i
+            }, {
+                name: 'stripe_customer_id',
+                lookup: /stripe_customer_id/i
+            }, {
+                name: 'complimentary_plan',
+                lookup: /complimentary_plan/i
+            }];
 
             return fsLib.readCSV({
                 path: filePath,
-                columnsToExtract: [{name: 'email', lookup: /email/i}, {name: 'name', lookup: /name/i}, {name: 'note', lookup: /note/i}]
+                columnsToExtract: columnsToExtract
             }).then((result) => {
-                return Promise.all(result.map((entry) => {
+                const sanitized = sanitizeInput(result);
+                invalid += result.length - sanitized.length;
+
+                return Promise.all(sanitized.map((entry) => {
                     const api = require('./index');
 
+                    cleanupUndefined(entry);
                     return Promise.resolve(api.members.add.query({
                         data: {
                             members: [{
                                 email: entry.email,
                                 name: entry.name,
-                                note: entry.note
+                                note: entry.note,
+                                subscribed: (String(entry.subscribed_to_emails).toLowerCase() === 'true'),
+                                stripe_customer_id: entry.stripe_customer_id,
+                                comped: (String(entry.complimentary_plan).toLocaleLowerCase() === 'true')
                             }]
                         },
                         options: {

core/server/api/canary/utils/serializers/output/members.js

@@ -1,6 +1,8 @@
+const _ = require('lodash');
 const common = require('../../../../../lib/common');
 const debug = require('ghost-ignition').debug('api:canary:utils:serializers:output:members');
 const mapper = require('./utils/mapper');
+const {formatCSV} = require('../../../../../lib/fs');
 
 module.exports = {
     browse(data, apiConfig, frame) {
@@ -45,32 +47,30 @@ module.exports = {
     exportCSV(models, apiConfig, frame) {
         debug('exportCSV');
 
-        const fields = ['id', 'email', 'name', 'note', 'created_at', 'deleted_at'];
+        const fields = ['id', 'email', 'name', 'note', 'subscribed', 'complimentary_plan', 'stripe_customer_id', 'created_at', 'deleted_at'];
 
-        function formatCSV(data) {
+        models.members = models.members.map((member) => {
-            let csv = `${fields.join(',')}\r\n`,
+            member = mapper.mapMember(member);
-                entry,
+            let stripeCustomerId;
-                field,
-                j,
-                i;
 
-            for (j = 0; j < data.length; j = j + 1) {
+            if (member.stripe) {
-                entry = data[j];
+                stripeCustomerId = _.get(member, 'stripe.subscriptions[0].customer.id');
-
-                for (i = 0; i < fields.length; i = i + 1) {
-                    field = fields[i];
-                    csv += entry[field] !== null ? entry[field] : '';
-                    if (i !== fields.length - 1) {
-                        csv += ',';
-                    }
-                }
-                csv += '\r\n';
             }
 
-            return csv;
+            return {
-        }
+                id: member.id,
+                email: member.email,
+                name: member.name,
+                note: member.note,
+                subscribed: member.subscribed,
+                complimentary_plan: member.comped,
+                stripe_customer_id: stripeCustomerId,
+                created_at: member.created_at,
+                deleted_at: member.deleted_at
+            };
+        });
 
-        frame.response = formatCSV(models.members);
+        frame.response = formatCSV(models.members, fields);
     },
 
     importCSV(data, apiConfig, frame) {

core/server/lib/fs/format-csv.js

@@ -0,0 +1,22 @@
+module.exports = function formatCSV(data, fields) {
+    let csv = `${fields.join(',')}\r\n`;
+    let entry;
+    let field;
+    let j;
+    let i;
+
+    for (j = 0; j < data.length; j = j + 1) {
+        entry = data[j];
+
+        for (i = 0; i < fields.length; i = i + 1) {
+            field = fields[i];
+            csv += entry[field] !== null ? entry[field] : '';
+            if (i !== fields.length - 1) {
+                csv += ',';
+            }
+        }
+        csv += '\r\n';
+    }
+
+    return csv;
+};

core/server/lib/fs/index.js

@@ -3,6 +3,10 @@ module.exports = {
         return require('./read-csv');
     },
 
+    get formatCSV() {
+        return require('./format-csv');
+    },
+
     get zipFolder() {
         return require('./zip-folder');
     }

core/test/regression/api/canary/admin/members_spec.js

@@ -276,7 +276,7 @@ describe('Members API', function () {
             .then((res) => {
                 should.not.exist(res.headers['x-cache-invalidate']);
                 res.headers['content-disposition'].should.match(/Attachment;\sfilename="members/);
-                res.text.should.match(/id,email,name,note,created_at,deleted_at/);
+                res.text.should.match(/id,email,name,note,subscribed,complimentary_plan,stripe_customer_id,created_at,deleted_at/);
                 res.text.should.match(/member1@test.com/);
                 res.text.should.match(/Mr Egg/);
             });
@@ -303,4 +303,26 @@ describe('Members API', function () {
                 jsonResponse.meta.stats.invalid.should.equal(0);
             });
     });
+
+    it('Can import file with duplicate stripe customer ids', function () {
+        return request
+            .post(localUtils.API.getApiQuery(`members/csv/`))
+            .attach('membersfile', path.join(__dirname, '/../../../../utils/fixtures/csv/members-with-duplicate-stripe-ids.csv'))
+            .set('Origin', config.get('url'))
+            .expect('Content-Type', /json/)
+            .expect('Cache-Control', testUtils.cacheRules.private)
+            .expect(201)
+            .then((res) => {
+                should.not.exist(res.headers['x-cache-invalidate']);
+                const jsonResponse = res.body;
+
+                should.exist(jsonResponse);
+                should.exist(jsonResponse.meta);
+                should.exist(jsonResponse.meta.stats);
+
+                jsonResponse.meta.stats.imported.should.equal(1);
+                jsonResponse.meta.stats.duplicates.should.equal(0);
+                jsonResponse.meta.stats.invalid.should.equal(2);
+            });
+    });
 });

core/test/utils/fixtures/csv/members-with-duplicate-stripe-ids.csv

@@ -0,0 +1,4 @@
+email,subscribed,stripe_customer_id
+member+duplicate_stripe_1@example.com,true,cus_GbbIQRd8TnFqHq
+member+duplicate_stripe_2@example.com,false,cus_GbbIQRd8TnFqHq
+member+unique_stripe_1@example.com,true,cus_GbbIQRd8TnFqHA

core/test/utils/fixtures/csv/valid-members-import.csv

@@ -1,3 +1,3 @@
-email,name
+email,name,subscribed_to_emails
-jbloggs@example.com,joe
+jbloggs@example.com,joe,true
-test@example.com,test
+test@example.com,test,false

package.json

@@ -42,7 +42,7 @@
     "@nexes/nql": "0.3.0",
     "@sentry/node": "5.11.1",
     "@tryghost/helpers": "1.1.22",
-    "@tryghost/members-api": "0.12.0",
+    "@tryghost/members-api": "0.13.0",
     "@tryghost/members-ssr": "0.7.4",
     "@tryghost/social-urls": "0.1.5",
     "@tryghost/string": "^0.1.3",

yarn.lock

@@ -316,10 +316,10 @@
     jsonwebtoken "^8.5.1"
     lodash "^4.17.15"
 
-"@tryghost/members-api@0.12.0":
+"@tryghost/members-api@0.13.0":
-  version "0.12.0"
+  version "0.13.0"
-  resolved "https://registry.yarnpkg.com/@tryghost/members-api/-/members-api-0.12.0.tgz#f3b32d8216a5bfc6c4c5404634ff661a86848548"
+  resolved "https://registry.yarnpkg.com/@tryghost/members-api/-/members-api-0.13.0.tgz#7b63a7bfc3e008d84dbf647ef4c1d1b4915bdca2"
-  integrity sha512-qQAFr+QcedQDWWUaxuDC1XI3Kgcq28fj0bCOY+FqkXgSmec0A6x6glps+GY/gzQF6FfH7GBrRByJ0a+wriIqog==
+  integrity sha512-xOtk+6jrC4TEwHXUghY/hhD6I8bsM4yKY+jnu44pb9Pap15ULkFJVr8Ya0cwTEyiD2Iv2GomZgZzbu58bzSUsw==
   dependencies:
     "@tryghost/magic-link" "^0.3.3"
     bluebird "^3.5.4"