Infrastructure/ghost
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2025-02-03 23:00:14 -05:00
Code Issues Activity

Subscribers: Move read CSV into separate utility

Browse source 
- split out read CSV function into utility and add tests
- update API response to follow JSONAPI more closely
- update the UI to match the new API response
This commit is contained in:
kirrg001 2016-05-08 09:12:37 +02:00 • committed by Hannah Wolfe
parent 4f1fae5e7d
commit 77fc9ea265
7 changed files with 352 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
core/client/app/components/modals/import-subscribers.js
View file

@ -25,7 +25,7 @@ export default ModalComponent.extend({
},
uploadSuccess(response) {
this.set('response', response);
this.set('response', response.meta.stats);
// invoke the passed in confirm action
invokeAction(this, 'confirm');
},

10
core/client/app/mirage/config.js
View file

@ -101,9 +101,13 @@ function mockSubscribers(server) {
server.createList('subscriber', 50);
return {
imported: 50,
duplicates: 3,
invalid: 2
meta: {
stats: {
imported: 50,
duplicates: 3,
invalid: 2
}
}
};
});
}

109
core/server/api/subscribers.js
View file

@ -3,11 +3,10 @@
var Promise = require('bluebird'),
_ = require('lodash'),
fs = require('fs'),
pUnlink = Promise.promisify(fs.unlink),
readline = require('readline'),
dataProvider = require('../models'),
errors = require('../errors'),
utils = require('./utils'),
serverUtils = require('../utils'),
pipeline = require('../utils/pipeline'),
i18n = require('../i18n'),
@ -108,7 +107,7 @@ subscribers = {
*/
function doQuery(options) {
return dataProvider.Subscriber.add(options.data.subscribers[0], _.omit(options, ['data'])).catch(function (error) {
if (error.errno) {
if (error.code) {
// DB error
return Promise.reject(cleanError(error));
}
@ -263,7 +262,6 @@ subscribers = {
*/
importCSV: function (options) {
var tasks = [];
options = options || {};
function validate(options) {
@ -281,83 +279,44 @@ subscribers = {
}
function importCSV(options) {
return new Promise(function (resolve, reject) {
var filePath = options.path,
importTasks = [],
emailIdx = -1,
firstLine = true,
rl;
var filePath = options.path,
fulfilled = 0,
invalid = 0,
duplicates = 0;
rl = readline.createInterface({
input: fs.createReadStream(filePath),
terminal: false
});
rl.on('line', function (line) {
var dataToImport = line.split(',');
if (firstLine) {
if (dataToImport.length === 1) {
emailIdx = 0;
return serverUtils.readCSV({
path: filePath,
columnsToExtract: ['email']
}).then(function (result) {
return Promise.all(result.map(function (entry) {
return subscribers.add(
{subscribers: [{email: entry.email}]},
{context: options.context}
).reflect();
})).each(function (inspection) {
if (inspection.isFulfilled()) {
fulfilled = fulfilled + 1;
} else {
if (inspection.reason() instanceof errors.DataImportError) {
duplicates = duplicates + 1;
} else {
emailIdx = _.findIndex(dataToImport, function (columnName) {
if (columnName.match(/email/g)) {
return true;
} else {
return false;
}
});
invalid = invalid + 1;
}
if (emailIdx === -1) {
return reject(new errors.ValidationError(
'Couldn\'t find your email addresses! Please use a column header which contains the word "email".'
));
}
firstLine = false;
} else if (emailIdx > -1) {
importTasks.push(function () {
return subscribers.add({
subscribers: [{
email: dataToImport[emailIdx]
}
]}, {context: options.context});
});
}
});
rl.on('close', function () {
var fulfilled = 0,
duplicates = 0,
invalid = 0;
Promise.all(importTasks.map(function (promise) {
return promise().reflect();
})).each(function (inspection) {
if (inspection.isFulfilled()) {
fulfilled = fulfilled + 1;
} else {
if (inspection.reason().errorType === 'ValidationError') {
invalid = invalid + 1;
} else if (inspection.reason().errorType === 'DataImportError') {
duplicates = duplicates + 1;
}
}).then(function () {
return {
meta: {
stats: {
imported: fulfilled,
duplicates: duplicates,
invalid: invalid
}
}).then(function () {
return resolve({
stats: [{
imported: fulfilled,
duplicates: duplicates,
invalid: invalid
}]
});
}).catch(function (err) {
return reject(err);
}).finally(function () {
// Remove uploaded file from tmp location
return pUnlink(filePath);
});
});
}
};
}).finally(function () {
// Remove uploaded file from tmp location
return Promise.promisify(fs.unlink)(filePath);
});
}

5
core/server/utils/index.js
View file

@ -1,5 +1,6 @@
var unidecode = require('unidecode'),
_ = require('lodash'),
readCSV = require('./read-csv'),
utils,
getRandomInt;
@ -99,7 +100,9 @@ utils = {
/*jslint unparam:true*/
res.set({'Cache-Control': 'public, max-age=' + utils.ONE_YEAR_S});
res.redirect(301, path);
}
},
readCSV: readCSV
};
module.exports = utils;

66
core/server/utils/read-csv.js Normal file
View file

@ -0,0 +1,66 @@
var readline = require('readline'),
Promise = require('bluebird'),
lodash = require('lodash'),
errors = require('../errors'),
fs = require('fs');
function readCSV(options) {
var path = options.path,
columnsToExtract = options.columnsToExtract || [],
firstLine = true,
mapping = {},
toReturn = [],
rl;
return new Promise(function (resolve, reject) {
rl = readline.createInterface({
input: fs.createReadStream(path),
terminal: false
});
rl.on('line', function (line) {
var values = line.split(','),
entry = {};
// CASE: column headers
if (firstLine) {
if (values.length === 1) {
mapping[columnsToExtract[0]] = 0;
} else {
try {
lodash.each(columnsToExtract, function (columnToExtract) {
mapping[columnToExtract] = lodash.findIndex(values, function (value) {
if (value.match(columnToExtract)) {
return true;
}
});
// CASE: column does not exist
if (mapping[columnToExtract] === -1) {
throw new errors.ValidationError(
'Column header missing: "{{column}}".'.replace('{{column}}', columnToExtract)
);
}
});
} catch (err) {
reject(err);
}
}
firstLine = false;
} else {
lodash.each(mapping, function (index, columnName) {
entry[columnName] = values[index];
});
toReturn.push(entry);
}
});
rl.on('close', function () {
resolve(toReturn);
});
});
}
module.exports = readCSV;

67
core/test/integration/api/api_subscription_spec.js
View file

@ -1,10 +1,14 @@
/*globals describe, before, beforeEach, afterEach, it */
var testUtils = require('../../utils'),
should = require('should'),
sinon = require('sinon'),
Promise = require('bluebird'),
fs = require('fs'),
_ = require('lodash'),
context = testUtils.context,
errors = require('../../../server/errors'),
serverUtils = require('../../../server/utils'),
apiUtils = require('../../../server/api/utils'),
SubscribersAPI = require('../../../server/api/subscribers');
describe('Subscribers API', function () {
@ -221,4 +225,67 @@ describe('Subscribers API', function () {
});
});
});
describe('Read CSV', function () {
var scope = {};
beforeEach(function () {
sinon.stub(fs, 'unlink', function (path, cb) {
cb();
});
sinon.stub(apiUtils, 'checkFileExists').returns(true);
sinon.stub(serverUtils, 'readCSV', function () {
if (scope.csvError) {
return Promise.reject(new Error('csv'));
}
return Promise.resolve(scope.values);
});
});
afterEach(function () {
fs.unlink.restore();
apiUtils.checkFileExists.restore();
serverUtils.readCSV.restore();
});
it('check that fn works in general', function (done) {
scope.values = [{email: 'lol@hallo.de'}, {email: 'test'}, {email:'lol@hallo.de'}];
SubscribersAPI.importCSV(_.merge(testUtils.context.internal, {path: '/somewhere'}))
.then(function (result) {
result.meta.stats.imported.should.eql(1);
result.meta.stats.duplicates.should.eql(1);
result.meta.stats.invalid.should.eql(1);
done();
})
.catch(done);
});
it('check that fn works in general', function (done) {
scope.values = [{email: 'lol@hallo.de'}, {email: '1@kate.de'}];
SubscribersAPI.importCSV(_.merge(testUtils.context.internal, {path: '/somewhere'}))
.then(function (result) {
result.meta.stats.imported.should.eql(2);
result.meta.stats.duplicates.should.eql(0);
result.meta.stats.invalid.should.eql(0);
done();
})
.catch(done);
});
it('read csv throws an error', function (done) {
scope.csvError = true;
SubscribersAPI.importCSV(_.merge(testUtils.context.internal, {path: '/somewhere'}))
.then(function () {
done(new Error('we expected an error here!'));
})
.catch(function (err) {
err.message.should.eql('csv');
done();
});
});
});
});

173
core/test/unit/utils/read-csv_spec.js Normal file
View file

@ -0,0 +1,173 @@
/*globals describe, beforeEach, afterEach, it*/
var utils = require('../../../server/utils'),
errors = require('../../../server/errors'),
sinon = require('sinon'),
should = require('should'),
fs = require('fs'),
lodash = require('lodash'),
readline = require('readline');
describe('read csv', function () {
var scope = {};
beforeEach(function () {
sinon.stub(fs, 'createReadStream');
sinon.stub(readline, 'createInterface', function () {
return {
on: function (eventName, cb) {
switch (eventName) {
case 'line':
lodash.each(scope.csv, function (line) {
cb(line);
});
break;
case 'close':
cb();
break;
}
}
};
});
});
afterEach(function () {
fs.createReadStream.restore();
readline.createInterface.restore();
});
it('read csv: one column', function (done) {
scope.csv = [
'email',
'hannah@ghost.org',
'kate@ghost.org'
];
utils.readCSV({
path: 'read-file-is-mocked',
columnsToExtract: ['email']
}).then(function (result) {
should.exist(result);
result.length.should.eql(2);
result[0].email.should.eql('hannah@ghost.org');
result[1].email.should.eql('kate@ghost.org');
done();
}).catch(done);
});
it('read csv: two columns', function (done) {
scope.csv = [
'id,email',
'1,hannah@ghost.org',
'1,kate@ghost.org'
];
utils.readCSV({
path: 'read-file-is-mocked',
columnsToExtract: ['email']
}).then(function (result) {
should.exist(result);
result.length.should.eql(2);
result[0].email.should.eql('hannah@ghost.org');
result[1].email.should.eql('kate@ghost.org');
should.not.exist(result[0].id);
done();
}).catch(done);
});
it('read csv: two columns', function (done) {
scope.csv = [
'id,email',
'1,hannah@ghost.org',
'2,kate@ghost.org'
];
utils.readCSV({
path: 'read-file-is-mocked',
columnsToExtract: ['email', 'id']
}).then(function (result) {
should.exist(result);
result.length.should.eql(2);
result[0].email.should.eql('hannah@ghost.org');
result[0].id.should.eql('1');
result[1].email.should.eql('kate@ghost.org');
result[1].id.should.eql('2');
done();
}).catch(done);
});
it('read csv: test email regex', function (done) {
scope.csv = [
'email_address',
'hannah@ghost.org',
'kate@ghost.org'
];
utils.readCSV({
path: 'read-file-is-mocked',
columnsToExtract: ['email']
}).then(function (result) {
should.exist(result);
result.length.should.eql(2);
result[0].email.should.eql('hannah@ghost.org');
result[1].email.should.eql('kate@ghost.org');
done();
}).catch(done);
});
it('read csv: support single column use case', function (done) {
scope.csv = [
'a_column',
'hannah@ghost.org',
'kate@ghost.org'
];
utils.readCSV({
path: 'read-file-is-mocked',
columnsToExtract: ['email']
}).then(function (result) {
should.exist(result);
result.length.should.eql(2);
result[0].email.should.eql('hannah@ghost.org');
result[1].email.should.eql('kate@ghost.org');
done();
}).catch(done);
});
it('read csv: support single column use case (we would loose the first entry)', function (done) {
scope.csv = [
'hannah@ghost.org',
'kate@ghost.org'
];
utils.readCSV({
path: 'read-file-is-mocked',
columnsToExtract: ['email']
}).then(function (result) {
should.exist(result);
result.length.should.eql(1);
result[0].email.should.eql('kate@ghost.org');
done();
}).catch(done);
});
it('read csv: broken', function (done) {
scope.csv = [
'id,test',
'1,2',
'1,2'
];
utils.readCSV({
path: 'read-file-is-mocked',
columnsToExtract: ['email', 'id']
}).then(function () {
return done(new Error('we expected an error from read csv!'));
}).catch(function (err) {
(err instanceof errors.ValidationError).should.eql(true);
done();
});
});
});