Merge pull request #4673 from ErisDS/importer-imprv

Data importer improvements and fixes

core/server/data/import/001.js

@@ -1,8 +0,0 @@
-var Importer000 = require('./000');
-
-module.exports = {
-    Importer001: Importer000,
-    importData: function (data) {
-        return new Importer000.importData(data);
-    }
-};

core/server/data/import/002.js

@@ -1,8 +0,0 @@
-var Importer000 = require('./000');
-
-module.exports = {
-    Importer002: Importer000,
-    importData: function (data) {
-        return new Importer000.importData(data);
-    }
-};

core/server/data/import/003.js

@@ -1,8 +0,0 @@
-var Importer000 = require('./000');
-
-module.exports = {
-    Importer003: Importer000,
-    importData: function (data) {
-        return new Importer000.importData(data);
-    }
-};

core/server/data/import/data-importer.js

@@ -3,37 +3,15 @@ var Promise = require('bluebird'),
     models  = require('../../models'),
     utils   = require('./utils'),
 
-    Importer000;
+    DataImporter;
 
-Importer000 = function () {
+DataImporter = function () {};
-    _.bindAll(this, 'doImport');
 
-    this.version = '000';
+DataImporter.prototype.importData = function (data) {
-
+    return this.doImport(data);
-    this.importFrom = {
-        '000': this.doImport,
-        '001': this.doImport,
-        '002': this.doImport,
-        '003': this.doImport
-    };
 };
 
-Importer000.prototype.importData = function (data) {
+DataImporter.prototype.loadUsers = function () {
-    return this.canImport(data)
-        .then(function (importerFunc) {
-            return importerFunc(data);
-        });
-};
-
-Importer000.prototype.canImport = function (data) {
-    if (data.meta && data.meta.version && this.importFrom[data.meta.version]) {
-        return Promise.resolve(this.importFrom[data.meta.version]);
-    }
-
-    return Promise.reject('Unsupported version of data: ' + data.meta.version);
-};
-
-Importer000.prototype.loadUsers = function () {
     var users = {all: {}};
 
     return models.User.findAll({include: ['roles']}).then(function (_users) {
@@ -52,11 +30,7 @@ Importer000.prototype.loadUsers = function () {
     });
 };
 
-// Importer000.prototype.importerFunction = function (t) {
+DataImporter.prototype.doUserImport = function (t, tableData, users, errors) {
-//
-// };
-
-Importer000.prototype.doUserImport = function (t, tableData, users, errors) {
     var userOps = [],
         imported = [];
 
@@ -89,7 +63,7 @@ Importer000.prototype.doUserImport = function (t, tableData, users, errors) {
     return Promise.resolve({});
 };
 
-Importer000.prototype.doImport = function (data) {
+DataImporter.prototype.doImport = function (data) {
     var self = this,
         tableData = data.data,
         imported = {},
@@ -171,8 +145,8 @@ Importer000.prototype.doImport = function (data) {
 };
 
 module.exports = {
-    Importer000: Importer000,
+    DataImporter: DataImporter,
     importData: function (data) {
-        return new Importer000().importData(data);
+        return new DataImporter().importData(data);
     }
 };

core/server/data/import/index.js

@@ -3,7 +3,7 @@ var Promise         = require('bluebird'),
     validation      = require('../validation'),
     errors          = require('../../errors'),
     uuid            = require('node-uuid'),
-    validator       = require('validator'),
+    importer        = require('./data-importer'),
     tables          = require('../schema').tables,
     validate,
     handleErrors,
@@ -89,14 +89,14 @@ sanitize = function sanitize(data) {
         });
 
     _.each(tableNames, function (tableName) {
-        // Sanitize the table data for duplicates and valid uuid values
+        // Sanitize the table data for duplicates and valid uuid and created_at values
         var sanitizedTableData = _.transform(data.data[tableName], function (memo, importValues) {
             var uuidMissing = (!importValues.uuid && tables[tableName].uuid) ? true : false,
-                uuidMalformed = (importValues.uuid && !validator.isUUID(importValues.uuid)) ? true : false,
+                uuidMalformed = (importValues.uuid && !validation.validator.isUUID(importValues.uuid)) ? true : false,
                 isDuplicate,
                 problemTag;
 
-            // Check for correct UUID and fix if neccessary
+            // Check for correct UUID and fix if necessary
             if (uuidMissing || uuidMalformed) {
                 importValues.uuid = uuid.v4();
             }
@@ -184,25 +184,12 @@ validate = function validate(data) {
     });
 };
 
-module.exports = function (version, data) {
+module.exports = function (data) {
-    var importer,
+    var sanitizeResults = sanitize(data);
-        sanitizeResults;
-
-    sanitizeResults = sanitize(data);
 
     data = sanitizeResults.data;
 
     return validate(data).then(function () {
-        try {
-            importer = require('./' + version);
-        } catch (ignore) {
-            // Zero effs given
-        }
-
-        if (!importer) {
-            return Promise.reject('No importer found');
-        }
-
         return importer.importData(data);
     }).then(function () {
         return sanitizeResults;

core/server/data/import/utils.js

@@ -185,34 +185,25 @@ utils = {
         var ops = [];
 
         tableData = stripProperties(['id'], tableData);
-
         _.each(tableData, function (post) {
              // Validate minimum post fields
             if (areEmpty(post, 'title', 'slug', 'markdown')) {
                 return;
             }
 
-            ops.push(function () {
+             // The post importer has auto-timestamping disabled
-                return models.Post.add(post, _.extend(internal, {transacting: transaction, importing: true}))
+            if (!post.created_at) {
+                post.created_at = Date.now();
+            }
+
+            ops.push(models.Post.add(post, _.extend(internal, {transacting: transaction, importing: true}))
                     .catch(function (error) {
                         return Promise.reject({raw: error, model: 'post', data: post});
-                    });
+                    })
-            });
+            );
         });
 
-        return Promise.reduce(ops, function (results, op) {
+        return Promise.settle(ops);
-            return op().then(function (result) {
-                results.push(result);
-
-                return results;
-            }).catch(function (error) {
-                if (error) {
-                    results.push(error);
-                }
-
-                return results;
-            });
-        }, []).settle();
     },
 
     importUsers: function importUsers(tableData, existingUsers, transaction) {

core/server/data/importer/handlers/json.js

@@ -20,7 +20,11 @@ JSONHandler = {
                 importData = JSON.parse(fileData);
 
                 // if importData follows JSON-API format `{ db: [exportedData] }`
-                if (_.keys(importData).length === 1 && Array.isArray(importData.db)) {
+                if (_.keys(importData).length === 1) {
+                    if (!importData.db || !Array.isArray(importData.db)) {
+                        throw new Error('Invalid JSON format, expected `{ db: [exportedData] }`');
+                    }
+
                     importData = importData.db[0];
                 }
 

core/server/data/importer/importers/data.js

@@ -8,7 +8,7 @@ DataImporter = {
         return importData;
     },
     doImport: function (importData) {
-        return importer('003', importData);
+        return importer(importData);
     }
 };
 

core/server/data/validation/index.js

@@ -27,9 +27,8 @@ validator.extend('isEmptyOrURL', function (str) {
     return (_.isEmpty(str) || validator.isURL(str, {require_protocol: false}));
 });
 
-// Validation validation against schema attributes
+// Validation against schema attributes
-// values are checked against the validation objects
+// values are checked against the validation objects from schema.js
-// form schema.js
 validateSchema = function (tableName, model) {
     var columns = _.keys(schema[tableName]),
         validationErrors = [];
@@ -163,6 +162,7 @@ validate = function (value, key, validations) {
 };
 
 module.exports = {
+    validator: validator,
     validateSchema: validateSchema,
     validateSettings: validateSettings,
     validateActiveTheme: validateActiveTheme

core/test/unit/importer_spec.js

@@ -1,9 +1,10 @@
 /*globals describe, afterEach, it*/
 /*jshint expr:true*/
-var should  = require('should'),
+var should    = require('should'),
-    sinon   = require('sinon'),
+    sinon     = require('sinon'),
-    Promise = require('bluebird'),
+    Promise   = require('bluebird'),
-    _       = require('lodash'),
+    _         = require('lodash'),
+    testUtils = require('../utils'),
 
     // Stuff we are testing
     ImportManager = require('../../server/data/importer'),
@@ -179,6 +180,32 @@ describe('Importer', function () {
             JSONHandler.types.should.containEql('application/json');
             JSONHandler.loadFile.should.be.instanceof(Function);
         });
+
+        it('correctly handles a valid db api wrapper', function (done) {
+            var file = [{
+                path: testUtils.fixtures.getExportFixturePath('export-003-api-wrapper'),
+                name: 'export-003-api-wrapper.json'
+            }];
+            JSONHandler.loadFile(file).then(function (result) {
+                _.keys(result).should.containEql('meta');
+                _.keys(result).should.containEql('data');
+                done();
+            });
+        });
+
+        it('correctly errors when given a bad db api wrapper', function (done) {
+            var file = [{
+                path: testUtils.fixtures.getExportFixturePath('export-003-api-wrapper-bad'),
+                name: 'export-003-api-wrapper-bad.json'
+            }];
+
+            JSONHandler.loadFile(file).then(function () {
+                done(new Error('Didn\'t error for bad db api wrapper'));
+            }).catch(function (response) {
+                response.type.should.equal('BadRequestError');
+                done();
+            });
+        });
     });
 
     describe('DataImporter', function () {

core/test/utils/fixtures/export-003-api-wrapper-bad.json

@@ -0,0 +1,31 @@
+{"asdadas":[{
+    "meta": {
+        "exported_on": 1388318311015,
+        "version": "003"
+    },
+    "data": {
+        "posts": [
+            {
+                "id": 1,
+                "title": "Welcome to Ghost",
+                "slug": "welcome-to-ghost",
+                "markdown": "You're live! Nice.",
+                "html": "<p>You're live! Nice.</p>",
+                "image": null,
+                "featured": 0,
+                "page": 0,
+                "status": "published",
+                "language": "en_US",
+                "meta_title": null,
+                "meta_description": null,
+                "author_id": 2,
+                "created_at": 1388318310782,
+                "created_by": 1,
+                "updated_at": 1388318310782,
+                "updated_by": 1,
+                "published_at": 1388318310783,
+                "published_by": 2
+            }
+        ]
+    }
+}]}

core/test/utils/fixtures/export-003-api-wrapper.json

@@ -0,0 +1,31 @@
+{"db":[{
+    "meta": {
+        "exported_on": 1388318311015,
+        "version": "003"
+    },
+    "data": {
+        "posts": [
+            {
+                "id": 1,
+                "title": "Welcome to Ghost",
+                "slug": "welcome-to-ghost",
+                "markdown": "You're live! Nice.",
+                "html": "<p>You're live! Nice.</p>",
+                "image": null,
+                "featured": 0,
+                "page": 0,
+                "status": "published",
+                "language": "en_US",
+                "meta_title": null,
+                "meta_description": null,
+                "author_id": 2,
+                "created_at": 1388318310782,
+                "created_by": 1,
+                "updated_at": 1388318310782,
+                "updated_by": 1,
+                "published_at": 1388318310783,
+                "published_by": 2
+            }
+        ]
+    }
+}]}

core/test/utils/fixtures/export-003-minimal.json

@@ -0,0 +1,15 @@
+{
+    "meta": {
+        "exported_on": 1388318311015,
+        "version": "003"
+    },
+    "data": {
+        "posts": [
+            {
+                "title": "Welcome to Ghost",
+                "slug": "welcome-to-ghost",
+                "markdown": "You're live! Nice."
+            }
+        ]
+    }
+}

core/test/utils/index.js

@@ -280,11 +280,15 @@ fixtures = {
         });
     },
 
+    getExportFixturePath: function (filename) {
+        return path.resolve(__dirname + '/fixtures/' + filename + '.json');
+    },
+
     loadExportFixture: function loadExportFixture(filename) {
-        var filepath = path.resolve(__dirname + '/fixtures/' + filename + '.json'),
+        var filePath = this.getExportFixturePath(filename),
             readFile = Promise.promisify(fs.readFile);
 
-        return readFile(filepath).then(function (fileContents) {
+        return readFile(filePath).then(function (fileContents) {
             var data;
 
             // Parse the json data