Infrastructure/ghost
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2025-04-08 02:52:39 -05:00
Code Issues Activity

🎨 refactor the importer (#8473)

Browse source 
refs #5422

- we can support null titles after this PR if we want
- user model: fix getAuthorRole
- user model: support adding roles by name
- we support this for roles as well, this makes it easier when importing related user roles (because usually roles already exists in the database and the related id's are wrong e.g. roles_users)
- base model: support for null created_at or updated_at values
- post or tag slugs are always safe strings
- enable an import of a null slug, no need to crash or to cover this on import layer
- add new DataImporter logic
    - uses a class inheritance mechanism to achieve an easier readability and maintenance
    - schema validation (happens on model layer) was ignored
    - allow to import unknown user id's (see https://github.com/TryGhost/Ghost/issues/8365)
    - most of the duplication handling happens on model layer (we can use the power of unique fields and errors from the database)
- the import is splitted into three steps:
  - beforeImport
    --> prepares the data to import, sorts out relations (roles, tags), detects fields (for LTS)
  - doImport
    --> does the actual import
  - afterImport
    --> updates the data after successful import e.g. update all user reference fields e.g. published_by (compares the imported data with the current state of the database)
- import images: markdown can be null
- show error message when json handler can't parse file
- do not request gravatar if email is null
- return problems/warnings after successful import
- optimise warnings in importer
- do not return warnings for role duplications, no helpful information
- error handler: return context information of error
- we show the affected json entries as one line in the UI
- show warning for: detected duplicated tag
- schema validation: fix valueMustBeBoolean translation
- remove context property from json parse error
This commit is contained in:
Katharina Irrgang 2017-05-23 18:18:13 +02:00 • committed by David Wolfe
parent 957f51e677
commit 1f37ff6053
29 changed files with 1196 additions and 956 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
core/server/api/db.js
View file

@ -60,7 +60,10 @@ db = {
function importContent(options) {
return importer.importFromFile(options)
.return({db: []});
.then(function (response) {
// NOTE: response can contain 2 objects if images are imported
return {db: [], problems: response.length === 2 ? response[1].problems : response[0].problems};
});
}
tasks = [

176
core/server/data/import/data-importer.js
View file

@ -1,176 +0,0 @@
var Promise = require('bluebird'),
_ = require('lodash'),
models = require('../../models'),
utils = require('./utils'),
i18n = require('../../i18n'),
internal = utils.internal,
DataImporter;
DataImporter = function () {};
DataImporter.prototype.importData = function (data) {
return this.doImport(data);
};
DataImporter.prototype.loadRoles = function () {
var options = _.extend({}, internal);
return models.Role.findAll(options).then(function (roles) {
return roles.toJSON();
});
};
DataImporter.prototype.loadUsers = function () {
var users = {all: {}},
options = _.extend({}, {include: ['roles']}, internal);
return models.User.findAll(options).then(function (_users) {
_users.forEach(function (user) {
users.all[user.get('email')] = {realId: user.get('id')};
if (user.related('roles').toJSON(options)[0] && user.related('roles').toJSON(options)[0].name === 'Owner') {
users.owner = user.toJSON(options);
}
});
if (!users.owner) {
return Promise.reject(i18n.t('errors.data.import.dataImporter.unableToFindOwner'));
}
return users;
});
};
DataImporter.prototype.doUserImport = function (t, tableData, owner, users, errors, roles) {
var userOps = [],
imported = [];
if (tableData.users && tableData.users.length) {
if (tableData.roles_users && tableData.roles_users.length) {
tableData = utils.preProcessRolesUsers(tableData, owner, roles);
}
// Import users, deduplicating with already present users
userOps = utils.importUsers(tableData.users, users, t).map(function (userImport) {
return userImport.reflect();
});
return Promise.all(userOps).then(function (descriptors) {
descriptors.forEach(function (d) {
if (!d.isFulfilled()) {
errors = errors.concat(d.reason());
} else {
imported.push(d.value().toJSON(internal));
}
});
// If adding the users fails,
if (errors.length > 0) {
t.rollback(errors);
} else {
return imported;
}
});
}
return Promise.resolve({});
};
DataImporter.prototype.doImport = function (data) {
var self = this,
tableData = data.data,
imported = {},
errors = [],
users = {},
owner = {}, roles = {};
return self.loadRoles().then(function (_roles) {
roles = _roles;
return self.loadUsers().then(function (result) {
owner = result.owner;
users = result.all;
return models.Base.transaction(function (t) {
// Step 1: Attempt to handle adding new users
self.doUserImport(t, tableData, owner, users, errors, roles).then(function (result) {
var importResults = [];
imported.users = result;
_.each(imported.users, function (user) {
users[user.email] = {realId: user.id};
});
// process user data - need to figure out what users we have available for assigning stuff to etc
try {
tableData = utils.processUsers(tableData, owner, users, ['posts', 'tags']);
} catch (error) {
return t.rollback([error]);
}
// Do any pre-processing of relationships (we can't depend on ids)
if (tableData.posts_tags && tableData.posts && tableData.tags) {
tableData = utils.preProcessPostTags(tableData);
}
// Import things in the right order
return utils.importTags(tableData.tags, t).then(function (results) {
if (results) {
importResults = importResults.concat(results);
}
return utils.importPosts(tableData.posts, t);
}).then(function (results) {
if (results) {
importResults = importResults.concat(results);
}
return utils.importSettings(tableData.settings, t);
}).then(function (results) {
if (results) {
importResults = importResults.concat(results);
}
return utils.importSubscribers(tableData.subscribers, t);
}).then(function (results) {
if (results) {
importResults = importResults.concat(results);
}
}).then(function () {
importResults.forEach(function (p) {
if (!p.isFulfilled()) {
errors = errors.concat(p.reason());
}
});
if (errors.length === 0) {
t.commit();
} else {
t.rollback(errors);
}
});
/** do nothing with these tables, the data shouldn't have changed from the fixtures
* permissions
* roles
* permissions_roles
* permissions_users
*/
});
}).then(function () {
// TODO: could return statistics of imported items
return Promise.resolve();
});
});
});
};
module.exports = {
DataImporter: DataImporter,
importData: function (data) {
return new DataImporter().importData(data);
}
};

207
core/server/data/import/index.js
View file

@ -1,207 +0,0 @@
var Promise = require('bluebird'),
_ = require('lodash'),
validation = require('../validation'),
errors = require('../../errors'),
uuid = require('uuid'),
importer = require('./data-importer'),
tables = require('../schema').tables,
i18n = require('../../i18n'),
validate,
handleErrors,
checkDuplicateAttributes,
sanitize,
cleanError,
doImport;
cleanError = function cleanError(error) {
var temp,
message,
offendingProperty,
value;
if (error.raw.message.toLowerCase().indexOf('unique') !== -1) {
// This is a unique constraint failure
if (error.raw.message.indexOf('ER_DUP_ENTRY') !== -1) {
temp = error.raw.message.split('\'');
if (temp.length === 5) {
value = temp[1];
temp = temp[3].split('_');
offendingProperty = temp.length === 3 ? temp[0] + '.' + temp[1] : error.model;
}
} else if (error.raw.message.indexOf('SQLITE_CONSTRAINT') !== -1) {
temp = error.raw.message.split('failed: ');
offendingProperty = temp.length === 2 ? temp[1] : error.model;
temp = offendingProperty.split('.');
value = temp.length === 2 ? error.data[temp[1]] : 'unknown';
} else if (error.raw.detail) {
value = error.raw.detail;
offendingProperty = error.model;
}
message = i18n.t('errors.data.import.index.duplicateEntryFound', {value: value, offendingProperty: offendingProperty});
}
offendingProperty = offendingProperty || error.model;
value = value || 'unknown';
message = message || error.raw.message;
return new errors.DataImportError({message: message, property: offendingProperty, value: value});
};
handleErrors = function handleErrors(errorList) {
var processedErrors = [];
if (!_.isArray(errorList)) {
return Promise.reject(errorList);
}
_.each(errorList, function (error) {
if (!error.raw) {
// These are validation errors
processedErrors.push(error);
} else if (_.isArray(error.raw)) {
processedErrors = processedErrors.concat(error.raw);
} else {
processedErrors.push(cleanError(error));
}
});
return Promise.reject(processedErrors);
};
checkDuplicateAttributes = function checkDuplicateAttributes(data, comparedValue, attribs) {
// Check if any objects in data have the same attribute values
return _.find(data, function (datum) {
return _.every(attribs, function (attrib) {
return datum[attrib] === comparedValue[attrib];
});
});
};
sanitize = function sanitize(data) {
var allProblems = {},
tablesInData = _.keys(data.data),
tableNames = _.sortBy(_.keys(tables), function (tableName) {
// We want to guarantee posts and tags go first
if (tableName === 'posts') {
return 1;
} else if (tableName === 'tags') {
return 2;
}
return 3;
});
tableNames = _.intersection(tableNames, tablesInData);
_.each(tableNames, function (tableName) {
// Sanitize the table data for duplicates and valid uuid and created_at values
var sanitizedTableData = _.transform(data.data[tableName], function (memo, importValues) {
var uuidMissing = (!importValues.uuid && tables[tableName].uuid) ? true : false,
uuidMalformed = (importValues.uuid && !validation.validator.isUUID(importValues.uuid)) ? true : false,
isDuplicate,
problemTag;
// Check for correct UUID and fix if necessary
if (uuidMissing || uuidMalformed) {
importValues.uuid = uuid.v4();
}
// Custom sanitize for posts, tags and users
if (tableName === 'posts') {
// Check if any previously added posts have the same
// title and slug
isDuplicate = checkDuplicateAttributes(memo.data, importValues, ['title', 'slug']);
// If it's a duplicate add to the problems and continue on
if (isDuplicate) {
// TODO: Put the reason why it was a problem?
memo.problems.push(importValues);
return;
}
} else if (tableName === 'tags') {
// Check if any previously added posts have the same
// name and slug
isDuplicate = checkDuplicateAttributes(memo.data, importValues, ['name', 'slug']);
// If it's a duplicate add to the problems and continue on
if (isDuplicate) {
// TODO: Put the reason why it was a problem?
// Remember this tag so it can be updated later
importValues.duplicate = isDuplicate;
memo.problems.push(importValues);
return;
}
} else if (tableName === 'posts_tags') {
// Fix up removed tags associations
problemTag = _.find(allProblems.tags, function (tag) {
return tag.id === importValues.tag_id;
});
// Update the tag id to the original "duplicate" id
if (problemTag) {
importValues.tag_id = problemTag.duplicate.id;
}
}
memo.data.push(importValues);
}, {
data: [],
problems: []
});
// Store the table data to return
data.data[tableName] = sanitizedTableData.data;
// Keep track of all problems for all tables
if (!_.isEmpty(sanitizedTableData.problems)) {
allProblems[tableName] = sanitizedTableData.problems;
}
});
return {
data: data,
problems: allProblems
};
};
validate = function validate(data) {
var validateOps = [];
_.each(_.keys(data.data), function (tableName) {
_.each(data.data[tableName], function (importValues) {
validateOps.push(validation.
validateSchema(tableName, importValues).reflect());
});
});
return Promise.all(validateOps).then(function (descriptors) {
var errorList = [];
_.each(descriptors, function (d) {
if (!d.isFulfilled()) {
errorList = errorList.concat(d.reason());
}
});
if (!_.isEmpty(errorList)) {
return Promise.reject(errorList);
}
});
};
doImport = function (data) {
var sanitizeResults = sanitize(data);
data = sanitizeResults.data;
return validate(data).then(function () {
return importer.importData(data);
}).then(function () {
return sanitizeResults;
}).catch(function (result) {
return handleErrors(result);
});
};
module.exports.doImport = doImport;

357
core/server/data/import/utils.js
View file

@ -1,357 +0,0 @@
var Promise = require('bluebird'),
_ = require('lodash'),
models = require('../../models'),
errors = require('../../errors'),
globalUtils = require('../../utils'),
i18n = require('../../i18n'),
internalContext = {context: {internal: true}},
utils,
areEmpty,
updatedSettingKeys,
stripProperties;
updatedSettingKeys = {
activePlugins: 'active_apps',
installedPlugins: 'installed_apps'
};
areEmpty = function (object) {
var fields = _.toArray(arguments).slice(1),
areEmpty = _.every(fields, function (field) {
return _.isEmpty(object[field]);
});
return areEmpty;
};
stripProperties = function stripProperties(properties, data) {
data = _.cloneDeep(data);
_.each(data, function (obj) {
_.each(properties, function (property) {
delete obj[property];
});
});
return data;
};
utils = {
internal: internalContext,
processUsers: function preProcessUsers(tableData, owner, existingUsers, objs) {
// We need to:
// 1. figure out who the owner of the blog is
// 2. figure out what users we have
// 3. figure out what users the import data refers to in foreign keys
// 4. try to map each one to a user
var userKeys = ['created_by', 'updated_by', 'published_by', 'author_id'],
userMap = {};
// Search the passed in objects for any user foreign keys
_.each(objs, function (obj) {
if (tableData[obj]) {
// For each object in the tableData that matches
_.each(tableData[obj], function (data) {
// For each possible user foreign key
_.each(userKeys, function (key) {
if (_.has(data, key) && data[key] !== null) {
userMap[data[key]] = {};
}
});
});
}
});
// We now have a list of users we need to figure out what their email addresses are
// tableData.users has id's as numbers (see fixtures/export)
// userIdToMap === tableData.users, but it's already a string, because it's an object key and they are always strings
_.each(_.keys(userMap), function (userIdToMap) {
var foundUser = _.find(tableData.users, function (tableDataUser) {
return tableDataUser.id.toString() === userIdToMap;
});
// we now know that userToMap's email is foundUser.email - look them up in existing users
if (foundUser && _.has(foundUser, 'email') && _.has(existingUsers, foundUser.email)) {
existingUsers[foundUser.email].importId = userIdToMap;
userMap[userIdToMap] = existingUsers[foundUser.email].realId;
} else if (models.User.isOwnerUser(userIdToMap)) {
existingUsers[owner.email].importId = userIdToMap;
userMap[userIdToMap] = existingUsers[owner.email].realId;
} else if (models.User.isExternalUser(userIdToMap)) {
userMap[userIdToMap] = models.User.externalUser;
} else {
throw new errors.DataImportError({
message: i18n.t('errors.data.import.utils.dataLinkedToUnknownUser', {userToMap: userIdToMap}),
property: 'user.id',
value: userIdToMap
});
}
});
// now replace any user foreign keys
_.each(objs, function (obj) {
if (tableData[obj]) {
// For each object in the tableData that matches
_.each(tableData[obj], function (data) {
// For each possible user foreign key
_.each(userKeys, function (key) {
if (_.has(data, key) && data[key] !== null) {
data[key] = userMap[data[key]];
}
});
});
}
});
return tableData;
},
preProcessPostTags: function preProcessPostTags(tableData) {
var postTags,
postsWithTags = new Map();
postTags = tableData.posts_tags;
_.each(postTags, function (postTag) {
if (!postsWithTags.get(postTag.post_id)) {
postsWithTags.set(postTag.post_id, []);
}
postsWithTags.get(postTag.post_id).push(postTag.tag_id);
});
postsWithTags.forEach(function (tagIds, postId) {
var post, tags;
post = _.find(tableData.posts, function (post) {
return post.id === postId;
});
if (post) {
tags = _.filter(tableData.tags, function (tag) {
return _.indexOf(tagIds, tag.id) !== -1;
});
post.tags = [];
_.each(tags, function (tag) {
// names are unique.. this should get the right tags added
// as long as tags are added first;
post.tags.push({name: tag.name});
});
}
});
return tableData;
},
preProcessRolesUsers: function preProcessRolesUsers(tableData, owner, roles) {
var validRoles = _.map(roles, 'name');
if (!tableData.roles || !tableData.roles.length) {
tableData.roles = roles;
}
_.each(tableData.roles, function (_role) {
var match = false;
// Check import data does not contain unknown roles
_.each(validRoles, function (validRole) {
if (_role.name === validRole) {
match = true;
_role.oldId = _role.id;
_role.id = _.find(roles, {name: validRole}).id;
}
});
// If unknown role is found then remove role to force down to Author
if (!match) {
_role.oldId = _role.id;
_role.id = _.find(roles, {name: 'Author'}).id;
}
});
_.each(tableData.roles_users, function (roleUser) {
var user = _.find(tableData.users, function (user) {
return user.id === roleUser.user_id;
});
// Map role_id to updated roles id
roleUser.role_id = _.find(tableData.roles, {oldId: roleUser.role_id}).id;
// Check for owner users that do not match current owner and change role to administrator
if (roleUser.role_id === owner.roles[0].id && user && user.email && user.email !== owner.email) {
roleUser.role_id = _.find(roles, {name: 'Administrator'}).id;
user.roles = [roleUser.role_id];
}
// just the one role for now
if (user && !user.roles) {
user.roles = [roleUser.role_id];
}
});
return tableData;
},
importTags: function importTags(tableData, transaction) {
if (!tableData) {
return Promise.resolve();
}
var ops = [];
tableData = stripProperties(['id'], tableData);
_.each(tableData, function (tag) {
// Validate minimum tag fields
if (areEmpty(tag, 'name', 'slug')) {
return;
}
ops.push(models.Tag.findOne({name: tag.name}, {transacting: transaction}).then(function (_tag) {
if (!_tag) {
return models.Tag.add(tag, _.extend({}, internalContext, {transacting: transaction}))
.catch(function (error) {
return Promise.reject({raw: error, model: 'tag', data: tag});
});
}
return _tag;
}).reflect());
});
return Promise.all(ops);
},
importPosts: function importPosts(tableData, transaction) {
if (!tableData) {
return Promise.resolve();
}
var ops = [];
tableData = stripProperties(['id'], tableData);
_.each(tableData, function (post) {
// Validate minimum post fields
if (areEmpty(post, 'title', 'slug', 'markdown')) {
return;
}
// The post importer has auto-timestamping disabled
if (!post.created_at) {
post.created_at = Date.now();
}
ops.push(models.Post.add(post, _.extend({}, internalContext, {transacting: transaction, importing: true}))
.catch(function (error) {
return Promise.reject({raw: error, model: 'post', data: post});
}).reflect()
);
});
return Promise.all(ops);
},
importUsers: function importUsers(tableData, existingUsers, transaction) {
var ops = [];
tableData = stripProperties(['id'], tableData);
_.each(tableData, function (user) {
// Validate minimum user fields
if (areEmpty(user, 'name', 'slug', 'email')) {
return;
}
if (_.has(existingUsers, user.email)) {
// User is already present, ignore
return;
}
// Set password to a random password, and lock the account
user.password = globalUtils.uid(50);
user.status = 'locked';
ops.push(models.User.add(user, _.extend({}, internalContext, {transacting: transaction}))
.catch(function (error) {
return Promise.reject({raw: error, model: 'user', data: user});
}));
});
return ops;
},
importSettings: function importSettings(tableData, transaction) {
if (!tableData) {
return Promise.resolve();
}
// for settings we need to update individual settings, and insert any missing ones
// settings we MUST NOT update are 'core' and 'theme' settings
// as all of these will cause side effects which don't make sense for an import
var blackList = ['core', 'theme'],
ops = [];
tableData = stripProperties(['id'], tableData);
tableData = _.filter(tableData, function (data) {
return blackList.indexOf(data.type) === -1;
});
// Clean up legacy plugin setting references
_.each(tableData, function (datum) {
datum.key = updatedSettingKeys[datum.key] || datum.key;
});
ops.push(models.Settings.edit(tableData, _.extend({}, internalContext, {transacting: transaction})).catch(function (error) {
// Ignore NotFound errors
if (!(error instanceof errors.NotFoundError)) {
return Promise.reject({raw: error, model: 'setting', data: tableData});
}
}).reflect());
return Promise.all(ops);
},
importSubscribers: function importSubscribers(tableData, transaction) {
if (!tableData) {
return Promise.resolve();
}
var ops = [];
tableData = stripProperties(['id'], tableData);
_.each(tableData, function (subscriber) {
ops.push(models.Subscriber.add(subscriber, _.extend({}, internalContext, {transacting: transaction}))
.catch(function (error) {
// ignore duplicates
if (error.code && error.message.toLowerCase().indexOf('unique') === -1) {
return Promise.reject({
raw: error,
model: 'subscriber',
data: subscriber
});
}
}).reflect());
});
return Promise.all(ops);
},
/** For later **/
importApps: function importApps(tableData, transaction) {
if (!tableData) {
return Promise.resolve();
}
var ops = [];
tableData = stripProperties(['id'], tableData);
_.each(tableData, function (app) {
// Avoid duplicates
ops.push(models.App.findOne({name: app.name}, {transacting: transaction}).then(function (_app) {
if (!_app) {
return models.App.add(app, _.extend({}, internalContext, {transacting: transaction}))
.catch(function (error) {
return Promise.reject({raw: error, model: 'app', data: app});
});
}
return _app;
}).reflect());
});
return Promise.all(ops);
}
};
module.exports = utils;

6
core/server/data/importer/handlers/json.js
View file

@ -24,7 +24,9 @@ JSONHandler = {
// if importData follows JSON-API format `{ db: [exportedData] }`
if (_.keys(importData).length === 1) {
if (!importData.db || !Array.isArray(importData.db)) {
throw new errors.GhostError({message: i18n.t('errors.data.importer.handlers.json.invalidJsonFormat')});
throw new errors.GhostError({
message: i18n.t('errors.data.importer.handlers.json.invalidJsonFormat')
});
}
importData = importData.db[0];
@ -34,7 +36,7 @@ JSONHandler = {
} catch (err) {
return Promise.reject(new errors.BadRequestError({
err: err,
context: i18n.t('errors.data.importer.handlers.json.apiDbImportContent'),
message: err.message,
help: i18n.t('errors.data.importer.handlers.json.checkImportJsonIsValid')
}));
}

15
core/server/data/importer/importers/data.js
View file

@ -1,15 +0,0 @@
var importer = require('../../import'),
DataImporter;
DataImporter = {
type: 'data',
preProcess: function (importData) {
importData.preProcessedByData = true;
return importData;
},
doImport: function (importData) {
return importer.doImport(importData);
}
};
module.exports = DataImporter;

191
core/server/data/importer/importers/data/base.js Normal file
View file

@ -0,0 +1,191 @@
'use strict';
const debug = require('ghost-ignition').debug('importer:base'),
errors = require('../../../../errors'),
models = require('../../../../models'),
_ = require('lodash'),
Promise = require('bluebird');
class Base {
constructor(options) {
let self = this;
this.modelName = options.modelName;
this.problems = [];
this.errorConfig = {
allowDuplicates: true,
returnDuplicates: true
};
this.dataKeyToImport = options.dataKeyToImport;
this.dataToImport = _.cloneDeep(options[this.dataKeyToImport] || []);
this.importedData = [];
// NOTE: e.g. properties are removed or properties are added/changed before importing
_.each(options, function (obj, key) {
if (options.requiredData.indexOf(key) !== -1) {
self[key] = _.cloneDeep(obj);
}
});
if (!this.users) {
this.users = _.cloneDeep(options.users);
}
}
/**
* Never ever import these attributes!
*/
stripProperties(properties) {
_.each(this.dataToImport, function (obj) {
_.each(properties, function (property) {
delete obj[property];
});
});
}
beforeImport() {
this.stripProperties(['id']);
return Promise.resolve();
}
handleError(errs, obj) {
let self = this, errorsToReject = [], problems = [];
// CASE: validation errors, see models/base/index.js onValidate
if (!_.isArray(errs)) {
errs = [errs];
}
_.each(errs, function (err) {
if (err.code && err.message.toLowerCase().indexOf('unique') !== -1) {
if (self.errorConfig.allowDuplicates) {
if (self.errorConfig.returnDuplicates) {
problems.push({
message: 'Entry was not imported and ignored. Detected duplicated entry.',
help: self.modelName,
context: JSON.stringify(obj),
err: err
});
}
} else {
errorsToReject.push(new errors.DataImportError({
message: 'Detected duplicated entry.',
help: self.modelName,
context: JSON.stringify(obj),
err: err
}));
}
} else if (err instanceof errors.NotFoundError) {
problems.push({
message: 'Entry was not imported and ignored. Could not find entry.',
help: self.modelName,
context: JSON.stringify(obj),
err: err
});
} else {
if (!errors.utils.isIgnitionError(err)) {
err = new errors.DataImportError({
message: err.message,
context: JSON.stringify(obj),
help: self.modelName,
errorType: err.errorType,
err: err
});
} else {
err.context = JSON.stringify(obj);
}
errorsToReject.push(err);
}
});
if (!errorsToReject.length) {
this.problems = this.problems.concat(problems);
debug('detected problem/warning', problems);
return Promise.resolve();
}
debug('err', errorsToReject, obj);
return Promise.reject(errorsToReject);
}
doImport(options) {
debug('doImport', this.modelName, this.dataToImport.length);
let self = this, ops = [];
_.each(this.dataToImport, function (obj) {
ops.push(models[self.modelName].add(obj, options)
.then(function (importedModel) {
obj.model = importedModel.toJSON();
self.importedData.push(obj.model);
return importedModel;
})
.catch(function (err) {
return self.handleError(err, obj);
})
.reflect()
);
});
return Promise.all(ops);
}
/**
* Update all user reference fields e.g. published_by
*
* Background:
* - we never import the id field
* - almost each imported model has a reference to a user reference
* - we update all fields after the import (!)
*/
afterImport(options) {
let self = this, dataToEdit = {}, oldUser;
debug('afterImport', this.modelName);
return Promise.each(this.dataToImport, function (obj) {
if (!obj.model) {
return;
}
return Promise.each(['author_id', 'published_by', 'created_by', 'updated_by'], function (key) {
if (!obj[key]) {
return;
}
if (models.User.isOwnerUser(obj[key])) {
return;
}
oldUser = _.find(self.users, {id: obj[key]});
if (!oldUser) {
self.problems.push({
message: 'Entry was imported, but we were not able to update user reference field: ' + key,
help: self.modelName,
context: JSON.stringify(obj)
});
return;
}
return models.User.findOne({
email: oldUser.email,
status: 'all'
}, options).then(function (userModel) {
dataToEdit = {};
dataToEdit[key] = userModel.id;
return models[self.modelName].edit(dataToEdit, _.extend(options, {id: obj.model.id}));
});
});
});
}
}
module.exports = Base;

102
core/server/data/importer/importers/data/index.js Normal file
View file

@ -0,0 +1,102 @@
var _ = require('lodash'),
Promise = require('bluebird'),
models = require('../../../../models'),
utils = require('../../../../utils'),
SubscribersImporter = require('./subscribers'),
PostsImporter = require('./posts'),
TagsImporter = require('./tags'),
SettingsImporter = require('./settings'),
UsersImporter = require('./users'),
RolesImporter = require('./roles'),
importers = {},
DataImporter;
DataImporter = {
type: 'data',
preProcess: function preProcess(importData) {
importData.preProcessedByData = true;
return importData;
},
init: function init(importData) {
importers.roles = new RolesImporter(importData.data);
importers.tags = new TagsImporter(importData.data);
importers.users = new UsersImporter(importData.data);
importers.subscribers = new SubscribersImporter(importData.data);
importers.posts = new PostsImporter(importData.data);
importers.settings = new SettingsImporter(importData.data);
return importData;
},
doImport: function doImport(importData) {
var ops = [], errors = [], results = [], options = {
importing: true,
context: {
internal: true
}
};
this.init(importData);
return models.Base.transaction(function (transacting) {
options.transacting = transacting;
_.each(importers, function (importer) {
ops.push(function doModelImport() {
return importer.beforeImport(options)
.then(function () {
return importer.doImport(options)
.then(function (_results) {
results = results.concat(_results);
});
});
});
});
_.each(importers, function (importer) {
ops.push(function afterImport() {
return importer.afterImport(options);
});
});
utils.sequence(ops)
.then(function () {
results.forEach(function (promise) {
if (!promise.isFulfilled()) {
errors = errors.concat(promise.reason());
}
});
if (errors.length === 0) {
transacting.commit();
} else {
transacting.rollback(errors);
}
});
}).then(function () {
/**
* data: imported data
* originalData: data from the json file
* problems: warnings
*/
var toReturn = {
data: {},
originalData: importData.data,
problems: []
};
_.each(importers, function (importer) {
toReturn.problems = toReturn.problems.concat(importer.problems);
toReturn.data[importer.dataKeyToImport] = importer.importedData;
});
return toReturn;
}).catch(function (errors) {
return Promise.reject(errors);
});
}
};
module.exports = DataImporter;

104
core/server/data/importer/importers/data/posts.js Normal file
View file

@ -0,0 +1,104 @@
'use strict';
const debug = require('ghost-ignition').debug('importer:posts'),
_ = require('lodash'),
uuid = require('uuid'),
BaseImporter = require('./base'),
validation = require('../../../validation');
class PostsImporter extends BaseImporter {
constructor(options) {
super(_.extend(options, {
modelName: 'Post',
dataKeyToImport: 'posts',
requiredData: ['tags', 'posts_tags']
}));
}
sanitizeAttributes() {
_.each(this.dataToImport, function (obj) {
if (!validation.validator.isUUID(obj.uuid || '')) {
obj.uuid = uuid.v4();
}
});
}
/**
* We don't have to worry about existing tag id's.
* e.g. you import a tag, which exists (doesn't get imported)
* ...because we add tags by unique name.
*/
addTagsToPosts() {
let postTags = this.posts_tags,
postsWithTags = new Map(),
self = this,
tags,
duplicatedTagsPerPost = {};
_.each(postTags, function (postTag) {
if (!postsWithTags.get(postTag.post_id)) {
postsWithTags.set(postTag.post_id, []);
}
if (postsWithTags.get(postTag.post_id).indexOf(postTag.tag_id) !== -1) {
if (!duplicatedTagsPerPost.hasOwnProperty(postTag.post_id)) {
duplicatedTagsPerPost[postTag.post_id] = [];
}
duplicatedTagsPerPost[postTag.post_id].push(postTag.tag_id);
}
postsWithTags.get(postTag.post_id).push(postTag.tag_id);
});
postsWithTags.forEach(function (tagIds, postId) {
tags = _.filter(self.tags, function (tag) {
return _.indexOf(tagIds, tag.id) !== -1;
});
_.each(tags, function (tag) {
_.each(self.dataToImport, function (obj) {
if (obj.id === postId) {
if (!_.isArray(obj.tags)) {
obj.tags = [];
}
if (duplicatedTagsPerPost.hasOwnProperty(postId) && duplicatedTagsPerPost[postId].length) {
self.problems.push({
message: 'Detected duplicated tags for: ' + obj.title || obj.slug,
help: self.modelName,
context: JSON.stringify({
tags: _.map(_.filter(self.tags, function (tag) {
return _.indexOf(duplicatedTagsPerPost[postId], tag.id) !== -1;
}), function (value) {
return value.slug || value.name;
})
})
});
}
obj.tags.push({
name: tag.name
});
}
});
});
});
}
beforeImport() {
debug('beforeImport');
this.sanitizeAttributes();
this.addTagsToPosts();
// NOTE: do after, because model properties are deleted e.g. post.id
return super.beforeImport();
}
doImport(options) {
return super.doImport(options);
}
}
module.exports = PostsImporter;

28
core/server/data/importer/importers/data/roles.js Normal file
View file

@ -0,0 +1,28 @@
'use strict';
const debug = require('ghost-ignition').debug('importer:roles'),
_ = require('lodash'),
BaseImporter = require('./base');
class RolesImporter extends BaseImporter {
constructor(options) {
super(_.extend(options, {
modelName: 'Role',
dataKeyToImport: 'roles',
requiredData: []
}));
this.errorConfig.returnDuplicates = false;
}
beforeImport() {
debug('beforeImport');
return super.beforeImport();
}
doImport(options) {
return super.doImport(options);
}
}
module.exports = RolesImporter;

70
core/server/data/importer/importers/data/settings.js Normal file
View file

@ -0,0 +1,70 @@
'use strict';
const debug = require('ghost-ignition').debug('importer:settings'),
Promise = require('bluebird'),
_ = require('lodash'),
BaseImporter = require('./base'),
models = require('../../../../models');
class SettingsImporter extends BaseImporter {
constructor(options) {
super(_.extend(options, {
modelName: 'Settings',
dataKeyToImport: 'settings',
requiredData: []
}));
this.legacyKeys = {
activePlugins: 'active_apps',
installedPlugins: 'installed_apps'
};
}
/**
* - 'core' and 'theme' are blacklisted
* - clean up legacy plugin setting references
*/
beforeImport() {
debug('beforeImport');
let self = this;
this.dataToImport = _.filter(this.dataToImport, function (data) {
return ['core', 'theme'].indexOf(data.type) === -1;
});
_.each(this.dataToImport, function (obj) {
obj.key = self.legacyKeys[obj.key] || obj.key;
});
return super.beforeImport();
}
doImport(options) {
debug('doImport', this.dataToImport.length);
let self = this, ops = [];
_.each(this.dataToImport, function (model) {
ops.push(
models.Settings.edit(model, options)
.catch(function (err) {
return self.handleError(err, model);
})
.reflect()
);
});
return Promise.all(ops);
}
/**
* We only update existing settings models.
* Nothing todo here.
*/
afterImport() {
return Promise.resolve();
}
}
module.exports = SettingsImporter;

26
core/server/data/importer/importers/data/subscribers.js Normal file
View file

@ -0,0 +1,26 @@
'use strict';
const debug = require('ghost-ignition').debug('importer:subscribers'),
_ = require('lodash'),
BaseImporter = require('./base');
class SubscribersImporter extends BaseImporter {
constructor(options) {
super(_.extend(options, {
modelName: 'Subscriber',
dataKeyToImport: 'subscribers',
requiredData: []
}));
}
beforeImport() {
debug('beforeImport');
return super.beforeImport();
}
doImport(options) {
return super.doImport(options);
}
}
module.exports = SubscribersImporter;

57
core/server/data/importer/importers/data/tags.js Normal file
View file

@ -0,0 +1,57 @@
'use strict';
const debug = require('ghost-ignition').debug('importer:tags'),
Promise = require('bluebird'),
_ = require('lodash'),
BaseImporter = require('./base'),
models = require('../../../../models');
class TagsImporter extends BaseImporter {
constructor(options) {
super(_.extend(options, {
modelName: 'Tag',
dataKeyToImport: 'tags',
requiredData: []
}));
}
beforeImport() {
debug('beforeImport');
return super.beforeImport();
}
/**
* Find tag before adding.
* Background:
* - the tag model is smart enough to regenerate unique fields
* - so if you import a tag name "test" and the same tag name exists, it would add "test-2"
* - that's we add a protection here to first find the tag
*/
doImport(options) {
debug('doImport', this.modelName, this.dataToImport.length);
let self = this, ops = [];
_.each(this.dataToImport, function (obj) {
ops.push(models[self.modelName].findOne({name: obj.name}, options).then(function (tag) {
if (tag) {
return Promise.resolve();
}
return models[self.modelName].add(obj, options)
.then(function (newModel) {
obj.model = newModel.toJSON();
self.importedData.push(obj.model);
return newModel;
})
.catch(function (err) {
return self.handleError(err, obj);
});
}).reflect());
});
return Promise.all(ops);
}
}
module.exports = TagsImporter;

70
core/server/data/importer/importers/data/users.js Normal file
View file

@ -0,0 +1,70 @@
'use strict';
const debug = require('ghost-ignition').debug('importer:users'),
_ = require('lodash'),
BaseImporter = require('./base'),
globalUtils = require('../../../../utils');
class UsersImporter extends BaseImporter {
constructor(options) {
super(_.extend(options, {
modelName: 'User',
dataKeyToImport: 'users',
requiredData: ['roles', 'roles_users']
}));
}
/**
* - all imported users are locked and get a random password
* - they have to follow the password forgotten flow
* - we add the role by name [supported by the user model, see User.add]
* - background: if you import roles, but they exist already, the related user roles reference to an old model id
*/
beforeImport() {
debug('beforeImport');
let self = this, role;
_.each(this.dataToImport, function (model) {
model.password = globalUtils.uid(50);
model.status = 'locked';
});
_.each(this.roles_users, function (attachedRole) {
role = _.find(self.roles, function (role) {
if (attachedRole.role_id === role.id) {
return role;
}
});
// CASE: default fallback role
if (!role) {
role = {name: 'Author'};
}
_.each(self.dataToImport, function (obj) {
if (attachedRole.user_id === obj.id) {
if (!_.isArray(obj.roles)) {
obj.roles = [];
}
// CASE: we never import the owner, the owner is always present in the database
// That's why it is not allowed to import the owner role
if (role.name === 'Owner') {
role.name = 'Administrator';
}
obj.roles.push(role.name);
}
});
});
return super.beforeImport();
}
doImport(options) {
return super.doImport(options);
}
}
module.exports = UsersImporter;

4
core/server/data/importer/importers/image.js
View file

@ -8,6 +8,10 @@ var _ = require('lodash'),
preProcessUsers;
replaceImage = function (markdown, image) {
if (!markdown) {
return;
}
// Normalizes to include a trailing slash if there was one
var regex = new RegExp('(/)?' + image.originalPath, 'gm');

1
core/server/middleware/error-handler.js
View file

@ -105,6 +105,7 @@ _private.JSONErrorRenderer = function JSONErrorRenderer(err, req, res, /*jshint
res.json({
errors: [{
message: err.message,
context: err.context,
errorType: err.errorType,
errorDetails: err.errorDetails
}]

8
core/server/models/base/index.js
View file

@ -134,6 +134,14 @@ ghostBookshelf.Model = ghostBookshelf.Model.extend({
if (schema.tables[this.tableName].hasOwnProperty('created_by') && !this.get('created_by')) {
this.set('created_by', this.contextUser(options));
}
if (!newObj.get('created_at')) {
newObj.set('created_at', new Date());
}
if (!newObj.get('updated_at')) {
newObj.set('updated_at', new Date());
}
},
onSaving: function onSaving(newObj, attr, options) {

6
core/server/models/post.js
View file

@ -253,8 +253,10 @@ Post = ghostBookshelf.Model.extend({
}
// disabling sanitization until we can implement a better version
title = this.get('title') || i18n.t('errors.models.post.untitled');
this.set('title', _.toString(title).trim());
if (!options.importing) {
title = this.get('title') || i18n.t('errors.models.post.untitled');
this.set('title', _.toString(title).trim());
}
// ### Business logic for published_at and published_by
// If the current status is 'published' and published_at is not set, set it to now

43
core/server/models/user.js
View file

@ -2,6 +2,7 @@ var _ = require('lodash'),
Promise = require('bluebird'),
bcrypt = require('bcryptjs'),
validator = require('validator'),
ObjectId = require('bson-objectid'),
ghostBookshelf = require('./base'),
baseUtils = require('./base/utils'),
errors = require('../errors'),
@ -106,7 +107,7 @@ User = ghostBookshelf.Model.extend({
ghostBookshelf.Model.prototype.onSaving.apply(this, arguments);
if (self.hasChanged('email')) {
if (self.hasChanged('email') && self.get('email')) {
tasks.gravatar = (function lookUpGravatar() {
return gravatar.lookup({
email: self.get('email')
@ -455,9 +456,10 @@ User = ghostBookshelf.Model.extend({
}
function getAuthorRole() {
return ghostBookshelf.model('Role').findOne({name: 'Author'}, _.pick(options, 'transacting')).then(function then(authorRole) {
return [authorRole.get('id')];
});
return ghostBookshelf.model('Role').findOne({name: 'Author'}, _.pick(options, 'transacting'))
.then(function then(authorRole) {
return [authorRole.get('id')];
});
}
/**
@ -466,16 +468,45 @@ User = ghostBookshelf.Model.extend({
* roles: [] -> no default role (used for owner creation, see fixtures.json)
* roles: undefined -> default role
*/
roles = data.roles || getAuthorRole();
roles = data.roles;
delete data.roles;
return ghostBookshelf.Model.add.call(self, userData, options)
.then(function then(addedUser) {
// Assign the userData to our created user so we can pass it back
userData = addedUser;
})
.then(function () {
if (!roles) {
return getAuthorRole();
}
return Promise.resolve(roles);
})
.then(function (_roles) {
roles = _roles;
// CASE: it is possible to add roles by name, by id or by object
if (_.isString(roles[0]) && !ObjectId.isValid(roles[0])) {
return Promise.map(roles, function (roleName) {
return ghostBookshelf.model('Role').findOne({
name: roleName
}, options);
}).then(function (roleModels) {
roles = [];
_.each(roleModels, function (roleModel) {
roles.push(roleModel.id);
});
});
}
return Promise.resolve();
})
.then(function () {
return baseUtils.attach(User, userData.id, 'roles', roles, options);
}).then(function then() {
})
.then(function then() {
// find and return the added user
return self.findOne({id: userData.id, status: 'all'}, options);
});

2
core/server/translations/en.json
View file

@ -569,7 +569,7 @@
"validation": {
"index": {
"valueCannotBeBlank": "Value in [{tableName}.{columnKey}] cannot be blank.",
"valueMustBeBoolean": "Value in [settings.key] must be one of true, false, 0 or 1.",
"valueMustBeBoolean": "Value in [{tableName}.{columnKey}] must be one of true, false, 0 or 1.",
"valueExceedsMaxLength": "Value in [{tableName}.{columnKey}] exceeds maximum length of {maxlength} characters.",
"valueIsNotInteger": "Value in [{tableName}.{columnKey}] is not an integer.",
"themeCannotBeActivated": "{themeName} cannot be activated because it is not currently installed.",

4
core/server/utils/index.js
View file

@ -57,6 +57,10 @@ utils = {
safeString: function (string, options) {
options = options || {};
if (string === null) {
string = '';
}
// Handle the £ symbol separately, since it needs to be removed before the unicode conversion.
string = string.replace(/£/g, '-');

2
core/test/functional/routes/api/posts_spec.js
View file

@ -771,7 +771,7 @@ describe('Post API', function () {
should.not.exist(res.headers['x-cache-invalidate']);
jsonResponse = res.body;
should.exist(jsonResponse.errors);
testUtils.API.checkResponseValue(jsonResponse.errors[0], ['message', 'errorType']);
testUtils.API.checkResponseValue(jsonResponse.errors[0], ['message', 'errorType', 'context']);
done();
});
});

4
core/test/functional/routes/api/public_api_spec.js
View file

@ -206,7 +206,7 @@ describe('Public API', function () {
var jsonResponse = res.body;
should.exist(jsonResponse);
should.exist(jsonResponse.errors);
testUtils.API.checkResponseValue(jsonResponse.errors[0], ['message', 'errorType']);
testUtils.API.checkResponseValue(jsonResponse.errors[0], ['message', 'errorType', 'context']);
done();
});
});
@ -227,7 +227,7 @@ describe('Public API', function () {
var jsonResponse = res.body;
should.exist(jsonResponse);
should.exist(jsonResponse.errors);
testUtils.API.checkResponseValue(jsonResponse.errors[0], ['message', 'errorType']);
testUtils.API.checkResponseValue(jsonResponse.errors[0], ['message', 'errorType', 'context']);
done();
});
});

278
core/test/integration/import_spec.js → core/test/integration/data/importer/importers/data_spec.js
View file

@ -1,6 +1,6 @@
var should = require('should'),
sinon = require('sinon'),
testUtils = require('../utils/index'),
testUtils = require('../../../../utils/index'),
Promise = require('bluebird'),
moment = require('moment'),
assert = require('assert'),
@ -8,10 +8,10 @@ var should = require('should'),
validator = require('validator'),
// Stuff we are testing
db = require('../../server/data/db'),
exporter = require('../../server/data/export'),
importer = require('../../server/data/import'),
DataImporter = require('../../server/data/import/data-importer'),
db = require('../../../../../server/data/db'),
exporter = require('../../../../../server/data/export'),
importer = require('../../../../../server/data/importer'),
dataImporter = importer.importers[1],
knex = db.knex,
sandbox = sinon.sandbox.create();
@ -19,6 +19,11 @@ var should = require('should'),
// Tests in here do an import for each test
describe('Import', function () {
before(testUtils.teardown);
beforeEach(function () {
sandbox.stub(importer, 'cleanUp');
});
afterEach(testUtils.teardown);
afterEach(function () {
sandbox.restore();
@ -27,25 +32,6 @@ describe('Import', function () {
should.exist(exporter);
should.exist(importer);
describe('Resolves', function () {
beforeEach(testUtils.setup());
it('resolves DataImporter', function (done) {
var importStub = sandbox.stub(DataImporter, 'importData', function () {
return Promise.resolve();
}),
fakeData = {test: true};
importer.doImport(fakeData).then(function () {
importStub.calledWith(fakeData).should.equal(true);
importStub.restore();
done();
}).catch(done);
});
});
describe('Sanitizes', function () {
beforeEach(testUtils.setup('roles', 'owner', 'settings'));
@ -54,7 +40,7 @@ describe('Import', function () {
testUtils.fixtures.loadExportFixture('export-003').then(function (exported) {
exportData = exported;
return importer.doImport(exportData);
return dataImporter.doImport(exportData);
}).then(function (importResult) {
should.exist(importResult);
should.exist(importResult.data);
@ -67,15 +53,14 @@ describe('Import', function () {
it('removes duplicate posts', function (done) {
var exportData;
testUtils.fixtures.loadExportFixture('export-003-duplicate-posts').then(function (exported) {
testUtils.fixtures.loadExportFixture('export-003').then(function (exported) {
exportData = exported;
return importer.doImport(exportData);
return dataImporter.doImport(exportData);
}).then(function (importResult) {
should.exist(importResult.data.data.posts);
should.exist(importResult.data.posts);
importResult.data.data.posts.length.should.equal(1);
importResult.problems.posts.length.should.equal(1);
importResult.data.posts.length.should.equal(1);
importResult.problems.length.should.eql(8);
done();
}).catch(done);
@ -86,21 +71,22 @@ describe('Import', function () {
testUtils.fixtures.loadExportFixture('export-003-duplicate-tags').then(function (exported) {
exportData = exported;
return importer.doImport(exportData);
return dataImporter.doImport(exportData);
}).then(function (importResult) {
should.exist(importResult.data.data.tags);
should.exist(importResult.data.data.posts_tags);
should.exist(importResult.data.tags);
should.exist(importResult.originalData.posts_tags);
importResult.data.data.tags.length.should.equal(1);
importResult.data.tags.length.should.equal(1);
// Check we imported all posts_tags associations
importResult.data.data.posts_tags.length.should.equal(2);
importResult.originalData.posts_tags.length.should.equal(2);
// Check the post_tag.tag_id was updated when we removed duplicate tag
_.every(importResult.data.data.posts_tags, function (postTag) {
_.every(importResult.originalData.posts_tags, function (postTag) {
return postTag.tag_id !== 2;
});
importResult.problems.tags.length.should.equal(1);
importResult.problems.length.should.equal(9);
done();
}).catch(done);
@ -110,15 +96,12 @@ describe('Import', function () {
describe('DataImporter', function () {
beforeEach(testUtils.setup('roles', 'owner', 'settings'));
should.exist(DataImporter);
it('imports data from 000', function (done) {
var exportData;
testUtils.fixtures.loadExportFixture('export-000').then(function (exported) {
exportData = exported;
return importer.doImport(exportData);
return dataImporter.doImport(exportData);
}).then(function () {
// Grab the data from tables
return Promise.all([
@ -160,18 +143,11 @@ describe('Import', function () {
});
it('safely imports data, from 001', function (done) {
var exportData,
timestamp = moment().startOf('day').valueOf(); // no ms
var exportData;
testUtils.fixtures.loadExportFixture('export-001').then(function (exported) {
exportData = exported;
// Modify timestamp data for testing
exportData.data.posts[0].created_at = timestamp;
exportData.data.posts[0].updated_at = timestamp;
exportData.data.posts[0].published_at = timestamp;
return importer.doImport(exportData);
return dataImporter.doImport(exportData);
}).then(function () {
// Grab the data from tables
return Promise.all([
@ -215,9 +191,9 @@ describe('Import', function () {
// in MySQL we're returned a date object.
// We pass the returned post always through the date object
// to ensure the return is consistent for all DBs.
assert.equal(moment(posts[0].created_at).valueOf(), timestamp);
assert.equal(moment(posts[0].updated_at).valueOf(), timestamp);
assert.equal(moment(posts[0].published_at).valueOf(), timestamp);
assert.equal(moment(posts[0].created_at).valueOf(), 1388318310000);
assert.equal(moment(posts[0].updated_at).valueOf(), 1388318310000);
assert.equal(moment(posts[0].published_at).valueOf(), 1388404710000);
done();
}).catch(done);
@ -226,14 +202,12 @@ describe('Import', function () {
it('doesn\'t import invalid settings data from 001', function (done) {
var exportData;
testUtils.fixtures.loadExportFixture('export-001').then(function (exported) {
testUtils.fixtures.loadExportFixture('export-001-invalid-setting').then(function (exported) {
exportData = exported;
// change to blank settings key
exportData.data.settings[3].key = null;
return importer.doImport(exportData);
return dataImporter.doImport(exportData);
}).then(function () {
(1).should.eql(0, 'Data import should not resolve promise.');
}, function (error) {
}).catch(function (error) {
error[0].message.should.eql('Value in [settings.key] cannot be blank.');
error[0].errorType.should.eql('ValidationError');
@ -259,7 +233,7 @@ describe('Import', function () {
done();
});
}).catch(done);
});
});
});
@ -267,18 +241,11 @@ describe('Import', function () {
beforeEach(testUtils.setup('roles', 'owner', 'settings'));
it('safely imports data from 002', function (done) {
var exportData,
timestamp = moment().startOf('day').valueOf(); // no ms
var exportData;
testUtils.fixtures.loadExportFixture('export-002').then(function (exported) {
exportData = exported;
// Modify timestamp data for testing
exportData.data.posts[0].created_at = timestamp;
exportData.data.posts[0].updated_at = timestamp;
exportData.data.posts[0].published_at = timestamp;
return importer.doImport(exportData);
return dataImporter.doImport(exportData);
}).then(function () {
// Grab the data from tables
return Promise.all([
@ -322,51 +289,13 @@ describe('Import', function () {
// in MySQL we're returned a date object.
// We pass the returned post always through the date object
// to ensure the return is consistant for all DBs.
assert.equal(moment(posts[0].created_at).valueOf(), timestamp);
assert.equal(moment(posts[0].updated_at).valueOf(), timestamp);
assert.equal(moment(posts[0].published_at).valueOf(), timestamp);
assert.equal(moment(posts[0].created_at).valueOf(), 1419940710000);
assert.equal(moment(posts[0].updated_at).valueOf(), 1420027110000);
assert.equal(moment(posts[0].published_at).valueOf(), 1420027110000);
done();
}).catch(done);
});
it('doesn\'t import invalid settings data from 002', function (done) {
var exportData;
testUtils.fixtures.loadExportFixture('export-002').then(function (exported) {
exportData = exported;
// change to blank settings key
exportData.data.settings[3].key = null;
return importer.doImport(exportData);
}).then(function () {
(1).should.eql(0, 'Data import should not resolve promise.');
}, function (error) {
error[0].message.should.eql('Value in [settings.key] cannot be blank.');
error[0].errorType.should.eql('ValidationError');
Promise.all([
knex('users').select(),
knex('posts').select(),
knex('tags').select()
]).then(function (importedData) {
should.exist(importedData);
importedData.length.should.equal(3, 'Did not get data successfully');
var users = importedData[0],
posts = importedData[1],
tags = importedData[2];
// we always have 1 user, the owner user we added
users.length.should.equal(1, 'There should only be one user');
// Nothing should have been imported
posts.length.should.equal(0, 'Wrong number of posts');
tags.length.should.equal(0, 'no new tags');
done();
});
}).catch(done);
});
});
describe('003', function () {
@ -377,7 +306,7 @@ describe('Import', function () {
testUtils.fixtures.loadExportFixture('export-003').then(function (exported) {
exportData = exported;
return importer.doImport(exportData);
return dataImporter.doImport(exportData);
}).then(function () {
// Grab the data from tables
return Promise.all([
@ -416,52 +345,88 @@ describe('Import', function () {
testUtils.fixtures.loadExportFixture('export-003-badValidation').then(function (exported) {
exportData = exported;
return importer.doImport(exportData);
return dataImporter.doImport(exportData);
}).then(function () {
done(new Error('Allowed import of duplicate data'));
}).catch(function (response) {
response.length.should.equal(5);
response.length.should.equal(4);
// NOTE: a duplicated tag.slug is a warning
response[0].errorType.should.equal('ValidationError');
response[0].message.should.eql('Value in [posts.title] cannot be blank.');
response[0].message.should.eql('Value in [tags.name] cannot be blank.');
response[1].errorType.should.equal('ValidationError');
response[1].message.should.eql('Value in [posts.slug] cannot be blank.');
response[1].message.should.eql('Value in [posts.title] cannot be blank.');
response[2].errorType.should.equal('ValidationError');
response[2].message.should.eql('Value in [settings.key] cannot be blank.');
response[2].message.should.eql('Value in [tags.name] cannot be blank.');
response[3].errorType.should.equal('ValidationError');
response[3].message.should.eql('Value in [tags.slug] cannot be blank.');
response[4].errorType.should.equal('ValidationError');
response[4].message.should.eql('Value in [tags.name] cannot be blank.');
response[3].message.should.eql('Value in [settings.key] cannot be blank.');
done();
}).catch(done);
});
it('handles database errors nicely', function (done) {
it('handles database errors nicely: duplicated tag slugs', function (done) {
var exportData;
testUtils.fixtures.loadExportFixture('export-003-dbErrors').then(function (exported) {
exportData = exported;
return importer.doImport(exportData);
}).then(function () {
done(new Error('Allowed import of duplicate data'));
}).catch(function (response) {
response.length.should.be.above(0);
response[0].errorType.should.equal('DataImportError');
return dataImporter.doImport(exportData);
}).then(function (importedData) {
importedData.problems.length.should.eql(3);
importedData.problems[0].message.should.eql('Entry was not imported and ignored. Detected duplicated entry.');
importedData.problems[0].help.should.eql('Tag');
importedData.problems[1].message.should.eql('Entry was not imported and ignored. Detected duplicated entry.');
importedData.problems[1].help.should.eql('Tag');
importedData.problems[2].message.should.eql('Entry was not imported and ignored. Detected duplicated entry.');
importedData.problems[2].help.should.eql('Post');
done();
}).catch(done);
});
it('doesn\'t import posts with an invalid author', function (done) {
it('does import posts with an invalid author', function (done) {
var exportData;
testUtils.fixtures.loadExportFixture('export-003-mu-unknownAuthor').then(function (exported) {
exportData = exported;
return dataImporter.doImport(exportData);
}).then(function (importedData) {
// NOTE: we detect invalid author references as warnings, because ember can handle this
// The owner can simply update the author reference in the UI
importedData.problems.length.should.eql(3);
importedData.problems[2].message.should.eql('Entry was imported, but we were not able to update user reference field: published_by');
importedData.problems[2].help.should.eql('Post');
return importer.doImport(exportData);
}).then(function () {
done(new Error('Allowed import of unknown author'));
}).catch(function (response) {
response.length.should.equal(1);
response[0].message.should.eql('Attempting to import data linked to unknown user id 2');
response[0].errorType.should.equal('DataImportError');
// Grab the data from tables
return Promise.all([
knex('users').select(),
knex('posts').select(),
knex('tags').select()
]);
}).then(function (importedData) {
should.exist(importedData);
importedData.length.should.equal(3, 'Did not get data successfully');
var users = importedData[0],
posts = importedData[1],
tags = importedData[2];
// user should still have the credentials from the original insert, not the import
users[0].email.should.equal(testUtils.DataGenerator.Content.users[0].email);
users[0].password.should.equal(testUtils.DataGenerator.Content.users[0].password);
// but the name, slug, and bio should have been overridden
users[0].name.should.equal('Joe Bloggs');
users[0].slug.should.equal('joe-bloggs');
should.not.exist(users[0].bio, 'bio is not imported');
// test posts
posts.length.should.equal(1, 'Wrong number of posts');
// this is just a string and ember can handle unknown authors
// the blog owner is able to simply set a new author
posts[0].author_id.should.eql('2');
// test tags
tags.length.should.equal(0, 'no tags');
done();
}).catch(done);
@ -472,11 +437,7 @@ describe('Import', function () {
testUtils.fixtures.loadExportFixture('export-003-nullTags').then(function (exported) {
exportData = exported;
exportData.data.tags.length.should.be.above(1);
exportData.data.posts_tags.length.should.be.above(1);
return importer.doImport(exportData);
return dataImporter.doImport(exportData);
}).then(function () {
done(new Error('Allowed import of invalid tags data'));
}).catch(function (response) {
@ -484,7 +445,7 @@ describe('Import', function () {
response[0].errorType.should.equal('ValidationError');
response[0].message.should.eql('Value in [tags.name] cannot be blank.');
response[1].errorType.should.equal('ValidationError');
response[1].message.should.eql('Value in [tags.slug] cannot be blank.');
response[1].message.should.eql('Value in [tags.name] cannot be blank.');
done();
}).catch(done);
});
@ -494,14 +455,20 @@ describe('Import', function () {
testUtils.fixtures.loadExportFixture('export-003-nullPosts').then(function (exported) {
exportData = exported;
exportData.data.posts.length.should.be.above(1);
return importer.doImport(exportData);
return dataImporter.doImport(exportData);
}).then(function () {
done(new Error('Allowed import of invalid post data'));
}).catch(function (response) {
response.length.should.equal(5, response);
response.length.should.equal(3, response);
response[0].errorType.should.equal('ValidationError');
response[0].message.should.eql('Value in [posts.title] cannot be blank.');
response[1].errorType.should.equal('ValidationError');
response[1].message.should.eql('Value in [posts.status] cannot be blank.');
response[2].errorType.should.equal('ValidationError');
response[2].message.should.eql('Value in [posts.language] cannot be blank.');
done();
}).catch(done);
});
@ -511,10 +478,7 @@ describe('Import', function () {
testUtils.fixtures.loadExportFixture('export-003-wrongUUID').then(function (exported) {
exportData = exported;
exportData.data.posts.length.should.be.above(0);
return importer.doImport(exportData);
return dataImporter.doImport(exportData);
}).then(function () {
// Grab the data from tables
return knex('posts').select();
@ -542,10 +506,10 @@ describe('Import', function () {
// change title to 1001 characters
exportData.data.posts[0].title = new Array(2002).join('a');
exportData.data.posts[0].tags = 'Tag';
return importer.doImport(exportData);
return dataImporter.doImport(exportData);
}).then(function () {
(1).should.eql(0, 'Data import should not resolve promise.');
}, function (error) {
}).catch(function (error) {
error[0].message.should.eql('Value in [posts.title] exceeds maximum length of 2000 characters.');
error[0].errorType.should.eql('ValidationError');
@ -588,11 +552,12 @@ describe('Import (new test structure)', function () {
return testUtils.fixtures.loadExportFixture('export-003-mu');
}).then(function (exported) {
exportData = exported;
return importer.doImport(exportData);
return dataImporter.doImport(exportData);
}).then(function () {
done();
}).catch(done);
});
after(testUtils.teardown);
it('gets the right data', function (done) {
@ -809,11 +774,12 @@ describe('Import (new test structure)', function () {
return testUtils.fixtures.loadExportFixture('export-003-mu-noOwner');
}).then(function (exported) {
exportData = exported;
return importer.doImport(exportData);
return dataImporter.doImport(exportData);
}).then(function () {
done();
}).catch(done);
});
after(testUtils.teardown);
it('gets the right data', function (done) {
@ -1031,11 +997,12 @@ describe('Import (new test structure)', function () {
return testUtils.fixtures.loadExportFixture('export-003-mu');
}).then(function (exported) {
exportData = exported;
return importer.doImport(exportData);
return dataImporter.doImport(exportData);
}).then(function () {
done();
}).catch(done);
});
after(testUtils.teardown);
it('gets the right data', function (done) {
@ -1259,11 +1226,12 @@ describe('Import (new test structure)', function () {
return testUtils.fixtures.loadExportFixture('export-003-mu-multipleOwner');
}).then(function (exported) {
exportData = exported;
return importer.doImport(exportData);
return dataImporter.doImport(exportData);
}).then(function () {
done();
}).catch(done);
});
after(testUtils.teardown);
it('imports users with correct roles and status', function (done) {

38
core/test/unit/importer_spec.js → core/test/unit/data/importer/index_spec.js
View file

@ -2,22 +2,22 @@ var should = require('should'),
sinon = require('sinon'),
Promise = require('bluebird'),
_ = require('lodash'),
testUtils = require('../utils'),
testUtils = require('../../../utils'),
moment = require('moment'),
path = require('path'),
errors = require('../../server/errors'),
errors = require('../../../../server/errors'),
// Stuff we are testing
ImportManager = require('../../server/data/importer'),
JSONHandler = require('../../server/data/importer/handlers/json'),
ImageHandler = require('../../server/data/importer/handlers/image'),
MarkdownHandler = require('../../server/data/importer/handlers/markdown'),
DataImporter = require('../../server/data/importer/importers/data'),
ImageImporter = require('../../server/data/importer/importers/image'),
ImportManager = require('../../../../server/data/importer'),
JSONHandler = require('../../../../server/data/importer/handlers/json'),
ImageHandler = require('../../../../server/data/importer/handlers/image'),
MarkdownHandler = require('../../../../server/data/importer/handlers/markdown'),
DataImporter = require('../../../../server/data/importer/importers/data'),
ImageImporter = require('../../../../server/data/importer/importers/image'),
storage = require('../../server/adapters/storage'),
storage = require('../../../../server/adapters/storage'),
configUtils = require('../utils/configUtils'),
configUtils = require('../../../utils/configUtils'),
sandbox = sinon.sandbox.create();
describe('Importer', function () {
@ -628,8 +628,6 @@ describe('Importer', function () {
});
describe('DataImporter', function () {
var importer = require('../../server/data/import');
it('has the correct interface', function () {
DataImporter.type.should.eql('data');
DataImporter.preProcess.should.be.instanceof(Function);
@ -637,7 +635,7 @@ describe('Importer', function () {
});
it('does preprocess posts, users and tags correctly', function () {
var inputData = require('../utils/fixtures/import/import-data-1.json'),
var inputData = require('../../../utils/fixtures/import/import-data-1.json'),
outputData = DataImporter.preProcess(_.cloneDeep(inputData));
// Data preprocess is a noop
@ -645,16 +643,6 @@ describe('Importer', function () {
inputData.data.data.tags[0].should.eql(outputData.data.data.tags[0]);
inputData.data.data.users[0].should.eql(outputData.data.data.users[0]);
});
it('does import the data correctly', function () {
var inputData = require('../utils/fixtures/import/import-data-1.json'),
importerSpy = sandbox.stub(importer, 'doImport').returns(Promise.resolve());
DataImporter.doImport(inputData.data).then(function () {
importerSpy.calledOnce.should.be.true();
importerSpy.calledWith(inputData.data).should.be.true();
});
});
});
describe('ImageImporter', function () {
@ -665,7 +653,7 @@ describe('Importer', function () {
});
it('does preprocess posts, users and tags correctly', function () {
var inputData = require('../utils/fixtures/import/import-data-1.json'),
var inputData = require('../../../utils/fixtures/import/import-data-1.json'),
outputData = ImageImporter.preProcess(_.cloneDeep(inputData));
inputData = inputData.data.data;
@ -694,7 +682,7 @@ describe('Importer', function () {
});
it('does import the images correctly', function () {
var inputData = require('../utils/fixtures/import/import-data-1.json'),
var inputData = require('../../../utils/fixtures/import/import-data-1.json'),
storageApi = {
save: sandbox.stub().returns(Promise.resolve())
},

5
core/test/unit/server_utils_spec.js
View file

@ -15,6 +15,11 @@ describe('Server Utilities', function () {
result.should.equal('stringwithspace');
});
it('can handle null strings', function () {
var result = safeString(null);
result.should.equal('');
});
it('should remove non ascii characters', function () {
var result = safeString('howtowin✓', options);
result.should.equal('howtowin');

331
core/test/utils/fixtures/export/export-001-invalid-setting.json Normal file
View file

@ -0,0 +1,331 @@
{
"meta": {
"exported_on": 1388318311015,
"version": "001"
},
"data": {
"posts": [
{
"id": 1,
"uuid": "8492fbba-1102-4b53-8e3e-abe207952f0c",
"title": "Welcome to Ghost",
"slug": "welcome-to-ghost",
"markdown": "You're live! Nice.",
"html": "<p>You're live! Nice.</p>",
"image": null,
"featured": 0,
"page": 0,
"status": "published",
"language": "en_US",
"meta_title": null,
"meta_description": null,
"author_id": 1,
"created_at": 1388318310000,
"created_by": 1,
"updated_at": 1388318310000,
"updated_by": 1,
"published_at": 1388404710000,
"published_by": 1
}
],
"users": [
{
"id": 1,
"uuid": "e5188224-4742-4c32-a2d6-e9c5c5d4c123",
"name": "Joe Bloggs",
"slug": "joe-bloggs",
"password": "$2a$10$.pZeeBE0gHXd0PTnbT/ph.GEKgd0Wd3q2pWna3ynTGBkPKnGIKABC",
"email": "jbloggs@example.com",
"image": null,
"cover": null,
"bio": "A blogger",
"website": null,
"location": null,
"accessibility": null,
"status": "active",
"language": "en_US",
"meta_title": null,
"meta_description": null,
"last_login": null,
"created_at": 1388319501897,
"created_by": 1,
"updated_at": null,
"updated_by": null
}
],
"roles": [
{
"id": 1,
"uuid": "d2ea9c7f-7e6b-4cae-b009-35c298206852",
"name": "Administrator",
"description": "Administrators",
"created_at": 1388318310794,
"created_by": 1,
"updated_at": 1388318310794,
"updated_by": 1
},
{
"id": 2,
"uuid": "b0d7d6b0-5b88-45b5-b0e5-a487741b843d",
"name": "Editor",
"description": "Editors",
"created_at": 1388318310796,
"created_by": 1,
"updated_at": 1388318310796,
"updated_by": 1
},
{
"id": 3,
"uuid": "9f72e817-5490-4ccf-bc78-c557dc9613ca",
"name": "Author",
"description": "Authors",
"created_at": 1388318310799,
"created_by": 1,
"updated_at": 1388318310799,
"updated_by": 1
}
],
"roles_users": [
{
"id": 1,
"role_id": 1,
"user_id": 1
}
],
"permissions": [
{
"id": 1,
"uuid": "bdfbd261-e0fb-4c8e-abab-aece7a9e8e34",
"name": "Edit posts",
"object_type": "post",
"action_type": "edit",
"object_id": null,
"created_at": 1388318310803,
"created_by": 1,
"updated_at": 1388318310803,
"updated_by": 1
},
{
"id": 2,
"uuid": "580d31c4-e3db-40f3-969d-9a1caea9d1bb",
"name": "Remove posts",
"object_type": "post",
"action_type": "remove",
"object_id": null,
"created_at": 1388318310814,
"created_by": 1,
"updated_at": 1388318310814,
"updated_by": 1
},
{
"id": 3,
"uuid": "c1f8b024-e383-494a-835d-6fb673f143db",
"name": "Create posts",
"object_type": "post",
"action_type": "create",
"object_id": null,
"created_at": 1388318310818,
"created_by": 1,
"updated_at": 1388318310818,
"updated_by": 1
}
],
"permissions_users": [],
"permissions_roles": [
{
"id": 1,
"role_id": 1,
"permission_id": 1
},
{
"id": 2,
"role_id": 1,
"permission_id": 2
},
{
"id": 3,
"role_id": 1,
"permission_id": 3
}
],
"settings": [
{
"id": 2,
"uuid": "95ce1c53-69b0-4f5f-be91-d3aeb39046b5",
"key": "dbHash",
"value": null,
"type": "core",
"created_at": 1388318310829,
"created_by": 1,
"updated_at": 1388318310829,
"updated_by": 1
},
{
"id": 3,
"uuid": "c356fbde-0bc5-4fe1-9309-2510291aa34d",
"key": "title",
"value": "Ghost",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 4,
"uuid": "858dc11f-8f9e-4011-99ee-d94c48d5a2ce",
"key": "description",
"value": "Just a blogging platform.",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 5,
"uuid": "37ca5ae7-bca6-4dd5-8021-4ef6c6dcb097",
"key": "email",
"value": "josephinebloggs@example.com",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 6,
"uuid": "1672d62c-fab7-4f22-b333-8cf760189f67",
"key": "logo",
"value": "",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 7,
"uuid": "cd8b0456-578b-467a-857e-551bad17a14d",
"key": "cover",
"value": "",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 8,
"uuid": "c4a074a4-05c7-49f7-83eb-068302c15d82",
"key": "defaultLang",
"value": "en_US",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 9,
"uuid": "21f2f5da-9bee-4dae-b3b7-b8d7baf8be33",
"key": "postsPerPage",
"value": "6",
"type": "blog",
"created_at": 1388318310830,
"created_by": 1,
"updated_at": 1388318310830,
"updated_by": 1
},
{
"id": 10,
"uuid": "2d21b736-f85a-4119-a0e3-5fc898b1bf47",
"key": "forceI18n",
"value": "true",
"type": "blog",
"created_at": 1388318310831,
"created_by": 1,
"updated_at": 1388318310831,
"updated_by": 1
},
{
"id": 11,
"uuid": "5c5b91b8-6062-4104-b855-9e121f72b0f0",
"key": "permalinks",
"value": "/:slug/",
"type": "blog",
"created_at": 1388318310831,
"created_by": 1,
"updated_at": 1388318310831,
"updated_by": 1
},
{
"id": 12,
"uuid": "795cb328-3e38-4906-81a8-fcdff19d914f",
"key": "activeTheme",
"value": "notcasper",
"type": "theme",
"created_at": 1388318310831,
"created_by": 1,
"updated_at": 1388318310831,
"updated_by": 1
},
{
"id": 13,
"uuid": "f3afce35-5166-453e-86c3-50dfff74dca7",
"key": "activeApps",
"value": "[]",
"type": "app",
"created_at": 1388318310831,
"created_by": 1,
"updated_at": 1388318310831,
"updated_by": 1
},
{
"id": 14,
"uuid": "2ea560a3-2304-449d-a62b-f7b622987510",
"key": "installedApps",
"value": "[]",
"type": "app",
"created_at": 1388318310831,
"created_by": 1,
"updated_at": 1388318310831,
"updated_by": 1
},
{
"id": 15,
"uuid": "2ea560a3-2304-449d-a62b-f7b622987510",
"key": null,
"value": "[]",
"type": "app",
"created_at": 1388318310831,
"created_by": 1,
"updated_at": 1388318310831,
"updated_by": 1
}
],
"tags": [
{
"id": 1,
"uuid": "a950117a-9735-4584-931d-25a28015a80d",
"name": "Getting Started",
"slug": "getting-started",
"description": null,
"parent_id": null,
"meta_title": null,
"meta_description": null,
"created_at": 1388318310790,
"created_by": 1,
"updated_at": 1388318310790,
"updated_by": 1
}
],
"posts_tags": [
{
"id": 1,
"post_id": 1,
"tag_id": 1
}
]
}
}

6
core/test/utils/fixtures/export/export-001.json
View file

@ -20,11 +20,11 @@
"meta_title": null,
"meta_description": null,
"author_id": 1,
"created_at": 1388318310782,
"created_at": 1388318310000,
"created_by": 1,
"updated_at": 1388318310782,
"updated_at": 1388318310000,
"updated_by": 1,
"published_at": 1388318310783,
"published_at": 1388404710000,
"published_by": 1
}
],

6
core/test/utils/fixtures/export/export-002.json
View file

@ -20,11 +20,11 @@
"meta_title": null,
"meta_description": null,
"author_id": 1,
"created_at": 1388318310782,
"created_at": 1419940710000,
"created_by": 1,
"updated_at": 1388318310782,
"updated_at": 1420027110000,
"updated_by": 1,
"published_at": 1388318310783,
"published_at": 1420027110000,
"published_by": 1
}
],