From 571333bb5d3cbeb86776c31c9337b2da173c9813 Mon Sep 17 00:00:00 2001 From: Hannah Wolfe Date: Wed, 18 Sep 2013 15:02:23 +0100 Subject: [PATCH] Temporary Importer - seems my very late night commit was a bit screwed. --- core/server/data/import/001.js | 54 --------- core/server/data/import/002.js | 116 ------------------- core/server/data/import/temp.js | 195 ++++++++++++++++++++++++++++++++ 3 files changed, 195 insertions(+), 170 deletions(-) delete mode 100644 core/server/data/import/001.js delete mode 100644 core/server/data/import/002.js create mode 100644 core/server/data/import/temp.js diff --git a/core/server/data/import/001.js b/core/server/data/import/001.js deleted file mode 100644 index fada8687f2..0000000000 --- a/core/server/data/import/001.js +++ /dev/null @@ -1,54 +0,0 @@ -var when = require("when"), - _ = require("underscore"), - knex = require('../../models/base').Knex, - Importer001; - -Importer001 = function () { - _.bindAll(this, "importFrom001"); - - this.version = "001"; - - this.importFrom = { - "001": this.importFrom001 - }; -}; - -Importer001.prototype.importData = function (data) { - return this.canImport(data) - .then(function (importerFunc) { - return importerFunc(data); - }, function (reason) { - return when.reject(reason); - }); -}; - -Importer001.prototype.canImport = function (data) { - if (data.meta && data.meta.version && this.importFrom[data.meta.version]) { - return when.resolve(this.importFrom[data.meta.version]); - } - - return when.reject("Unsupported version of data"); -}; - -Importer001.prototype.importFrom001 = function (data) { - var insertOps = []; - - _.each(data.data, function (tableData, name) { - if (tableData && tableData.length) { - insertOps.push(knex(name).insert(tableData)); - } - }); - - return when.all(insertOps).then(function (results) { - return when.resolve(results); - }, function (err) { - console.log("Error inserting imported data: ", err.message || err, err.stack); - }); -}; - -module.exports = { - Importer001: Importer001, - importData: function (data) { - new Importer001().importData(data); - } -}; \ No newline at end of file diff --git a/core/server/data/import/002.js b/core/server/data/import/002.js deleted file mode 100644 index 067057d336..0000000000 --- a/core/server/data/import/002.js +++ /dev/null @@ -1,116 +0,0 @@ -var when = require("when"), - _ = require("underscore"), - knex = require('../../models/base').Knex, - errors = require('../../errorHandling'), - Importer002; - -function stripProperties(properties, data) { - _.each(data, function (obj) { - _.each(properties, function (property) { - delete obj[property]; - }); - }); - return data; -} - -Importer002 = function () { - _.bindAll(this, "basicImport"); - - this.version = "002"; - - this.importFrom = { - "001": this.basicImport, - "002": this.basicImport - }; -}; - -Importer002.prototype.importData = function (data) { - return this.canImport(data) - .then(function (importerFunc) { - return importerFunc(data); - }, function (reason) { - return when.reject(reason); - }); -}; - -Importer002.prototype.canImport = function (data) { - if (data.meta && data.meta.version && this.importFrom[data.meta.version]) { - return when.resolve(this.importFrom[data.meta.version]); - } - - return when.reject("Unsupported version of data"); -}; - -// No data needs modifying, we just import whatever tables are available -Importer002.prototype.basicImport = function (data) { - var ops = []; - - _.each(data.data, function (tableData, name) { - - switch (name) { - case 'posts': - // we want to import all posts as new posts for now - // TODO: eventually we should be smart about posts which have the same title & content - // so that we don't create duplicates - if (tableData && tableData.length) { - tableData = stripProperties(['id'], tableData); - ops.push(knex(name).insert(tableData)); - } - break; - case 'users': - // the current data model should only ever have one user. - // So we update the current one with the first one from the imported data - if (tableData && tableData.length) { - tableData = stripProperties(['id'], tableData); - ops.push(knex(name).where('id', 1) - .update(tableData[0])); - } - break; - case 'settings': - // for settings we need to update individual settings, and insert any missing ones - // the one setting we MUST NOT update is the databaseVersion settings - var blackList = ['databaseVersion']; - if (tableData && tableData.length) { - tableData = stripProperties(['id'], tableData); - _.each(tableData, function (data) { - if (blackList.indexOf(data.key) === -1) { - ops.push(knex(name).where('key', data.key) - .update(data).then(function (success) { - // if no lines were updated then we need to insert instead - return success === 0 ? knex(name).insert(data) : when.resolve(success); - })); - } - }); - } - - break; - case 'permissions': - case 'roles': - case 'permissions_roles': - case 'permissions_users': - case 'roles_users': - // do nothing with these tables, the data shouldn't have changed from the fixtures - break; - default: - // any other tables, if they have data, remove the primary key and insert it - if (tableData && tableData.length) { - tableData = stripProperties(['id'], tableData); - ops.push(knex(name).insert(tableData)); - } - break; - } - }); - - return when.all(ops).then(function (results) { - return when.resolve(results); - }, function (err) { - return when.reject("Error importing data: " + err.message || err, err.stack); - }); -}; - -module.exports = { - Importer002: Importer002, - importData: function (data) { - return new Importer002().importData(data); - } -}; \ No newline at end of file diff --git a/core/server/data/import/temp.js b/core/server/data/import/temp.js new file mode 100644 index 0000000000..5e87f628b7 --- /dev/null +++ b/core/server/data/import/temp.js @@ -0,0 +1,195 @@ +var when = require("when"), + _ = require("underscore"), + models = require('../../models'), + errors = require('../../errorHandling'), + TempImporter; + + +TempImporter = function () { + _.bindAll(this, "basicImport"); + + this.version = "003"; + + this.importFrom = { + "001": this.basicImport, + "002": this.basicImport + }; +}; + +TempImporter.prototype.importData = function (data) { + return this.canImport(data) + .then(function (importerFunc) { + return importerFunc(data); + }, function (reason) { + return when.reject(reason); + }); +}; + +TempImporter.prototype.canImport = function (data) { + if (data.meta && data.meta.version && this.importFrom[data.meta.version]) { + return when.resolve(this.importFrom[data.meta.version]); + } + + return when.reject("Unsupported version of data"); +}; + + +function stripProperties(properties, data) { + _.each(data, function (obj) { + _.each(properties, function (property) { + delete obj[property]; + }); + }); + return data; +} + +function defaultProperty(obj, name, defaultVal) { + if (!obj[name]) { + obj[name] = defaultVal; + } + + return obj; +} + +function preProcessPostTags(tableData) { + var postTags, + postsWithTags = {}; + + + postTags = tableData.posts_tags; + _.each(postTags, function (post_tag) { + if (!postsWithTags.hasOwnProperty(post_tag.post_id)) { + postsWithTags[post_tag.post_id] = []; + } + postsWithTags[post_tag.post_id].push(post_tag.tag_id); + }); + + _.each(postsWithTags, function (tag_ids, post_id) { + var post, tags; + post = _.find(tableData.posts, function (post) { + return post.id === parseInt(post_id, 10); + }); + if (post) { + tags = _.filter(tableData.tags, function (tag) { + return _.indexOf(tag_ids, tag.id) !== -1; + }); + post.tags = []; + _.each(tags, function (tag) { + // names are unique.. this should get the right tags added + // as long as tags are added first; + post.tags.push({name: tag.name}); + }); + } + }); + + return tableData; +} + +function importTags(ops, tableData) { + tableData = stripProperties(['id', 'meta_keywords'], tableData); + _.each(tableData, function (tag) { + ops.push(models.Tag.add(tag)); + }); +} + +function importPosts(ops, tableData) { + tableData = stripProperties(['id'], tableData); + + _.each(tableData, function (post) { + post = defaultProperty(post, "language", "en_US"); + post = defaultProperty(post, "featured", false); + post.page = false; + post.markdown = post.content_raw; + post.html = post.content; + post.status = post.status || 'draft'; + post.created_by = post.created_by || 1; + + ops.push(models.Post.add(post)); + }); +} + +function importUsers(ops, tableData) { + _.each(tableData, function (user) { + user.name = user.full_name || 'Joe Bloggs'; + user.email = user.email_address; + user.website = user.url; + user.language = 'en_US'; + user.status = 'active'; + user.image = user.profile_picture; + user.cover = user.cover_picture; + user.created_by = user.created_by || 1; + }); + + tableData = stripProperties(['id', 'full_name', 'email_address', 'url'], tableData); + tableData[0].id = 1; + + ops.push(models.User.edit(tableData[0])); +} + +function importSettings(ops, tableData) { + // for settings we need to update individual settings, and insert any missing ones + // the one setting we MUST NOT update is the databaseVersion settings + var blackList = ['databaseVersion', 'currentVersion', 'language', 'url']; + tableData = stripProperties(['id'], tableData); + tableData = _.filter(tableData, function (data) { + return blackList.indexOf(data.key) === -1; + }); + + _.each(tableData, function (setting) { + if (setting.key === 'activeTheme' && setting.value.indexOf('/')) { + setting.value = setting.value.substring(setting.value.lastIndexOf('/') + 1); + } + setting.type = setting.type || 'custom'; + }); + + ops.push(models.Settings.edit(tableData)); +} + +// No data needs modifying, we just import whatever tables are available +TempImporter.prototype.basicImport = function (data) { + var ops = [], + tableData = data.data; + + // Do any pre-processing of relationships (we can't depend on ids) + if (tableData.posts_tags && tableData.posts && tableData.tags) { + tableData = preProcessPostTags(tableData); + } + + // Import things in the right order: + if (tableData.tags && tableData.tags.length) { + importTags(ops, tableData.tags); + } + + if (tableData.posts && tableData.posts.length) { + importPosts(ops, tableData.posts); + } + + if (tableData.users && tableData.users.length) { + importUsers(ops, tableData.users); + } + + if (tableData.settings && tableData.settings.length) { + importSettings(ops, tableData.settings); + } + + /** do nothing with these tables, the data shouldn't have changed from the fixtures + * permissions + * roles + * permissions_roles + * permissions_users + * roles_users + */ + + return when.all(ops).then(function (results) { + return when.resolve(results); + }, function (err) { + return when.reject("Error importing data: " + err.message || err, err.stack); + }); +}; + +module.exports = { + TempImporter: TempImporter, + importData: function (data) { + return new TempImporter().importData(data); + } +}; \ No newline at end of file