0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2025-04-15 03:01:37 -05:00

Add support for importing images

closes #4608, #4609

- image handler loads in any image files & figures out where they'll get stored
- image importer has a preprocessor which replaces image paths in
  pertinent spots of post, tag and user models
- image importer stores images, keeping the path where it makes sense
- basic test for the preprocessor
This commit is contained in:
Hannah Wolfe 2014-12-14 11:31:00 +00:00
parent 0af2bc646e
commit ba3d4b3689
26 changed files with 461 additions and 35 deletions

View file

@ -0,0 +1,47 @@
var _ = require('lodash'),
Promise = require('bluebird'),
path = require('path'),
config = require('../../../config'),
storage = require('../../../storage'),
ImageHandler;
ImageHandler = {
type: 'images',
extensions: config.uploads.extensions,
types: config.uploads.contentTypes,
loadFile: function (files, startDir) {
var store = storage.getStorage(),
startDirRegex = startDir ? new RegExp('^' + startDir + '/') : new RegExp(''),
imageFolderRegexes = _.map(config.paths.imagesRelPath.split('/'), function (dir) {
return new RegExp('^' + dir + '/');
});
// normalize the directory structure
files = _.map(files, function (file) {
var noStartDir = file.name.replace(startDirRegex, ''),
noGhostDirs = noStartDir;
_.each(imageFolderRegexes, function (regex) {
noGhostDirs = noGhostDirs.replace(regex, '');
});
file.originalPath = noStartDir;
file.name = noGhostDirs;
file.targetDir = path.join(config.paths.imagesPath, path.dirname(noGhostDirs));
return file;
});
return Promise.map(files, function (image) {
return store.getUniqueFileName(store, image, image.targetDir).then(function (targetFilename) {
image.newPath = (config.paths.subdir + '/' +
config.paths.imagesRelPath + '/' + path.relative(config.paths.imagesPath, targetFilename))
.replace(new RegExp('\\' + path.sep, 'g'), '/');
return image;
});
});
}
};
module.exports = ImageHandler;

View file

@ -0,0 +1,73 @@
var _ = require('lodash'),
Promise = require('bluebird'),
storage = require('../../../storage'),
replaceImage,
ImageImporter,
preProcessPosts,
preProcessTags,
preProcessUsers;
replaceImage = function (markdown, image) {
// Normalizes to include a trailing slash if there was one
var regex = new RegExp('(/)?' + image.originalPath, 'gm');
return markdown.replace(regex, image.newPath);
};
preProcessPosts = function (data, image) {
_.each(data.posts, function (post) {
post.markdown = replaceImage(post.markdown, image);
if (post.html) {
post.html = replaceImage(post.html, image);
}
if (post.image) {
post.image = replaceImage(post.image, image);
}
});
};
preProcessTags = function (data, image) {
_.each(data.tags, function (tag) {
if (tag.image) {
tag.image = replaceImage(tag.image, image);
}
});
};
preProcessUsers = function (data, image) {
_.each(data.users, function (user) {
if (user.cover) {
user.cover = replaceImage(user.cover, image);
}
if (user.image) {
user.image = replaceImage(user.image, image);
}
});
};
ImageImporter = {
type: 'images',
preProcess: function (importData) {
if (importData.images && importData.data) {
_.each(importData.images, function (image) {
preProcessPosts(importData.data.data, image);
preProcessTags(importData.data.data, image);
preProcessUsers(importData.data.data, image);
});
}
importData.preProcessedByImage = true;
return importData;
},
doImport: function (imageData) {
var store = storage.getStorage();
return Promise.map(imageData, function (image) {
return store.save(image, image.targetDir).then(function (result) {
return {originalPath: image.originalPath, newPath: image.newPath, stored: result};
});
});
}
};
module.exports = ImageImporter;

View file

@ -9,8 +9,10 @@ var _ = require('lodash'),
uuid = require('node-uuid'),
extract = require('extract-zip'),
errors = require('../../errors'),
JSONHandler = require('./handlers/json'),
DataImporter = require('./importers/data'),
ImageHandler = require('./handlers/image'),
JSONHandler = require('./handlers/json'),
ImageImporter = require('./importers/image'),
DataImporter = require('./importers/data'),
defaults;
@ -20,8 +22,8 @@ defaults = {
};
function ImportManager() {
this.importers = [DataImporter];
this.handlers = [JSONHandler];
this.importers = [ImageImporter, DataImporter];
this.handlers = [ImageHandler, JSONHandler];
}
/**

View file

@ -19,9 +19,9 @@ util.inherits(LocalFileStore, baseStore);
// Saves the image to storage (the file system)
// - image is the express image object
// - returns a promise which ultimately returns the full url to the uploaded image
LocalFileStore.prototype.save = function (image) {
var targetDir = this.getTargetDir(config.paths.imagesPath),
targetFilename;
LocalFileStore.prototype.save = function (image, targetDir) {
targetDir = targetDir || this.getTargetDir(config.paths.imagesPath);
var targetFilename;
return this.getUniqueFileName(this, image, targetDir).then(function (filename) {
targetFilename = filename;

View file

@ -1,4 +1,4 @@
/*globals describe, it, before, beforeEach, afterEach */
/*globals describe, it, before, beforeEach, afterEach, after */
/*jshint expr:true*/
var should = require('should'),
sinon = require('sinon'),
@ -13,13 +13,22 @@ var should = require('should'),
// Thing we are testing
defaultConfig = require('../../../config.example')[process.env.NODE_ENV],
config = require('../../server/config'),
origConfig = _.cloneDeep(config),
// storing current environment
currentEnv = process.env.NODE_ENV;
// To stop jshint complaining
should.equal(true, true);
function resetConfig() {
config.set(_.merge({}, origConfig, defaultConfig));
}
describe('Config', function () {
after(function () {
resetConfig();
});
describe('Theme', function () {
beforeEach(function () {
config.set({
@ -34,7 +43,7 @@ describe('Config', function () {
});
afterEach(function () {
config.set(_.merge({}, defaultConfig));
resetConfig();
});
it('should have exactly the right keys', function () {
@ -61,7 +70,7 @@ describe('Config', function () {
// Make a copy of the default config file
// so we can restore it after every test.
// Using _.merge to recursively apply every property.
config.set(_.merge({}, config));
resetConfig();
});
it('should have exactly the right keys', function () {
@ -140,11 +149,11 @@ describe('Config', function () {
describe('urlFor', function () {
before(function () {
config.set(_.merge({}, defaultConfig));
resetConfig();
});
afterEach(function () {
config.set({url: defaultConfig.url});
resetConfig();
});
it('should return the home url with no options', function () {
@ -274,6 +283,7 @@ describe('Config', function () {
afterEach(function () {
config = rewire('../../server/config');
resetConfig();
sandbox.restore();
});

View file

@ -10,8 +10,11 @@ var assert = require('assert'),
// Stuff we are testing
api = require('../../server/api'),
config = rewire('../../server/config'),
frontend = rewire('../../server/controllers/frontend');
frontend = rewire('../../server/controllers/frontend'),
config = require('../../server/config'),
origConfig = _.cloneDeep(config),
defaultConfig = require('../../../config.example')[process.env.NODE_ENV];
// To stop jshint complaining
should.equal(true, true);
@ -19,7 +22,11 @@ should.equal(true, true);
describe('Frontend Controller', function () {
var sandbox,
apiSettingsStub,
adminEditPagePath = '/ghost/editor/';
adminEditPagePath = '/ghost/editor/',
resetConfig = function () {
config.set(_.merge({}, origConfig, defaultConfig));
};
beforeEach(function () {
sandbox = sinon.sandbox.create();
@ -29,6 +36,7 @@ describe('Frontend Controller', function () {
});
afterEach(function () {
resetConfig();
sandbox.restore();
});
@ -63,6 +71,12 @@ describe('Frontend Controller', function () {
}));
});
afterEach(function () {
config.set({paths: origConfig.paths});
frontend.__set__('config', {paths: origConfig.paths});
sandbox.restore();
});
it('Redirects to home if page number is -1', function () {
var req = {params: {page: -1}, route: {path: '/page/:page/'}};
@ -1294,11 +1308,7 @@ describe('Frontend Controller', function () {
describe('rss redirects', function () {
var res,
apiUsersStub,
overwriteConfig = function (newConfig) {
var existingConfig = frontend.__get__('config');
config.set(_.extend(existingConfig, newConfig));
};
apiUsersStub;
beforeEach(function () {
res = {
@ -1365,7 +1375,7 @@ describe('Frontend Controller', function () {
});
it('Redirects to home if page number is 0 with subdirectory', function () {
overwriteConfig({paths: {subdir: '/blog'}});
config.set({url: 'http://testurl.com/blog'});
var req = {params: {page: 0}, route: {path: '/rss/:page/'}};
@ -1377,7 +1387,7 @@ describe('Frontend Controller', function () {
});
it('Redirects to home if page number is 1 with subdirectory', function () {
overwriteConfig({paths: {subdir: '/blog'}});
config.set({url: 'http://testurl.com/blog'});
var req = {params: {page: 1}, route: {path: '/rss/:page/'}};
@ -1389,7 +1399,7 @@ describe('Frontend Controller', function () {
});
it('Redirects to last page if page number too big', function (done) {
overwriteConfig({paths: {subdir: ''}});
config.set({url: 'http://testurl.com/'});
var req = {params: {page: 4}, route: {path: '/rss/:page/'}};
@ -1402,7 +1412,7 @@ describe('Frontend Controller', function () {
});
it('Redirects to last page if page number too big with subdirectory', function (done) {
overwriteConfig({paths: {subdir: '/blog'}});
config.set({url: 'http://testurl.com/blog'});
var req = {params: {page: 4}, route: {path: '/rss/:page/'}};

View file

@ -5,13 +5,17 @@ var should = require('should'),
Promise = require('bluebird'),
_ = require('lodash'),
testUtils = require('../utils'),
config = require('../../server/config'),
// Stuff we are testing
ImportManager = require('../../server/data/importer'),
JSONHandler = require('../../server/data/importer/handlers/json'),
ImageHandler = require('../../server/data/importer/handlers/image'),
DataImporter = require('../../server/data/importer/importers/data'),
ImageImporter = require('../../server/data/importer/importers/image'),
sandbox = sinon.sandbox.create();
sandbox = sinon.sandbox.create(),
storage = require('../../server/storage');
// To stop jshint complaining
should.equal(true, true);
@ -23,8 +27,8 @@ describe('Importer', function () {
describe('ImportManager', function () {
it('has the correct interface', function () {
ImportManager.handlers.should.be.instanceof(Array).and.have.lengthOf(1);
ImportManager.importers.should.be.instanceof(Array).and.have.lengthOf(1);
ImportManager.handlers.should.be.instanceof(Array).and.have.lengthOf(2);
ImportManager.importers.should.be.instanceof(Array).and.have.lengthOf(2);
ImportManager.loadFile.should.be.instanceof(Function);
ImportManager.preProcess.should.be.instanceof(Function);
ImportManager.doImport.should.be.instanceof(Function);
@ -32,13 +36,14 @@ describe('Importer', function () {
});
it('gets the correct extensions', function () {
ImportManager.getExtensions().should.be.instanceof(Array).and.have.lengthOf(2);
ImportManager.getExtensions().should.be.instanceof(Array).and.have.lengthOf(8);
ImportManager.getExtensions().should.containEql('.json');
ImportManager.getExtensions().should.containEql('.zip');
ImportManager.getExtensions().should.containEql('.jpg');
});
it('gets the correct types', function () {
ImportManager.getTypes().should.be.instanceof(Array).and.have.lengthOf(4);
ImportManager.getTypes().should.be.instanceof(Array).and.have.lengthOf(8);
ImportManager.getTypes().should.containEql('application/octet-stream');
ImportManager.getTypes().should.containEql('application/json');
ImportManager.getTypes().should.containEql('application/zip');
@ -85,14 +90,19 @@ describe('Importer', function () {
testZip = {name: 'myFile.zip', path: '/my/path/myFile.zip'},
// need to stub out the extract and glob function for zip
extractSpy = sandbox.stub(ImportManager, 'extractZip').returns(Promise.resolve('/tmp/dir/')),
getFileSpy = sandbox.stub(ImportManager, 'getFilesFromZip').returns(['/tmp/dir/myFile.json']),
getFileSpy = sandbox.stub(ImportManager, 'getFilesFromZip'),
jsonSpy = sandbox.stub(JSONHandler, 'loadFile').returns(Promise.resolve({posts: []})),
imageSpy = sandbox.stub(ImageHandler, 'loadFile'),
cleanSpy = sandbox.stub(ImportManager, 'cleanUp').returns(Promise.resolve());
getFileSpy.withArgs(JSONHandler).returns(['/tmp/dir/myFile.json']);
getFileSpy.withArgs(ImageHandler).returns([]);
ImportManager.processZip(testZip).then(function (zipResult) {
extractSpy.calledOnce.should.be.true;
getFileSpy.calledOnce.should.be.true;
getFileSpy.calledTwice.should.be.true;
jsonSpy.calledOnce.should.be.true;
imageSpy.called.should.be.false;
cleanSpy.calledOnce.should.be.true;
ImportManager.processFile(testFile, '.json').then(function (fileResult) {
@ -140,17 +150,24 @@ describe('Importer', function () {
dataSpy = sandbox.stub(DataImporter, 'doImport', function (i) {
return Promise.resolve(i);
}),
imageSpy = sandbox.stub(ImageImporter, 'doImport', function (i) {
return Promise.resolve(i);
}),
// The data importer should get the data object
expect = input.data;
expectedData = input.data,
expectedImages = input.images;
ImportManager.doImport(inputCopy).then(function (output) {
// eql checks for equality
// equal checks the references are for the same object
dataSpy.calledOnce.should.be.true;
dataSpy.getCall(0).args[0].should.eql(expect);
imageSpy.calledOnce.should.be.true;
dataSpy.getCall(0).args[0].should.eql(expectedData);
imageSpy.getCall(0).args[0].should.eql(expectedImages);
// we stubbed this as a noop but ImportManager calls with sequence, so we should get an array
output.should.eql([expect]);
output.should.eql([expectedImages, expectedData]);
done();
});
});
@ -208,6 +225,155 @@ describe('Importer', function () {
});
});
describe('ImageHandler', function () {
var origConfig = _.cloneDeep(config),
storage = require('../../server/storage'),
store = storage.getStorage();
afterEach(function () {
config.set(_.merge({}, origConfig));
});
it('has the correct interface', function () {
ImageHandler.type.should.eql('images');
ImageHandler.extensions.should.be.instanceof(Array).and.have.lengthOf(6);
ImageHandler.extensions.should.containEql('.jpg');
ImageHandler.extensions.should.containEql('.jpeg');
ImageHandler.extensions.should.containEql('.gif');
ImageHandler.extensions.should.containEql('.png');
ImageHandler.extensions.should.containEql('.svg');
ImageHandler.extensions.should.containEql('.svgz');
ImageHandler.types.should.be.instanceof(Array).and.have.lengthOf(4);
ImageHandler.types.should.containEql('image/jpeg');
ImageHandler.types.should.containEql('image/png');
ImageHandler.types.should.containEql('image/gif');
ImageHandler.types.should.containEql('image/svg+xml');
ImageHandler.loadFile.should.be.instanceof(Function);
});
it('can load a single file', function (done) {
var filename = 'test-image.jpeg',
file = [{
path: '/my/test/' + filename,
name: filename
}],
storeSpy = sandbox.spy(store, 'getUniqueFileName'),
storageSpy = sandbox.spy(storage, 'getStorage');
ImageHandler.loadFile(_.clone(file)).then(function () {
storageSpy.calledOnce.should.be.true;
storeSpy.calledOnce.should.be.true;
storeSpy.firstCall.args[1].originalPath.should.equal('test-image.jpeg');
storeSpy.firstCall.args[1].targetDir.should.match(/\/content\/images$/);
storeSpy.firstCall.args[1].newPath.should.eql('/content/images/test-image.jpeg');
done();
});
});
it('can load a single file, maintaining structure', function (done) {
var filename = 'photos/my-cat.jpeg',
file = [{
path: '/my/test/' + filename,
name: filename
}],
storeSpy = sandbox.spy(store, 'getUniqueFileName'),
storageSpy = sandbox.spy(storage, 'getStorage');
ImageHandler.loadFile(_.clone(file)).then(function () {
storageSpy.calledOnce.should.be.true;
storeSpy.calledOnce.should.be.true;
storeSpy.firstCall.args[1].originalPath.should.equal('photos/my-cat.jpeg');
storeSpy.firstCall.args[1].targetDir.should.match(/\/content\/images\/photos$/);
storeSpy.firstCall.args[1].newPath.should.eql('/content/images/photos/my-cat.jpeg');
done();
});
});
it('can load a single file, removing ghost dirs', function (done) {
var filename = 'content/images/my-cat.jpeg',
file = [{
path: '/my/test/content/images/' + filename,
name: filename
}],
storeSpy = sandbox.spy(store, 'getUniqueFileName'),
storageSpy = sandbox.spy(storage, 'getStorage');
ImageHandler.loadFile(_.clone(file)).then(function () {
storageSpy.calledOnce.should.be.true;
storeSpy.calledOnce.should.be.true;
storeSpy.firstCall.args[1].originalPath.should.equal('content/images/my-cat.jpeg');
storeSpy.firstCall.args[1].targetDir.should.match(/\/content\/images$/);
storeSpy.firstCall.args[1].newPath.should.eql('/content/images/my-cat.jpeg');
done();
});
});
it('can load a file (subdirectory)', function (done) {
config.set({url: 'http://testurl.com/subdir'});
var filename = 'test-image.jpeg',
file = [{
path: '/my/test/' + filename,
name: filename
}],
storeSpy = sandbox.spy(store, 'getUniqueFileName'),
storageSpy = sandbox.spy(storage, 'getStorage');
ImageHandler.loadFile(_.clone(file)).then(function () {
storageSpy.calledOnce.should.be.true;
storeSpy.calledOnce.should.be.true;
storeSpy.firstCall.args[1].originalPath.should.equal('test-image.jpeg');
storeSpy.firstCall.args[1].targetDir.should.match(/\/content\/images$/);
storeSpy.firstCall.args[1].newPath.should.eql('/subdir/content/images/test-image.jpeg');
done();
});
});
it('can load multiple files', function (done) {
var files = [{
path: '/my/test/testing.png',
name: 'testing.png'
},
{
path: '/my/test/photo/kitten.jpg',
name: 'photo/kitten.jpg'
},
{
path: '/my/test/content/images/animated/bunny.gif',
name: 'content/images/animated/bunny.gif'
},
{
path: '/my/test/images/puppy.jpg',
name: 'images/puppy.jpg'
}],
storeSpy = sandbox.spy(store, 'getUniqueFileName'),
storageSpy = sandbox.spy(storage, 'getStorage');
ImageHandler.loadFile(_.clone(files)).then(function () {
storageSpy.calledOnce.should.be.true;
storeSpy.callCount.should.eql(4);
storeSpy.firstCall.args[1].originalPath.should.equal('testing.png');
storeSpy.firstCall.args[1].targetDir.should.match(/\/content\/images$/);
storeSpy.firstCall.args[1].newPath.should.eql('/content/images/testing.png');
storeSpy.secondCall.args[1].originalPath.should.equal('photo/kitten.jpg');
storeSpy.secondCall.args[1].targetDir.should.match(/\/content\/images\/photo$/);
storeSpy.secondCall.args[1].newPath.should.eql('/content/images/photo/kitten.jpg');
storeSpy.thirdCall.args[1].originalPath.should.equal('content/images/animated/bunny.gif');
storeSpy.thirdCall.args[1].targetDir.should.match(/\/content\/images\/animated$/);
storeSpy.thirdCall.args[1].newPath.should.eql('/content/images/animated/bunny.gif');
storeSpy.lastCall.args[1].originalPath.should.equal('images/puppy.jpg');
storeSpy.lastCall.args[1].targetDir.should.match(/\/content\/images$/);
storeSpy.lastCall.args[1].newPath.should.eql('/content/images/puppy.jpg');
done();
});
});
});
describe('DataImporter', function () {
it('has the correct interface', function () {
DataImporter.type.should.eql('data');
@ -215,4 +381,45 @@ describe('Importer', function () {
DataImporter.doImport.should.be.instanceof(Function);
});
});
describe('ImageImporter', function () {
it('has the correct interface', function () {
ImageImporter.type.should.eql('images');
ImageImporter.preProcess.should.be.instanceof(Function);
ImageImporter.doImport.should.be.instanceof(Function);
});
it('does preprocess posts correctly', function () {
var inputData = require('../utils/fixtures/import/import-data-1.json'),
outputData = ImageImporter.preProcess(_.cloneDeep(inputData));
inputData.data.data.posts[0].markdown.should.not.containEql('/content/images/my-image.png');
inputData.data.data.posts[0].html.should.not.containEql('/content/images/my-image.png');
outputData.data.data.posts[0].markdown.should.containEql('/content/images/my-image.png');
outputData.data.data.posts[0].html.should.containEql('/content/images/my-image.png');
inputData.data.data.posts[0].markdown.should.not.containEql('/content/images/photos/cat.jpg');
inputData.data.data.posts[0].html.should.not.containEql('/content/images/photos/cat.jpg');
outputData.data.data.posts[0].markdown.should.containEql('/content/images/photos/cat.jpg');
outputData.data.data.posts[0].html.should.containEql('/content/images/photos/cat.jpg');
inputData.data.data.posts[0].image.should.eql('/images/my-image.png');
outputData.data.data.posts[0].image.should.eql('/content/images/my-image.png');
});
it('does import the images correctly', function () {
var inputData = require('../utils/fixtures/import/import-data-1.json'),
storageApi = {
save: sandbox.stub().returns(Promise.resolve())
},
storageSpy = sandbox.stub(storage, 'getStorage', function () {
return storageApi;
});
ImageImporter.doImport(inputData.images).then(function () {
storageSpy.calledOnce.should.be.true;
storageApi.save.calledTwice.should.be.true;
});
});
});
});

View file

@ -0,0 +1,73 @@
{
"images": [
{
"name": "my-image.png",
"path": "/tmp/test/path/images/my-image.png",
"originalPath": "images/my-image.png",
"targetDir": "/test/content/images",
"newPath": "/content/images/my-image.png"
},
{
"name": "photos/cat.jpg",
"path": "/tmp/test/path/images/photos/cat.jpg",
"originalPath": "images/photos/cat.jpg",
"targetDir": "/test/content/images/photos",
"newPath": "/content/images/photos/cat.jpg"
}
],
"data": {
"meta": {
"exported_on": 1388318311015,
"version": "000"
},
"data": {
"posts": [
{
"id": 1,
"uuid": "8492fbba-1102-4b53-8e3e-abe207952f0c",
"title": "Welcome to Ghost",
"slug": "welcome-to-ghost",
"markdown": "You're live! Nice. \n![Cat Photo](/images/photos/cat.jpg)\n\n![Cat Photo](/images/photos/cat.jpg)\n![My pic](/images/my-image.png)",
"html": "<p>You're live! Nice. <br>\n<img src=\"/images/photos/cat.jpg\" alt=\"Cat Photo\"></p>\n\n<p><img src=\"/images/photos/cat.jpg\" alt=\"Cat Photo\">\n<img src=\"/images/my-image.png\" alt=\"My pic\"></p>",
"image": "/images/my-image.png",
"featured": 0,
"page": 0,
"status": "published",
"language": "en_US",
"meta_title": null,
"meta_description": null,
"author_id": 1,
"created_at": 1388318310782,
"created_by": 1,
"updated_at": 1388318310782,
"updated_by": 1,
"published_at": 1388318310783,
"published_by": 1
}
],
"tags": [
{
"id": 1,
"uuid": "a950117a-9735-4584-931d-25a28015a80d",
"name": "Getting Started",
"slug": "getting-started",
"description": null,
"parent_id": null,
"meta_title": null,
"meta_description": null,
"created_at": 1388318310790,
"created_by": 1,
"updated_at": 1388318310790,
"updated_by": 1
}
],
"posts_tags": [
{
"id": 1,
"post_id": 1,
"tag_id": 1
}
]
}
}
}

View file

@ -280,8 +280,12 @@ fixtures = {
});
},
getImportFixturePath: function (filename) {
return path.resolve(__dirname + '/fixtures/import/' + filename);
},
getExportFixturePath: function (filename) {
return path.resolve(__dirname + '/fixtures/' + filename + '.json');
return path.resolve(__dirname + '/fixtures/export/' + filename + '.json');
},
loadExportFixture: function loadExportFixture(filename) {