Infrastructure/ghost
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2025-03-11 02:12:21 -05:00
Code Issues Activity

🐛 Fixed large imports

Browse source 
closes #9348

- do not run import with `Promise.all`
- with a large import file, we run an enormous amount of queries in parallel, which does not allow Node to cleanup memory
- tested with an 13mb import file
- requires bookshelf-relations 0.1.4
This commit is contained in:
kirrg001 2018-01-02 23:21:45 +01:00
parent fccfa7614d
commit 29e143fa9a
1 changed files with 22 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
core/server/data/importer/importers/data/base.js
View file

@ -2,6 +2,7 @@
const debug = require('ghost-ignition').debug('importer:base'),
common = require('../../../../lib/common'),
sequence = require('../../../../lib/promise/sequence'),
models = require('../../../../models'),
_ = require('lodash'),
Promise = require('bluebird');
@ -152,21 +153,29 @@ class Base {
let self = this, ops = [];
_.each(this.dataToImport, function (obj) {
ops.push(models[self.modelName].add(obj, options)
.then(function (importedModel) {
obj.model = importedModel.toJSON();
self.importedData.push(obj.model);
return importedModel;
})
.catch(function (err) {
return self.handleError(err, obj);
})
.reflect()
);
_.each(this.dataToImport, function forEachDataToImport(obj) {
ops.push(function addModel() {
return models[self.modelName].add(obj, options)
.then(function (importedModel) {
obj.model = importedModel.toJSON();
self.importedData.push(obj.model);
return importedModel;
})
.catch(function (err) {
return self.handleError(err, obj);
})
.reflect();
});
});
return Promise.all(ops);
/**
* NOTE: Do not run with Promise.all in this case. With a large import file, we run an enormous
* amount of queries in parallel. Node will very fast eat lot's of memory, because all queries start
* at the same time, but memory can only be released if the query finished.
*
* Promise.map(.., {concurrency: Int}) was not really improving the end performance for me.
*/
return sequence(ops);
}
/**