Add ghost-backup client to trigger export (#8911)

no issue
- adds a ghost-backup client
- adds a client authenticated endpoint to export blog for ghost-backup client only
- allows some additional overrides during import
- allows for an import by file to override locking a user and double hashing the password

core/server/api/db.js

@@ -7,6 +7,10 @@ var Promise          = require('bluebird'),
     models           = require('../models'),
     errors           = require('../errors'),
     utils            = require('./utils'),
+    path             = require('path'),
+    fs               = require('fs'),
+    utilsUrl         = require('../utils/url'),
+    config           = require('../config'),
     pipeline         = require('../utils/pipeline'),
     docName          = 'db',
     db;
@@ -17,6 +21,29 @@ var Promise          = require('bluebird'),
  * **See:** [API Methods](index.js.html#api%20methods)
  */
 db = {
+    /**
+     * ### Archive Content
+     * Generate the JSON to export - for Moya only
+     *
+     * @public
+     * @returns {Promise} Ghost Export JSON format
+     */
+    backupContent: function () {
+        var props = {
+            data: exporter.doExport(),
+            filename: exporter.fileName()
+        };
+
+        return Promise.props(props)
+            .then(function successMessage(exportResult) {
+                var filename = path.resolve(utilsUrl.urlJoin(config.get('paths').contentPath, 'data', exportResult.filename));
+
+                return Promise.promisify(fs.writeFile)(filename, JSON.stringify(exportResult.data))
+                    .then(function () {
+                        return filename;
+                    });
+            });
+    },
     /**
      * ### Export Content
      * Generate the JSON to export

core/server/api/middleware.js

@@ -32,3 +32,16 @@ module.exports.authenticatePrivate = [
     cors,
     prettyURLs
 ];
+
+/**
+ * Authentication for client endpoints
+ */
+module.exports.authenticateClient = function authenticateClient(client) {
+    return [
+        auth.authenticate.authenticateClient,
+        auth.authenticate.authenticateUser,
+        auth.authorize.requiresAuthorizedClient(client),
+        cors,
+        prettyURLs
+    ];
+};

core/server/api/routes.js

@@ -175,6 +175,8 @@ module.exports = function apiRoutes() {
         api.http(api.uploads.add)
     );
 
+    apiRouter.post('/db/backup',  mw.authenticateClient('Ghost Backup'), api.http(api.db.backupContent));
+
     apiRouter.post('/uploads/icon',
         mw.authenticatePrivate,
         upload.single('uploadimage'),

core/server/auth/authorize.js

@@ -25,6 +25,17 @@ authorize = {
                 return next(new errors.NoPermissionError({message: i18n.t('errors.middleware.auth.pleaseSignIn')}));
             }
         }
+    },
+
+    // Requires the authenticated client to match specific client
+    requiresAuthorizedClient: function requiresAuthorizedClient(client) {
+        return function doAuthorizedClient(req, res, next) {
+            if (!req.client || !req.client.name || req.client.name !== client) {
+                return next(new errors.NoPermissionError({message: i18n.t('errors.permissions.noPermissionToAction')}));
+            }
+
+            return next();
+        };
     }
 };
 

core/server/data/importer/importers/data/index.js

@@ -30,24 +30,28 @@ DataImporter = {
         return importData;
     },
 
-    doImport: function doImport(importData) {
-        var ops = [], errors = [], results = [], options = {
+    // Allow importing with an options object that is passed through the importer
+    doImport: function doImport(importData, importOptions) {
+        var ops = [], errors = [], results = [], modelOptions = {
             importing: true,
             context: {
                 internal: true
             }
         };
 
+        if (importOptions && importOptions.importPersistUser) {
+            modelOptions.importPersistUser = importOptions.importPersistUser;
+        }
         this.init(importData);
 
         return models.Base.transaction(function (transacting) {
-            options.transacting = transacting;
+            modelOptions.transacting = transacting;
 
             _.each(importers, function (importer) {
                 ops.push(function doModelImport() {
-                    return importer.beforeImport(options)
+                    return importer.beforeImport(modelOptions, importOptions)
                         .then(function () {
-                            return importer.doImport(options)
+                            return importer.doImport(modelOptions)
                                 .then(function (_results) {
                                     results = results.concat(_results);
                                 });
@@ -57,7 +61,7 @@ DataImporter = {
 
             _.each(importers, function (importer) {
                 ops.push(function afterImport() {
-                    return importer.afterImport(options);
+                    return importer.afterImport(modelOptions);
                 });
             });
 

core/server/data/importer/importers/data/users.js

@@ -22,12 +22,14 @@ class UsersImporter extends BaseImporter {
     }
 
     /**
-     * - all imported users are locked and get a random password
+     * - by default all imported users are locked and get a random password
      * - they have to follow the password forgotten flow
      * - we add the role by name [supported by the user model, see User.add]
      *   - background: if you import roles, but they exist already, the related user roles reference to an old model id
+     *
+     *   If importOptions object is supplied with a property of importPersistUser then the user status is not locked
      */
-    beforeImport() {
+    beforeImport(importOptions) {
         debug('beforeImport');
 
         let self = this, role, lookup = {};
@@ -39,12 +41,14 @@ class UsersImporter extends BaseImporter {
 
         this.dataToImport = this.dataToImport.map(self.legacyMapper);
 
-        _.each(this.dataToImport, function (model) {
-            model.password = globalUtils.uid(50);
-            if (model.status !== 'inactive') {
-                model.status = 'locked';
-            }
-        });
+        if (importOptions.importPersistUser !== true) {
+            _.each(this.dataToImport, function (model) {
+                model.password = globalUtils.uid(50);
+                if (model.status !== 'inactive') {
+                    model.status = 'locked';
+                }
+            });
+        }
 
         // NOTE: sort out duplicated roles based on incremental id
         _.each(this.roles_users, function (attachedRole) {

core/server/data/importer/index.js

@@ -324,14 +324,16 @@ _.extend(ImportManager.prototype, {
      * Each importer gets passed the data from importData which has the key matching its type - i.e. it only gets the
      * data that it should import. Each importer then handles actually importing that data into Ghost
      * @param {ImportData} importData
+     * @param {importOptions} importOptions to allow override of certain import features such as locking a user
      * @returns {Promise(ImportData)}
      */
-    doImport: function (importData) {
+    doImport: function (importData, importOptions) {
+        importOptions = importOptions || {};
         var ops = [];
         _.each(this.importers, function (importer) {
             if (importData.hasOwnProperty(importer.type)) {
                 ops.push(function () {
-                    return importer.doImport(importData[importer.type]);
+                    return importer.doImport(importData[importer.type], importOptions);
                 });
             }
         });
@@ -353,9 +355,11 @@ _.extend(ImportManager.prototype, {
      * Import From File
      * The main method of the ImportManager, call this to kick everything off!
      * @param {File} file
+     * @param {importOptions} importOptions to allow override of certain import features such as locking a user
      * @returns {Promise}
      */
-    importFromFile: function (file) {
+    importFromFile: function (file, importOptions) {
+        importOptions = importOptions || {};
         var self = this;
 
         // Step 1: Handle converting the file to usable data
@@ -365,7 +369,7 @@ _.extend(ImportManager.prototype, {
         }).then(function (importData) {
             // Step 3: Actually do the import
             // @TODO: It would be cool to have some sort of dry run flag here
-            return self.doImport(importData);
+            return self.doImport(importData, importOptions);
         }).then(function (importData) {
             // Step 4: Report on the import
             return self.generateReport(importData)

core/server/data/migrations/versions/1.7/1-add-backup-client.js

@@ -0,0 +1,27 @@
+'use strict';
+
+const models = require('../../../../models'),
+    logging = require('../../../../logging'),
+    fixtures = require('../../../schema/fixtures'),
+    _ = require('lodash'),
+    backupClient = fixtures.utils.findModelFixtureEntry('Client', {slug: 'ghost-backup'}),
+    Promise = require('bluebird'),
+    message = 'Adding "Ghost Backup" fixture into clients table';
+
+module.exports = function addGhostBackupClient(options) {
+    var localOptions = _.merge({
+        context: {internal: true}
+    }, options);
+
+    return models.Client
+        .findOne({slug: backupClient.slug}, localOptions)
+        .then(function (client) {
+            if (!client) {
+                logging.info(message);
+                return fixtures.utils.addFixturesForModel(backupClient, localOptions);
+            } else {
+                logging.warn(message);
+                return Promise.resolve();
+            }
+        });
+};

core/server/data/schema/fixtures/fixtures.json

@@ -134,6 +134,12 @@
                     "slug":             "ghost-scheduler",
                     "status":           "enabled",
                     "type":             "web"
+                },
+                {
+                    "name": "Ghost Backup",
+                    "slug": "ghost-backup",
+                    "status": "enabled",
+                    "type": "web"
                 }
             ]
         },

core/server/models/user.js

@@ -154,6 +154,11 @@ User = ghostBookshelf.Model.extend({
                 return Promise.reject(new errors.ValidationError({message: i18n.t('errors.models.user.passwordDoesNotComplyLength')}));
             }
 
+            // An import with importOptions supplied can prevent re-hashing a user password
+            if (options.importPersistUser) {
+                return;
+            }
+
             tasks.hashPassword = (function hashPassword() {
                 return generatePasswordHash(self.get('password'))
                     .then(function (hash) {
@@ -298,7 +303,8 @@ User = ghostBookshelf.Model.extend({
             validOptions = {
                 findOne: ['withRelated', 'status'],
                 setup: ['id'],
-                edit: ['withRelated', 'id'],
+                edit: ['withRelated', 'id', 'importPersistUser'],
+                add: ['importPersistUser'],
                 findPage: ['page', 'limit', 'columns', 'filter', 'order', 'status'],
                 findAll: ['filter']
             };

core/test/functional/routes/api/db_spec.js

@@ -2,12 +2,18 @@ var should = require('should'),
     supertest = require('supertest'),
     testUtils = require('../../../utils'),
     path = require('path'),
+    sinon = require('sinon'),
     config = require('../../../../../core/server/config'),
+    models = require('../../../../../core/server/models'),
+    fs = require('fs'),
+    _ = require('lodash'),
     ghost = testUtils.startGhost,
-    request;
+    request,
+
+    sandbox = sinon.sandbox.create();
 
 describe('DB API', function () {
-    var accesstoken = '', ghostServer;
+    var accesstoken = '', ghostServer, clients, backupClient, schedulerClient, backupQuery, schedulerQuery, fsStub;
 
     before(function (done) {
         // starting ghost automatically populates the db
@@ -21,10 +27,20 @@ describe('DB API', function () {
             return testUtils.doAuth(request);
         }).then(function (token) {
             accesstoken = token;
+            return models.Client.findAll();
+        }).then(function (result) {
+            clients = result.toJSON();
+            backupClient = _.find(clients, {slug: 'ghost-backup'});
+            schedulerClient = _.find(clients, {slug: 'ghost-scheduler'});
+
             done();
         }).catch(done);
     });
 
+    afterEach(function () {
+        sandbox.restore();
+    });
+
     after(function () {
         return testUtils.clearData()
             .then(function () {
@@ -97,4 +113,40 @@ describe('DB API', function () {
                 done();
             });
     });
+
+    it('export can be triggered by backup client', function (done) {
+        backupQuery = '?client_id=' + backupClient.slug + '&client_secret=' + backupClient.secret;
+        fsStub = sandbox.stub(fs, 'writeFile').yields();
+        request.post(testUtils.API.getApiQuery('db/backup' + backupQuery))
+            .expect('Content-Type', /json/)
+            .expect(200)
+            .end(function (err, res) {
+                if (err) {
+                    return done(err);
+                }
+                res.body.should.match(/content\/data/);
+                fsStub.calledOnce.should.eql(true);
+
+                done();
+            });
+    });
+
+    it('export can be triggered by backup client', function (done) {
+        schedulerQuery = '?client_id=' + schedulerClient.slug + '&client_secret=' + schedulerClient.secret;
+        fsStub = sandbox.stub(fs, 'writeFile').yields();
+        request.post(testUtils.API.getApiQuery('db/backup' + schedulerQuery))
+            .expect('Content-Type', /json/)
+            .expect(403)
+            .end(function (err, res) {
+                if (err) {
+                    return done(err);
+                }
+
+                should.exist(res.body.errors);
+                res.body.errors[0].errorType.should.eql('NoPermissionError');
+                fsStub.called.should.eql(false);
+
+                done();
+            });
+    });
 });

core/test/integration/migration_spec.js

@@ -195,10 +195,11 @@ describe('Database Migration (special functions)', function () {
 
                     // Clients
                     should.exist(result.clients);
-                    result.clients.length.should.eql(3);
+                    result.clients.length.should.eql(4);
                     result.clients.at(0).get('name').should.eql('Ghost Admin');
                     result.clients.at(1).get('name').should.eql('Ghost Frontend');
                     result.clients.at(2).get('name').should.eql('Ghost Scheduler');
+                    result.clients.at(3).get('name').should.eql('Ghost Backup');
 
                     // User (Owner)
                     should.exist(result.users);

core/test/unit/migration_spec.js

@@ -20,7 +20,7 @@ var should = require('should'), // jshint ignore:line
 describe('DB version integrity', function () {
     // Only these variables should need updating
     var currentSchemaHash = 'af4028653a7c0804f6bf7b98c50db5dc',
-        currentFixturesHash = '6948548fee557adc738330522dc06d24';
+        currentFixturesHash = 'a8ccedee7058e68eafd268b73458e954';
 
     // If this test is failing, then it is likely a change has been made that requires a DB version bump,
     // and the values above will need updating as confirmation

core/test/utils/fixtures/data-generator.js

@@ -545,7 +545,8 @@ DataGenerator.forKnex = (function () {
     clients = [
         createClient({name: 'Ghost Admin', slug: 'ghost-admin', type: 'ua'}),
         createClient({name: 'Ghost Scheduler', slug: 'ghost-scheduler', type: 'web'}),
-        createClient({name: 'Ghost Auth', slug: 'ghost-auth', type: 'web'})
+        createClient({name: 'Ghost Auth', slug: 'ghost-auth', type: 'web'}),
+        createClient({name: 'Ghost Backup', slug: 'ghost-backup', type: 'web'})
     ];
 
     roles_users = [