Migrated schedules controller to v2

closes #10060

- Implemented scheduling for posts and pages
- Added cache invalidation when scheduling
- Refactored admin token eneration function to accept existing key as parameter in tests
- Added Ghost Scheduler Integration fixture
- Added fixture for permissions for post publish action
- Migrated getScheduled method to v2
- Did not add support for 'from' and 'to' parameters as they were not used by DefaultScheduler
- This method needs rethinking in a long run as it's an ugly hack and should rather become proper endpoint that returns JSON data instead of models
- Removed unused auth middleware from v2 routes
- Added internal scheduler role
- Implemetnted transactions in v2 frame
- This takes into account scenario mentioned in c93f03b87e
- Specifically:
>if two queries happening in a transaction we have to signalise
  knex/mysql that we select for an update
  otherwise the following case happens:
  you fetch posts for an update
  a user requests comes in and updates the post (e.g. sets title to "X")
  you update the fetched posts, title would get overriden to the old one

core/server/adapters/scheduling/post-scheduling/index.js

@@ -15,7 +15,7 @@ _private.normalize = function normalize(options) {
     const {model, apiUrl, client} = options;
 
     return {
-        // NOTE: The scheduler expects a unix timestmap.
+        // NOTE: The scheduler expects a unix timestamp.
         time: moment(model.get('published_at')).valueOf(),
         // @TODO: We are still using API v0.1
         url: `${urlUtils.urlJoin(apiUrl, 'schedules', 'posts', model.get('id'))}?client_id=${client.get('slug')}&client_secret=${client.get('secret')}`,
@@ -41,6 +41,7 @@ _private.loadClient = function loadClient() {
  * @return {Promise}
  */
 _private.loadScheduledPosts = function () {
+    // TODO: make this version aware?
     const api = require('../../../api');
     return api.schedules.getScheduledPosts()
         .then((result) => {

core/server/api/shared/validators/input/all.js

@@ -197,5 +197,10 @@ module.exports = {
     setup() {
         debug('validate setup');
         return this.add(...arguments);
+    },
+
+    publish() {
+        debug('validate schedule');
+        return this.browse(...arguments);
     }
 };

core/server/api/v2/index.js

@@ -23,6 +23,10 @@ module.exports = {
         return require('./session');
     },
 
+    get schedules() {
+        return shared.pipeline(require('./schedules'), localUtils);
+    },
+
     get pages() {
         return shared.pipeline(require('./pages'), localUtils);
     },

core/server/api/v2/pages.js

@@ -43,7 +43,10 @@ module.exports = {
             'fields',
             'formats',
             'debug',
-            'absolute_urls'
+            'absolute_urls',
+            // NOTE: only for internal context
+            'forUpdate',
+            'transacting'
         ],
         data: [
             'id',
@@ -113,7 +116,10 @@ module.exports = {
         headers: {},
         options: [
             'include',
-            'id'
+            'id',
+            // NOTE: only for internal context
+            'forUpdate',
+            'transacting'
         ],
         validation: {
             options: {

core/server/api/v2/posts.js

@@ -42,7 +42,10 @@ module.exports = {
             'fields',
             'formats',
             'debug',
-            'absolute_urls'
+            'absolute_urls',
+            // NOTE: only for internal context
+            'forUpdate',
+            'transacting'
         ],
         data: [
             'id',
@@ -115,7 +118,10 @@ module.exports = {
         options: [
             'include',
             'id',
-            'source'
+            'source',
+            // NOTE: only for internal context
+            'forUpdate',
+            'transacting'
         ],
         validation: {
             options: {

core/server/api/v2/schedules.js

@@ -0,0 +1,130 @@
+const _ = require('lodash');
+const moment = require('moment');
+const config = require('../../config');
+const models = require('../../models');
+const urlUtils = require('../../lib/url-utils');
+const common = require('../../lib/common');
+const api = require('./index');
+
+module.exports = {
+    docName: 'schedules',
+    publish: {
+        headers: {},
+        options: [
+            'id',
+            'resource'
+        ],
+        data: [
+            'force'
+        ],
+        validation: {
+            options: {
+                id: {
+                    required: true
+                },
+                resource: {
+                    required: true,
+                    values: ['posts', 'pages']
+                }
+            }
+        },
+        permissions: {
+            docName: 'posts'
+        },
+        query(frame) {
+            let resource;
+            const resourceType = frame.options.resource;
+            const publishAPostBySchedulerToleranceInMinutes = config.get('times').publishAPostBySchedulerToleranceInMinutes;
+
+            return models.Base.transaction((transacting) => {
+                const options = {
+                    transacting: transacting,
+                    status: 'scheduled',
+                    forUpdate: true,
+                    id: frame.options.id,
+                    context: {
+                        internal: true
+                    }
+                };
+
+                return api[resourceType].read({id: frame.options.id}, options)
+                    .then((result) => {
+                        resource = result[resourceType][0];
+                        const publishedAtMoment = moment(resource.published_at);
+
+                        if (publishedAtMoment.diff(moment(), 'minutes') > publishAPostBySchedulerToleranceInMinutes) {
+                            return Promise.reject(new common.errors.NotFoundError({message: common.i18n.t('errors.api.job.notFound')}));
+                        }
+
+                        if (publishedAtMoment.diff(moment(), 'minutes') < publishAPostBySchedulerToleranceInMinutes * -1 && frame.data.force !== true) {
+                            return Promise.reject(new common.errors.NotFoundError({message: common.i18n.t('errors.api.job.publishInThePast')}));
+                        }
+
+                        const editedResource = {};
+                        editedResource[resourceType] = [{
+                            status: 'published',
+                            updated_at: moment(resource.updated_at).toISOString(true)
+                        }];
+
+                        return api[resourceType].edit(
+                            editedResource,
+                            _.pick(options, ['context', 'id', 'transacting', 'forUpdate'])
+                        );
+                    })
+                    .then((result) => {
+                        const scheduledResource = result[resourceType][0];
+
+                        if (
+                            (scheduledResource.status === 'published' && resource.status !== 'published') ||
+                            (scheduledResource.status === 'draft' && resource.status === 'published')
+                        ) {
+                            this.headers.cacheInvalidate = true;
+                        } else if (
+                            (scheduledResource.status === 'draft' && resource.status !== 'published') ||
+                            (scheduledResource.status === 'scheduled' && resource.status !== 'scheduled')
+                        ) {
+                            this.headers.cacheInvalidate = {
+                                value: urlUtils.urlFor({
+                                    relativeUrl: urlUtils.urlJoin('/p', scheduledResource.uuid, '/')
+                                })
+                            };
+                        } else {
+                            this.headers.cacheInvalidate = false;
+                        }
+
+                        return result;
+                    });
+            });
+        }
+    },
+
+    getScheduled: {
+        // NOTE: this method is for internal use only by DefaultScheduler
+        //       it is not exposed anywhere!
+        permissions: false,
+        validation: {
+            options: {
+                resource: {
+                    required: true,
+                    values: ['posts', 'pages']
+                }
+            }
+        },
+        query(frame) {
+            const resourceType = frame.options.resource;
+            const resourceModel = (resourceType === 'posts') ? 'Post' : 'Page';
+
+            const cleanOptions = {};
+            cleanOptions.filter = 'status:scheduled';
+            cleanOptions.columns = ['id', 'published_at', 'created_at'];
+
+            return models[resourceModel].findAll(cleanOptions)
+                .then((result) => {
+                    let response = {};
+                    response[resourceType] = result;
+
+                    return response;
+                });
+        }
+    }
+};

core/server/api/v2/utils/serializers/output/index.js

@@ -31,6 +31,10 @@ module.exports = {
         return require('./slugs');
     },
 
+    get schedules() {
+        return require('./schedules');
+    },
+
     get webhooks() {
         return require('./webhooks');
     },

core/server/api/v2/utils/serializers/output/schedules.js

@@ -0,0 +1,5 @@
+module.exports = {
+    all(model, apiConfig, frame) {
+        frame.response = model;
+    }
+};

core/server/data/schema/fixtures/fixtures.json

@@ -70,6 +70,10 @@
                 {
                     "name": "DB Backup Integration",
                     "description": "Internal DB Backup Client"
+                },
+                {
+                    "name": "Scheduler Integration",
+                    "description": "Internal Scheduler Client"
                 }
             ]
         },
@@ -141,6 +145,11 @@
                     "action_type": "destroy",
                     "object_type": "post"
                 },
+                {
+                    "name": "Publish posts",
+                    "action_type": "publish",
+                    "object_type": "post"
+                },
                 {
                     "name": "Browse settings",
                     "action_type": "browse",
@@ -584,6 +593,13 @@
                     "description": "Internal DB Backup integration",
                     "type": "internal",
                     "api_keys": [{"type": "admin", "role": "DB Backup Integration"}]
+                },
+                {
+                    "slug": "ghost-scheduler",
+                    "name": "Ghost Scheduler",
+                    "description": "Internal Scheduler integration",
+                    "type": "internal",
+                    "api_keys": [{"type": "admin", "role": "Scheduler Integration"}]
                 }
             ]
         }
@@ -624,6 +640,9 @@
                 "DB Backup Integration": {
                     "db": "all"
                 },
+                "Scheduler Integration": {
+                    "post": "publish"
+                },
                 "Admin Integration": {
                     "mail": "all",
                     "notification": "all",

core/server/web/api/v2/admin/middleware.js

@@ -21,7 +21,8 @@ const notImplemented = function (req, res, next) {
         themes: ['POST', 'PUT'],
         subscribers: ['GET', 'PUT', 'DELETE', 'POST'],
         config: ['GET'],
-        webhooks: ['POST', 'DELETE']
+        webhooks: ['POST', 'DELETE'],
+        schedules: ['PUT']
     };
 
     const match = req.url.match(/^\/(\w+)\/?/);

core/server/web/api/v2/admin/routes.js

@@ -3,7 +3,6 @@ const api = require('../../../../api');
 const apiv2 = require('../../../../api/v2');
 const mw = require('./middleware');
 
-const auth = require('../../../../services/auth');
 const shared = require('../../../shared');
 
 // Handling uploads & imports
@@ -50,10 +49,7 @@ module.exports = function apiRoutes() {
     router.del('/integrations/:id', mw.authAdminApi, http(apiv2.integrations.destroy));
 
     // ## Schedules
-    router.put('/schedules/posts/:id', [
-        auth.authenticate.authenticateClient,
-        auth.authenticate.authenticateUser
-    ], api.http(api.schedules.publishPost));
+    router.put('/schedules/:resource/:id', mw.authAdminApi, http(apiv2.schedules.publish));
 
     // ## Settings
     router.get('/settings/routes/yaml', mw.authAdminApi, http(apiv2.settings.download));

core/test/acceptance/old/admin/roles_spec.js

@@ -35,7 +35,7 @@ describe('Roles API', function () {
                 should.exist(response);
                 should.exist(response.roles);
                 localUtils.API.checkResponse(response, 'roles');
-                response.roles.should.have.length(7);
+                response.roles.should.have.length(8);
                 localUtils.API.checkResponse(response.roles[0], 'role');
                 localUtils.API.checkResponse(response.roles[1], 'role');
                 localUtils.API.checkResponse(response.roles[2], 'role');
@@ -43,6 +43,7 @@ describe('Roles API', function () {
                 localUtils.API.checkResponse(response.roles[4], 'role');
                 localUtils.API.checkResponse(response.roles[5], 'role');
                 localUtils.API.checkResponse(response.roles[6], 'role');
+                localUtils.API.checkResponse(response.roles[7], 'role');
 
                 done();
             });

core/test/regression/api/v2/admin/schedules_spec.js

@@ -0,0 +1,183 @@
+const _ = require('lodash');
+const should = require('should');
+const supertest = require('supertest');
+const Promise = require('bluebird');
+const sinon = require('sinon');
+const moment = require('moment-timezone');
+const SchedulingDefault = require('../../../../../server/adapters/scheduling/SchedulingDefault');
+const models = require('../../../../../server/models/index');
+const config = require('../../../../../server/config/index');
+const testUtils = require('../../../../utils/index');
+const localUtils = require('./utils');
+
+const ghost = testUtils.startGhost;
+
+describe('Schedules API', function () {
+    const resources = [];
+    let request;
+
+    before(function () {
+        models.init();
+
+        // @NOTE: mock the post scheduler, otherwise it will auto publish the post
+        sinon.stub(SchedulingDefault.prototype, '_pingUrl').resolves();
+    });
+
+    after(function () {
+        sinon.restore();
+    });
+
+    before(function () {
+        return ghost()
+            .then(() => {
+                request = supertest.agent(config.get('url'));
+            });
+    });
+
+    before(function () {
+        return ghost()
+            .then(function () {
+                resources.push(testUtils.DataGenerator.forKnex.createPost({
+                    created_by: testUtils.existingData.users[0].id,
+                    author_id: testUtils.existingData.users[0].id,
+                    published_by: testUtils.existingData.users[0].id,
+                    published_at: moment().add(30, 'seconds').toDate(),
+                    status: 'scheduled',
+                    slug: 'first'
+                }));
+
+                resources.push(testUtils.DataGenerator.forKnex.createPost({
+                    created_by: testUtils.existingData.users[0].id,
+                    author_id: testUtils.existingData.users[0].id,
+                    published_by: testUtils.existingData.users[0].id,
+                    published_at: moment().subtract(30, 'seconds').toDate(),
+                    status: 'scheduled',
+                    slug: 'second'
+                }));
+
+                resources.push(testUtils.DataGenerator.forKnex.createPost({
+                    created_by: testUtils.existingData.users[0].id,
+                    author_id: testUtils.existingData.users[0].id,
+                    published_by: testUtils.existingData.users[0].id,
+                    published_at: moment().add(10, 'minute').toDate(),
+                    status: 'scheduled',
+                    slug: 'third'
+                }));
+
+                resources.push(testUtils.DataGenerator.forKnex.createPost({
+                    created_by: testUtils.existingData.users[0].id,
+                    author_id: testUtils.existingData.users[0].id,
+                    published_by: testUtils.existingData.users[0].id,
+                    published_at: moment().subtract(10, 'minute').toDate(),
+                    status: 'scheduled',
+                    slug: 'fourth'
+                }));
+
+                resources.push(testUtils.DataGenerator.forKnex.createPost({
+                    created_by: testUtils.existingData.users[0].id,
+                    author_id: testUtils.existingData.users[0].id,
+                    published_by: testUtils.existingData.users[0].id,
+                    published_at: moment().add(30, 'seconds').toDate(),
+                    status: 'scheduled',
+                    slug: 'fifth',
+                    page: true
+                }));
+
+                return Promise.mapSeries(resources, function (post) {
+                    return models.Post.add(post, {context: {internal: true}});
+                }).then(function (result) {
+                    result.length.should.eql(5);
+                });
+            });
+    });
+
+    describe('publish', function () {
+        let schedulerKey;
+
+        before(() => {
+             schedulerKey = _.find(testUtils.existingData.apiKeys, {integration: {slug: 'ghost-scheduler'}});
+        });
+
+        it('publishes posts', function () {
+            return request
+                .put(localUtils.API.getApiQuery(`schedules/posts/${resources[0].id}/`))
+                .set('Authorization', `Ghost ${localUtils.getValidAdminToken('/v2/admin/', schedulerKey)}`)
+                .set('Origin', config.get('url'))
+                .expect('Content-Type', /json/)
+                .expect('Cache-Control', testUtils.cacheRules.private)
+                .expect(200)
+                .then((res) => {
+                    should.exist(res.headers['x-cache-invalidate']);
+                    const jsonResponse = res.body;
+                    should.exist(jsonResponse);
+                    jsonResponse.posts[0].id.should.eql(resources[0].id);
+                    jsonResponse.posts[0].status.should.eql('published');
+                });
+        });
+
+        it('publishes page', function () {
+            return request
+                .put(localUtils.API.getApiQuery(`schedules/pages/${resources[4].id}/`))
+                .set('Authorization', `Ghost ${localUtils.getValidAdminToken('/v2/admin/', schedulerKey)}`)
+                .expect('Content-Type', /json/)
+                .expect('Cache-Control', testUtils.cacheRules.private)
+                .expect(200)
+                .then((res) => {
+                    should.exist(res.headers['x-cache-invalidate']);
+                    const jsonResponse = res.body;
+                    should.exist(jsonResponse);
+                    jsonResponse.pages[0].id.should.eql(resources[4].id);
+                    jsonResponse.pages[0].status.should.eql('published');
+                });
+        });
+
+        it('no access', function () {
+            const zapierKey = _.find(testUtils.existingData.apiKeys, {integration: {slug: 'ghost-backup'}});
+            return request
+                .put(localUtils.API.getApiQuery(`schedules/posts/${resources[0].id}/`))
+                .set('Authorization', `Ghost ${localUtils.getValidAdminToken('/v2/admin/', zapierKey)}`)
+                .expect('Content-Type', /json/)
+                .expect('Cache-Control', testUtils.cacheRules.private)
+                .expect(403);
+        });
+
+        it('should fail with invalid resource type', function () {
+            return request
+                .put(localUtils.API.getApiQuery(`schedules/this_is_invalid/${resources[0].id}/`))
+                .set('Authorization', `Ghost ${localUtils.getValidAdminToken('/v2/admin/', schedulerKey)}`)
+                .expect('Content-Type', /json/)
+                .expect('Cache-Control', testUtils.cacheRules.private)
+                .expect(422);
+        });
+
+        it('published_at is x seconds in past, but still in tolerance', function () {
+            return request
+                .put(localUtils.API.getApiQuery(`schedules/posts/${resources[1].id}/`))
+                .set('Authorization', `Ghost ${localUtils.getValidAdminToken('/v2/admin/', schedulerKey)}`)
+                .expect('Content-Type', /json/)
+                .expect('Cache-Control', testUtils.cacheRules.private)
+                .expect(200);
+        });
+
+        it('not found', function () {
+            return request
+                .put(localUtils.API.getApiQuery(`schedules/posts/${resources[2].id}/`))
+                .set('Authorization', `Ghost ${localUtils.getValidAdminToken('/v2/admin/', schedulerKey)}`)
+                .expect('Content-Type', /json/)
+                .expect('Cache-Control', testUtils.cacheRules.private)
+                .expect(404);
+        });
+
+        it('force publish', function () {
+            return request
+                .put(localUtils.API.getApiQuery(`schedules/posts/${resources[3].id}/`))
+                .send({
+                    force: true
+                })
+                .set('Authorization', `Ghost ${localUtils.getValidAdminToken('/v2/admin/', schedulerKey)}`)
+                .expect('Content-Type', /json/)
+                .expect('Cache-Control', testUtils.cacheRules.private)
+                .expect(200);
+        });
+    });
+});

core/test/regression/api/v2/admin/utils.js

@@ -102,10 +102,12 @@ module.exports = {
         return testUtils.API.doAuth(`${API_URL}session/`, ...args);
     },
 
-    getValidAdminToken(endpoint) {
+    getValidAdminToken(endpoint, key) {
         const jwt = require('jsonwebtoken');
+        key = key || testUtils.DataGenerator.Content.api_keys[0];
+
         const JWT_OPTIONS = {
-            keyid: testUtils.DataGenerator.Content.api_keys[0].id,
+            keyid: key.id,
             algorithm: 'HS256',
             expiresIn: '5m',
             audience: endpoint
@@ -113,7 +115,7 @@ module.exports = {
 
         return jwt.sign(
             {},
-            Buffer.from(testUtils.DataGenerator.Content.api_keys[0].secret, 'hex'),
+            Buffer.from(key.secret, 'hex'),
             JWT_OPTIONS
         );
     }

core/test/unit/data/schema/fixtures/utils_spec.js

@@ -150,19 +150,19 @@ describe('Migration Fixture Utils', function () {
             fixtureUtils.addFixturesForRelation(fixtures.relations[0]).then(function (result) {
                 should.exist(result);
                 result.should.be.an.Object();
-                result.should.have.property('expected', 65);
-                result.should.have.property('done', 65);
+                result.should.have.property('expected', 66);
+                result.should.have.property('done', 66);
 
                 // Permissions & Roles
                 permsAllStub.calledOnce.should.be.true();
                 rolesAllStub.calledOnce.should.be.true();
-                dataMethodStub.filter.callCount.should.eql(65);
-                dataMethodStub.find.callCount.should.eql(6);
-                baseUtilAttachStub.callCount.should.eql(65);
+                dataMethodStub.filter.callCount.should.eql(66);
+                dataMethodStub.find.callCount.should.eql(7);
+                baseUtilAttachStub.callCount.should.eql(66);
 
-                fromItem.related.callCount.should.eql(65);
-                fromItem.findWhere.callCount.should.eql(65);
-                toItem[0].get.callCount.should.eql(130);
+                fromItem.related.callCount.should.eql(66);
+                fromItem.findWhere.callCount.should.eql(66);
+                toItem[0].get.callCount.should.eql(132);
 
                 done();
             }).catch(done);

core/test/utils/index.js

@@ -948,6 +948,10 @@ startGhost = function startGhost(options) {
                     })
                     .then((tags) => {
                         module.exports.existingData.tags = tags.toJSON();
+                        return models.ApiKey.findAll({withRelated: 'integration'});
+                    })
+                    .then((keys) => {
+                        module.exports.existingData.apiKeys = keys.toJSON(module.exports.context.internal);
                     })
                     .return(ghostServer);
             });
@@ -1022,6 +1026,11 @@ startGhost = function startGhost(options) {
                 })
                 .then((tags) => {
                     module.exports.existingData.tags = tags.toJSON();
+
+                    return models.ApiKey.findAll({columns: ['id', 'secret']});
+                })
+                .then((keys) => {
+                    module.exports.existingData.apiKeys = keys.toJSON();
                 })
                 .return(ghostServer);
         });