diff --git a/core/server/api/index.js b/core/server/api/index.js index 955a7d42be..1fb6c6ddfe 100644 --- a/core/server/api/index.js +++ b/core/server/api/index.js @@ -81,7 +81,7 @@ cacheInvalidationHeader = function (req, result) { // Don't set x-cache-invalidate header for drafts if (hasStatusChanged || wasDeleted || wasPublishedUpdated) { - cacheInvalidate = '/, /page/*, /rss/, /rss/*, /tag/*, /author/*'; + cacheInvalidate = '/, /page/*, /rss/, /rss/*, /tag/*, /author/*, /sitemap-*.xml'; if (id && post.slug) { return config.urlForPost(settings, post).then(function (postUrl) { return cacheInvalidate + ', ' + postUrl; diff --git a/core/server/data/sitemap/base-generator.js b/core/server/data/sitemap/base-generator.js new file mode 100644 index 0000000000..adcc1971e1 --- /dev/null +++ b/core/server/data/sitemap/base-generator.js @@ -0,0 +1,174 @@ + +var _ = require('lodash'), + xml = require('xml'), + moment = require('moment'), + api = require('../../api'), + config = require('../../config'), + Promise = require('bluebird'), + CHANGE_FREQ = 'weekly', + XMLNS_DECLS; + +// Sitemap specific xml namespace declarations that should not change +XMLNS_DECLS = { + _attr: { + xmlns: 'http://www.sitemaps.org/schemas/sitemap/0.9', + 'xmlns:image': 'http://www.google.com/schemas/sitemap-image/1.1' + } +}; + +function BaseSiteMapGenerator() { + this.lastModified = 0; + this.nodeLookup = {}; + this.siteMapContent = ''; +} + +_.extend(BaseSiteMapGenerator.prototype, { + init: function () { + return this.refreshAll(); + }, + + getData: function () { + return Promise.resolve([]); + }, + + refreshAll: function () { + var self = this; + + // Load all data + return this.getData().then(function (data) { + // Generate SiteMap from data + return self.generateXmlFromData(data); + }).then(function (generatedXml) { + self.siteMapContent = generatedXml; + }); + }, + + generateXmlFromData: function (data) { + // This has to be async because of the permalinks retrieval + var self = this; + + // Fetch the permalinks value only once for all the urlFor calls + return this.getPermalinksValue().then(function (permalinks) { + // Create all the url elements in JSON + return _.map(data, function (datum) { + var node = self.createUrlNodeFromDatum(datum, permalinks); + self.updateLastModified(datum); + self.nodeLookup[datum.id] = node; + + return node; + }); + }).then(self.generateXmlFromNodes); + }, + + getPermalinksValue: function () { + var self = this; + + if (this.permalinks) { + return Promise.resolve(this.permalinks); + } + + return api.settings.read('permalinks').then(function (response) { + self.permalinks = response.settings[0]; + return self.permalinks; + }); + }, + + updatePermalinksValue: function (permalinks) { + this.permalinks = permalinks; + + // Re-generate xml with new permalinks values + this.updateXmlFromNodes(_.values(this.nodeLookup)); + }, + + generateXmlFromNodes: function (urlElements) { + var data = { + // Concat the elements to the _attr declaration + urlset: [XMLNS_DECLS].concat(urlElements) + }; + + // Return the xml + return xml(data, { + declaration: true + }); + }, + + updateXmlFromNodes: function (urlElements) { + var content = this.generateXmlFromNodes(urlElements); + + this.setSiteMapContent(content); + + return content; + }, + + addUrl: function (datum) { + var self = this; + return this.getPermalinksValue().then(function (permalinks) { + var node = self.createUrlNodeFromDatum(datum, permalinks); + self.updateLastModified(datum); + self.nodeLookup[datum.id] = node; + + return self.updateXmlFromNodes(_.values(self.nodeLookup)); + }); + }, + + removeUrl: function (datum) { + var lookup = this.nodeLookup; + delete lookup[datum.id]; + + this.lastModified = Date.now(); + + return this.updateXmlFromNodes(_.values(lookup)); + }, + + updateUrl: function (datum) { + var self = this; + return this.getPermalinksValue().then(function (permalinks) { + var node = self.createUrlNodeFromDatum(datum, permalinks); + self.updateLastModified(datum); + // TODO: Check if the node values changed, and if not don't regenerate + self.nodeLookup[datum.id] = node; + + return self.updateXmlFromNodes(_.values(self.nodeLookup)); + }); + }, + + getUrlForDatum: function () { + return config.urlFor('home', true); + }, + + getUrlForImage: function (image) { + return config.urlFor('image', {image: image}, true); + }, + + getPriorityForDatum: function () { + return 1.0; + }, + + createUrlNodeFromDatum: function (datum, permalinks) { + var url = this.getUrlForDatum(datum, permalinks), + priority = this.getPriorityForDatum(datum); + + return { + url: [ + {loc: url}, + {lastmod: moment(datum.updated_at || datum.published_at || datum.created_at).toISOString()}, + {changefreq: CHANGE_FREQ}, + {priority: priority} + ] + }; + }, + + setSiteMapContent: function (content) { + this.siteMapContent = content; + }, + + updateLastModified: function (datum) { + var lastModified = datum.updated_at || datum.published_at || datum.created_at; + + if (lastModified > this.lastModified) { + this.lastModified = lastModified; + } + } +}); + +module.exports = BaseSiteMapGenerator; diff --git a/core/server/data/sitemap/handler.js b/core/server/data/sitemap/handler.js new file mode 100644 index 0000000000..45f08e07dd --- /dev/null +++ b/core/server/data/sitemap/handler.js @@ -0,0 +1,44 @@ +var _ = require('lodash'), + utils = require('../utils'), + sitemap = require('./index'); + +// Responsible for handling requests for sitemap files +module.exports = function (blogApp) { + var resourceTypes = ['posts', 'authors', 'tags', 'pages'], + verifyResourceType = function (req, res, next) { + if (!_.contains(resourceTypes, req.param('resource'))) { + return res.send(404); + } + + next(); + }, + getResourceSiteMapXml = function (type, page) { + return sitemap.getSiteMapXml(type, page); + }; + + // Redirect normal sitemap.xml requests to sitemap-index.xml + blogApp.get('/sitemap.xml', function (req, res) { + res.set({'Cache-Control': 'public, max-age=' + utils.ONE_YEAR_S}); + res.redirect(301, '/sitemap-index.xml'); + }); + + blogApp.get('/sitemap-index.xml', function (req, res) { + res.set({ + 'Cache-Control': 'public, max-age=' + utils.ONE_HOUR_S, + 'Content-Type': 'text/xml' + }); + res.send(sitemap.getIndexXml()); + }); + + blogApp.get('/sitemap-:resource.xml', verifyResourceType, function (req, res) { + var type = req.param('resource'), + page = 1, + siteMapXml = getResourceSiteMapXml(type, page); + + res.set({ + 'Cache-Control': 'public, max-age=' + utils.ONE_HOUR_S, + 'Content-Type': 'text/xml' + }); + res.send(siteMapXml); + }); +}; diff --git a/core/server/data/sitemap/index-generator.js b/core/server/data/sitemap/index-generator.js new file mode 100644 index 0000000000..4691350053 --- /dev/null +++ b/core/server/data/sitemap/index-generator.js @@ -0,0 +1,54 @@ +var _ = require('lodash'), + xml = require('xml'), + moment = require('moment'), + config = require('../../config'), + RESOURCES, + XMLNS_DECLS; + +RESOURCES = ['pages', 'posts', 'authors', 'tags']; + +XMLNS_DECLS = { + _attr: { + xmlns: 'http://www.sitemaps.org/schemas/sitemap/0.9' + } +}; + +function SiteMapIndexGenerator(opts) { + // Grab the other site map generators from the options + _.extend(this, _.pick(opts, RESOURCES)); +} + +_.extend(SiteMapIndexGenerator.prototype, { + getIndexXml: function () { + var urlElements = this.generateSiteMapUrlElements(), + data = { + // Concat the elements to the _attr declaration + sitemapindex: [XMLNS_DECLS].concat(urlElements) + }; + + // Return the xml + return xml(data, { + declaration: true + }); + }, + + generateSiteMapUrlElements: function () { + var self = this; + + return _.map(RESOURCES, function (resourceType) { + var url = config.urlFor({ + relativeUrl: '/sitemap-' + resourceType + '.xml' + }, true), + lastModified = self[resourceType].lastModified; + + return { + sitemap: [ + {loc: url}, + {lastmod: moment(lastModified).toISOString()} + ] + }; + }); + } +}); + +module.exports = SiteMapIndexGenerator; diff --git a/core/server/data/sitemap/index.js b/core/server/data/sitemap/index.js new file mode 100644 index 0000000000..195e1d4839 --- /dev/null +++ b/core/server/data/sitemap/index.js @@ -0,0 +1,4 @@ + +var SiteMapManager = require('./manager'); + +module.exports = new SiteMapManager(); diff --git a/core/server/data/sitemap/manager.js b/core/server/data/sitemap/manager.js new file mode 100644 index 0000000000..f6300e8039 --- /dev/null +++ b/core/server/data/sitemap/manager.js @@ -0,0 +1,220 @@ + +var _ = require('lodash'), + Promise = require('bluebird'), + IndexMapGenerator = require('./index-generator'), + PagesMapGenerator = require('./page-generator'), + PostsMapGenerator = require('./post-generator'), + UsersMapGenerator = require('./user-generator'), + TagsMapGenerator = require('./tag-generator'), + SiteMapManager; + +SiteMapManager = function (opts) { + opts = opts || {}; + + this.initialized = false; + + this.pages = opts.pages || this.createPagesGenerator(opts); + this.posts = opts.posts || this.createPostsGenerator(opts); + this.authors = opts.authors || this.createUsersGenerator(opts); + this.tags = opts.tags || this.createTagsGenerator(opts); + + this.index = opts.index || this.createIndexGenerator(opts); +}; + +_.extend(SiteMapManager.prototype, { + createIndexGenerator: function () { + return new IndexMapGenerator(_.pick(this, 'pages', 'posts', 'authors', 'tags')); + }, + + createPagesGenerator: function (opts) { + return new PagesMapGenerator(opts); + }, + + createPostsGenerator: function (opts) { + return new PostsMapGenerator(opts); + }, + + createUsersGenerator: function (opts) { + return new UsersMapGenerator(opts); + }, + + createTagsGenerator: function (opts) { + return new TagsMapGenerator(opts); + }, + + init: function () { + var self = this, + initOps = [ + this.pages.init(), + this.posts.init(), + this.authors.init(), + this.tags.init() + ]; + + return Promise.all(initOps).then(function () { + self.initialized = true; + }); + }, + + getIndexXml: function () { + if (!this.initialized) { + return ''; + } + + return this.index.getIndexXml(); + }, + + getSiteMapXml: function (type) { + if (!this.initialized || !this[type]) { + return null; + } + + return this[type].siteMapContent; + }, + + pageAdded: function (page) { + if (!this.initialized) { + return; + } + + if (page.get('status') !== 'published') { + return; + } + + this.pages.addUrl(page.toJSON()); + }, + + pageEdited: function (page) { + if (!this.initialized) { + return; + } + + var pageData = page.toJSON(), + wasPublished = page.updated('status') === 'published', + isPublished = pageData.status === 'published'; + + // Published status hasn't changed and it's published + if (isPublished === wasPublished && isPublished) { + this.pages.updateUrl(pageData); + } else if (!isPublished && wasPublished) { + // Handle page going from published to draft + this.pageDeleted(page); + } else if (isPublished && !wasPublished) { + // ... and draft to published + this.pageAdded(page); + } + }, + + pageDeleted: function (page) { + if (!this.initialized) { + return; + } + + this.pages.removeUrl(page.toJSON()); + }, + + postAdded: function (post) { + if (!this.initialized) { + return; + } + + this.posts.addUrl(post.toJSON()); + }, + + postEdited: function (post) { + if (!this.initialized) { + return; + } + + var postData = post.toJSON(), + wasPublished = post.updated('status') === 'published', + isPublished = postData.status === 'published'; + + // Published status hasn't changed and it's published + if (isPublished === wasPublished && isPublished) { + this.posts.updateUrl(postData); + } else if (!isPublished && wasPublished) { + // Handle post going from published to draft + this.postDeleted(post); + } else if (isPublished && !wasPublished) { + // ... and draft to published + this.postAdded(post); + } + }, + + postDeleted: function (post) { + if (!this.initialized) { + return; + } + + this.posts.removeUrl(post.toJSON()); + }, + + userAdded: function (user) { + if (!this.initialized) { + return; + } + + this.authors.addUrl(user.toJSON()); + }, + + userEdited: function (user) { + if (!this.initialized) { + return; + } + + var userData = user.toJSON(); + + this.authors.updateUrl(userData); + }, + + userDeleted: function (user) { + if (!this.initialized) { + return; + } + + this.authors.removeUrl(user.toJSON()); + }, + + tagAdded: function (tag) { + if (!this.initialized) { + return; + } + + this.tags.addUrl(tag.toJSON()); + }, + + tagEdited: function (tag) { + if (!this.initialized) { + return; + } + + this.tags.updateUrl(tag.toJSON()); + }, + + tagDeleted: function (tag) { + if (!this.initialized) { + return; + } + + this.tags.removeUrl(tag.toJSON()); + }, + + // TODO: Call this from settings model when it's changed + permalinksUpdated: function (permalinks) { + if (!this.initialized) { + return; + } + + this.posts.updatePermalinksValue(permalinks.toJSON ? permalinks.toJSON() : permalinks); + }, + + _refreshAllPosts: _.throttle(function () { + this.posts.refreshAllPosts(); + }, 3000, { + leading: false, + trailing: true + }) +}); + +module.exports = SiteMapManager; diff --git a/core/server/data/sitemap/page-generator.js b/core/server/data/sitemap/page-generator.js new file mode 100644 index 0000000000..4858354632 --- /dev/null +++ b/core/server/data/sitemap/page-generator.js @@ -0,0 +1,75 @@ +var _ = require('lodash'), + path = require('path'), + api = require('../../api'), + BaseMapGenerator = require('./base-generator'), + config = require('../../config'); + +// A class responsible for generating a sitemap from posts and keeping it updated +function PageMapGenerator(opts) { + _.extend(this, _.defaults(opts || {}, PageMapGenerator.Defaults)); + + BaseMapGenerator.apply(this, arguments); +} + +PageMapGenerator.Defaults = { + // TODO? +}; + +// Inherit from the base generator class +_.extend(PageMapGenerator.prototype, BaseMapGenerator.prototype); + +_.extend(PageMapGenerator.prototype, { + getData: function () { + return api.posts.browse({ + context: { + internal: true + }, + status: 'published', + staticPages: true + }).then(function (resp) { + var homePage = { + id: 0, + name: 'home' + }; + return [homePage].concat(resp.posts); + }); + }, + + getUrlForDatum: function (post, permalinks) { + if (post.id === 0 && !_.isEmpty(post.name)) { + return config.urlFor(post.name, true); + } + + return config.urlFor('post', {post: post, permalinks: permalinks}, true); + }, + + getPriorityForDatum: function (post) { + // TODO: We could influence this with priority or meta information + return post && post.name === 'home' ? 1.0 : 0.8; + }, + + createUrlNodeFromDatum: function (datum) { + var orig = BaseMapGenerator.prototype.createUrlNodeFromDatum.apply(this, arguments), + imageUrl, + imageEl; + + // Check for image and add it + if (datum.image) { + // Grab the image url + imageUrl = this.getUrlForImage(datum.image); + // Create the weird xml node syntax structure that is expected + imageEl = [ + {'image:loc': imageUrl}, + {'image:caption': path.basename(imageUrl)} + ]; + // Add the node to the url xml node + orig.url.push({ + 'image:image': imageEl + }); + } + + return orig; + } +}); + +module.exports = PageMapGenerator; diff --git a/core/server/data/sitemap/post-generator.js b/core/server/data/sitemap/post-generator.js new file mode 100644 index 0000000000..9e44107ac8 --- /dev/null +++ b/core/server/data/sitemap/post-generator.js @@ -0,0 +1,67 @@ +var _ = require('lodash'), + path = require('path'), + api = require('../../api'), + BaseMapGenerator = require('./base-generator'), + config = require('../../config'); + +// A class responsible for generating a sitemap from posts and keeping it updated +function PostMapGenerator(opts) { + _.extend(this, _.defaults(opts || {}, PostMapGenerator.Defaults)); + + BaseMapGenerator.apply(this, arguments); +} + +PostMapGenerator.Defaults = { + // TODO? +}; + +// Inherit from the base generator class +_.extend(PostMapGenerator.prototype, BaseMapGenerator.prototype); + +_.extend(PostMapGenerator.prototype, { + getData: function () { + return api.posts.browse({ + context: { + internal: true + }, + status: 'published', + staticPages: false + }).then(function (resp) { + return resp.posts; + }); + }, + + getUrlForDatum: function (post, permalinks) { + return config.urlFor('post', {post: post, permalinks: permalinks}, true); + }, + + getPriorityForDatum: function () { + // TODO: We could influence this with meta information + return 0.8; + }, + + createUrlNodeFromDatum: function (datum) { + var orig = BaseMapGenerator.prototype.createUrlNodeFromDatum.apply(this, arguments), + imageUrl, + imageEl; + + // Check for image and add it + if (datum.image) { + // Grab the image url + imageUrl = this.getUrlForImage(datum.image); + // Create the weird xml node syntax structure that is expected + imageEl = [ + {'image:loc': imageUrl}, + {'image:caption': path.basename(imageUrl)} + ]; + // Add the node to the url xml node + orig.url.push({ + 'image:image': imageEl + }); + } + + return orig; + } +}); + +module.exports = PostMapGenerator; diff --git a/core/server/data/sitemap/tag-generator.js b/core/server/data/sitemap/tag-generator.js new file mode 100644 index 0000000000..01e0da48cc --- /dev/null +++ b/core/server/data/sitemap/tag-generator.js @@ -0,0 +1,41 @@ +var _ = require('lodash'), + api = require('../../api'), + BaseMapGenerator = require('./base-generator'), + config = require('../../config'); + +// A class responsible for generating a sitemap from posts and keeping it updated +function TagsMapGenerator(opts) { + _.extend(this, _.defaults(opts || {}, TagsMapGenerator.Defaults)); + + BaseMapGenerator.apply(this, arguments); +} + +TagsMapGenerator.Defaults = { + // TODO? +}; + +// Inherit from the base generator class +_.extend(TagsMapGenerator.prototype, BaseMapGenerator.prototype); + +_.extend(TagsMapGenerator.prototype, { + getData: function () { + return api.tags.browse({ + context: { + internal: true + } + }).then(function (resp) { + return resp.tags; + }); + }, + + getUrlForDatum: function (tag, permalinks) { + return config.urlFor('tag', {tag: tag, permalinks: permalinks}, true); + }, + + getPriorityForDatum: function () { + // TODO: We could influence this with meta information + return 0.6; + } +}); + +module.exports = TagsMapGenerator; diff --git a/core/server/data/sitemap/user-generator.js b/core/server/data/sitemap/user-generator.js new file mode 100644 index 0000000000..9d3c803b27 --- /dev/null +++ b/core/server/data/sitemap/user-generator.js @@ -0,0 +1,65 @@ +var _ = require('lodash'), + path = require('path'), + api = require('../../api'), + BaseMapGenerator = require('./base-generator'), + config = require('../../config'); + +// A class responsible for generating a sitemap from posts and keeping it updated +function UserMapGenerator(opts) { + _.extend(this, _.defaults(opts || {}, UserMapGenerator.Defaults)); + + BaseMapGenerator.apply(this, arguments); +} + +UserMapGenerator.Defaults = { + // TODO? +}; + +// Inherit from the base generator class +_.extend(UserMapGenerator.prototype, BaseMapGenerator.prototype); + +_.extend(UserMapGenerator.prototype, { + getData: function () { + return api.users.browse({ + context: { + internal: true + } + }).then(function (resp) { + return resp.users; + }); + }, + + getUrlForDatum: function (user, permalinks) { + return config.urlFor('author', {author: user, permalinks: permalinks}, true); + }, + + getPriorityForDatum: function () { + // TODO: We could influence this with meta information + return 0.6; + }, + + createUrlNodeFromDatum: function (datum) { + var orig = BaseMapGenerator.prototype.createUrlNodeFromDatum.apply(this, arguments), + imageUrl, + imageEl; + + // Check for image and add it + if (datum.image) { + // Grab the image url + imageUrl = this.getUrlForImage(datum.image); + // Create the weird xml node syntax structure that is expected + imageEl = [ + {'image:loc': imageUrl}, + {'image:caption': path.basename(imageUrl)} + ]; + // Add the node to the url xml node + orig.url.push({ + 'image:image': imageEl + }); + } + + return orig; + } +}); + +module.exports = UserMapGenerator; diff --git a/core/server/index.js b/core/server/index.js index 22107fb9c7..ee0c3e0d19 100644 --- a/core/server/index.js +++ b/core/server/index.js @@ -17,6 +17,7 @@ var express = require('express'), models = require('./models'), permissions = require('./permissions'), apps = require('./apps'), + sitemap = require('./data/sitemap'), GhostServer = require('./ghost-server'), // Variables @@ -166,7 +167,9 @@ function init(options) { // Initialize mail mailer.init(), // Initialize apps - apps.init() + apps.init(), + // Initialize sitemaps + sitemap.init() ); }).then(function () { var adminHbs = hbs.create(); diff --git a/core/server/middleware/index.js b/core/server/middleware/index.js index c2f35bc468..dce53325a8 100644 --- a/core/server/middleware/index.js +++ b/core/server/middleware/index.js @@ -23,6 +23,7 @@ var api = require('../api'), oauth2orize = require('oauth2orize'), authStrategies = require('./auth-strategies'), utils = require('../utils'), + sitemapHandler = require('../data/sitemap/handler'), blogApp, setupMiddleware; @@ -291,6 +292,9 @@ setupMiddleware = function (blogAppInstance, adminApp) { // Serve robots.txt if not found in theme blogApp.use(serveSharedFile('robots.txt', 'text/plain', utils.ONE_HOUR_S)); + // site map + sitemapHandler(blogApp); + // Add in all trailing slashes, properly include the subdir path // in the redirect. blogApp.use(slashes(true, { diff --git a/core/server/models/post.js b/core/server/models/post.js index b5d264368e..321bd07000 100644 --- a/core/server/models/post.js +++ b/core/server/models/post.js @@ -8,6 +8,7 @@ var _ = require('lodash'), converter = new Showdown.converter({extensions: [ghostgfm]}), ghostBookshelf = require('./base'), xmlrpc = require('../xmlrpc'), + sitemap = require('../data/sitemap'), Post, Posts; @@ -34,6 +35,43 @@ Post = ghostBookshelf.Model.extend({ } return self.updateTags(model, attributes, options); }); + + this.on('created', function (model) { + var isPage = !!model.get('page'); + if (isPage) { + sitemap.pageAdded(model); + } else { + sitemap.postAdded(model); + } + }); + this.on('updated', function (model) { + var isPage = !!model.get('page'), + wasPage = !!model.updated('page'); + + if (isPage && wasPage) { + // Page value didn't change, remains a page + sitemap.pageEdited(model); + } else if (!isPage && !wasPage) { + // Remains a Post + sitemap.postEdited(model); + } else if (isPage && !wasPage) { + // Switched from Post to Page + sitemap.postDeleted(model); + sitemap.pageAdded(model); + } else if (!isPage && wasPage) { + // Switched from Page to Post + sitemap.pageDeleted(model); + sitemap.postAdded(model); + } + }); + this.on('destroyed', function (model) { + var isPage = !!model.get('page'); + if (isPage) { + sitemap.pageDeleted(model); + } else { + sitemap.postDeleted(model); + } + }); }, saving: function (newPage, attr, options) { diff --git a/core/server/models/tag.js b/core/server/models/tag.js index 9c95ad9046..e17632d340 100644 --- a/core/server/models/tag.js +++ b/core/server/models/tag.js @@ -1,6 +1,7 @@ var _ = require('lodash'), errors = require('../errors'), ghostBookshelf = require('./base'), + sitemap = require('../data/sitemap'), Tag, Tags; @@ -9,6 +10,20 @@ Tag = ghostBookshelf.Model.extend({ tableName: 'tags', + initialize: function () { + ghostBookshelf.Model.prototype.initialize.apply(this, arguments); + + this.on('created', function (model) { + sitemap.tagAdded(model); + }); + this.on('updated', function (model) { + sitemap.tagEdited(model); + }); + this.on('destroyed', function (model) { + sitemap.tagDeleted(model); + }); + }, + saving: function (newPage, attr, options) { /*jshint unused:false*/ diff --git a/core/server/models/user.js b/core/server/models/user.js index 444f8e500a..1ac86bc895 100644 --- a/core/server/models/user.js +++ b/core/server/models/user.js @@ -8,6 +8,7 @@ var _ = require('lodash'), request = require('request'), validation = require('../data/validation'), config = require('../config'), + sitemap = require('../data/sitemap'), bcryptGenSalt = Promise.promisify(bcrypt.genSalt), bcryptHash = Promise.promisify(bcrypt.hash), @@ -42,6 +43,20 @@ User = ghostBookshelf.Model.extend({ tableName: 'users', + initialize: function () { + ghostBookshelf.Model.prototype.initialize.apply(this, arguments); + + this.on('created', function (model) { + sitemap.userAdded(model); + }); + this.on('updated', function (model) { + sitemap.userEdited(model); + }); + this.on('destroyed', function (model) { + sitemap.userDeleted(model); + }); + }, + saving: function (newPage, attr, options) { /*jshint unused:false*/ diff --git a/core/shared/robots.txt b/core/shared/robots.txt index e185180a06..c6b421f684 100644 --- a/core/shared/robots.txt +++ b/core/shared/robots.txt @@ -1,2 +1,3 @@ User-agent: * -Disallow: /ghost/ \ No newline at end of file +Sitemap: /sitemap-index.xml +Disallow: /ghost/ diff --git a/core/test/functional/routes/api/posts_test.js b/core/test/functional/routes/api/posts_test.js index 5c181ed796..5a5a615156 100644 --- a/core/test/functional/routes/api/posts_test.js +++ b/core/test/functional/routes/api/posts_test.js @@ -363,7 +363,7 @@ describe('Post API', function () { var publishedPost = res.body; _.has(res.headers, 'x-cache-invalidate').should.equal(true); res.headers['x-cache-invalidate'].should.eql( - '/, /page/*, /rss/, /rss/*, /tag/*, /author/*, /' + publishedPost.posts[0].slug + '/' + '/, /page/*, /rss/, /rss/*, /tag/*, /author/*, /sitemap-*.xml, /' + publishedPost.posts[0].slug + '/' ); publishedPost.should.exist; @@ -782,7 +782,7 @@ describe('Post API', function () { jsonResponse.should.exist; jsonResponse.posts.should.exist; res.headers['x-cache-invalidate'].should.eql( - '/, /page/*, /rss/, /rss/*, /tag/*, /author/*, /' + jsonResponse.posts[0].slug + '/' + '/, /page/*, /rss/, /rss/*, /tag/*, /author/*, /sitemap-*.xml, /' + jsonResponse.posts[0].slug + '/' ); testUtils.API.checkResponse(jsonResponse.posts[0], 'post'); jsonResponse.posts[0].id.should.eql(deletePostId); diff --git a/core/test/functional/routes/frontend_test.js b/core/test/functional/routes/frontend_test.js index a7c98344a6..249af822df 100644 --- a/core/test/functional/routes/frontend_test.js +++ b/core/test/functional/routes/frontend_test.js @@ -925,4 +925,44 @@ describe('Frontend Routing', function () { }); }); }); + + describe('Site Map', function () { + before(function (done) { + testUtils.initData().then(function () { + return testUtils.fixtures.insertPosts(); + }).then(function () { + done(); + }).catch(done); + }); + + it('should redirect for /sitemap.xml', function (done) { + request.get('/sitemap.xml') + .expect(301) + .expect('location', /sitemap-index.xml/) + .end(doEnd(done)); + }); + + it('should serve sitemap-index.xml', function (done) { + request.get('/sitemap-index.xml') + .expect(200) + .expect('Content-Type', 'text/xml; charset=utf-8') + .end(doEnd(done)); + }); + + it('should serve sitemap-posts.xml', function (done) { + request.get('/sitemap-posts.xml') + .expect(200) + .expect('Content-Type', 'text/xml; charset=utf-8') + .end(doEnd(done)); + }); + + it('should serve sitemap-pages.xml', function (done) { + request.get('/sitemap-posts.xml') + .expect(200) + .expect('Content-Type', 'text/xml; charset=utf-8') + .end(doEnd(done)); + }); + + // TODO: Other pages and verify content + }); }); diff --git a/core/test/integration/sitemap_spec.js b/core/test/integration/sitemap_spec.js new file mode 100644 index 0000000000..f893b4e962 --- /dev/null +++ b/core/test/integration/sitemap_spec.js @@ -0,0 +1,469 @@ +/*globals describe, before, afterEach, it */ +/*jshint expr:true*/ +var testUtils = require('../utils/index'), + _ = require('lodash'), + should = require('should'), + sinon = require('sinon'), + Promise = require('bluebird'), + validator = require('validator'), + + // Stuff we are testing + SiteMapManager = require('../../server/data/sitemap/manager'), + BaseGenerator = require('../../server/data/sitemap/base-generator'), + PostGenerator = require('../../server/data/sitemap/post-generator'), + PageGenerator = require('../../server/data/sitemap/page-generator'), + TagGenerator = require('../../server/data/sitemap/tag-generator'), + UserGenerator = require('../../server/data/sitemap/user-generator'), + + sandbox = sinon.sandbox.create(); + +describe('Sitemap', function () { + var makeStubManager = function () { + return new SiteMapManager({ + pages: { + init: sandbox.stub().returns(Promise.resolve()), + addUrl: sandbox.stub(), + removeUrl: sandbox.stub(), + updateUrl: sandbox.stub() + }, + posts: { + init: sandbox.stub().returns(Promise.resolve()), + addUrl: sandbox.stub(), + removeUrl: sandbox.stub(), + updateUrl: sandbox.stub() + }, + authors: { + init: sandbox.stub().returns(Promise.resolve()), + addUrl: sandbox.stub(), + removeUrl: sandbox.stub(), + updateUrl: sandbox.stub() + }, + tags: { + init: sandbox.stub().returns(Promise.resolve()), + addUrl: sandbox.stub(), + removeUrl: sandbox.stub(), + updateUrl: sandbox.stub() + }, + index: { + init: sandbox.stub().returns(Promise.resolve()), + addUrl: sandbox.stub(), + removeUrl: sandbox.stub(), + updateUrl: sandbox.stub() + } + }); + }; + + before(testUtils.teardown); + afterEach(testUtils.teardown); + afterEach(function () { + sandbox.restore(); + }); + + describe('SiteMapManager', function () { + should.exist(SiteMapManager); + + it('can create a SiteMapManager instance', function () { + var manager = makeStubManager(); + + should.exist(manager); + }); + + it('can initialize', function (done) { + var manager = makeStubManager(); + + manager.initialized.should.equal(false); + + manager.init().then(function () { + manager.posts.init.called.should.equal(true); + manager.pages.init.called.should.equal(true); + manager.authors.init.called.should.equal(true); + manager.tags.init.called.should.equal(true); + + manager.initialized.should.equal(true); + + done(); + }).catch(done); + }); + + it('responds to calls before being initialized', function () { + var manager = makeStubManager(); + + manager.initialized.should.equal(false); + + manager.getIndexXml(); + manager.getSiteMapXml(); + manager.pageAdded(); + manager.pages.addUrl.called.should.equal(false); + manager.pageEdited(); + manager.pageDeleted(); + manager.postAdded(); + manager.pages.addUrl.called.should.equal(false); + manager.postEdited(); + manager.postDeleted(); + manager.userAdded(); + manager.pages.addUrl.called.should.equal(false); + manager.userEdited(); + manager.userDeleted(); + manager.tagAdded(); + manager.pages.addUrl.called.should.equal(false); + manager.tagEdited(); + manager.tagDeleted(); + manager.permalinksUpdated(); + + manager.initialized.should.equal(false); + }); + + it('updates page site map', function (done) { + var manager = makeStubManager(), + fake = { + toJSON: sandbox.stub().returns({ + status: 'published' + }), + get: sandbox.stub().returns('published'), + updated: sandbox.stub().returns('published') + }; + + manager.init().then(function () { + manager.pageAdded(fake); + manager.pages.addUrl.called.should.equal(true); + manager.pageEdited(fake); + manager.pages.updateUrl.called.should.equal(true); + manager.pageDeleted(fake); + manager.pages.removeUrl.called.should.equal(true); + + done(); + }).catch(done); + }); + + it('adds pages that were published', function (done) { + var manager = makeStubManager(), + fake = { + toJSON: sandbox.stub().returns({ + status: 'published' + }), + get: sandbox.stub().returns('published'), + updated: sandbox.stub().returns('draft') + }; + + manager.init().then(function () { + manager.pageAdded = sandbox.stub(); + + manager.pageEdited(fake); + + manager.pages.updateUrl.called.should.equal(false); + manager.pageAdded.called.should.equal(true); + + done(); + }).catch(done); + }); + + it('deletes pages that were unpublished', function (done) { + var manager = makeStubManager(), + fake = { + toJSON: sandbox.stub().returns({ + status: 'draft' + }), + get: sandbox.stub().returns('draft'), + updated: sandbox.stub().returns('published') + }; + + manager.init().then(function () { + manager.pageAdded = sandbox.stub(); + manager.pageDeleted = sandbox.stub(); + + manager.pageEdited(fake); + + manager.pages.updateUrl.called.should.equal(false); + manager.pageAdded.called.should.equal(false); + manager.pageDeleted.called.should.equal(true); + + done(); + }).catch(done); + }); + + it('updates post site map', function (done) { + var manager = makeStubManager(), + fake = { + toJSON: sandbox.stub().returns({ + status: 'published' + }), + get: sandbox.stub().returns('published'), + updated: sandbox.stub().returns('published') + }; + + manager.init().then(function () { + manager.postAdded(fake); + manager.posts.addUrl.called.should.equal(true); + manager.postEdited(fake); + manager.posts.updateUrl.called.should.equal(true); + manager.postDeleted(fake); + manager.posts.removeUrl.called.should.equal(true); + + done(); + }).catch(done); + }); + + it('adds posts that were published', function (done) { + var manager = makeStubManager(), + fake = { + toJSON: sandbox.stub().returns({ + status: 'published' + }), + get: sandbox.stub().returns('published'), + updated: sandbox.stub().returns('draft') + }; + + manager.init().then(function () { + manager.postAdded = sandbox.stub(); + + manager.postEdited(fake); + + manager.posts.updateUrl.called.should.equal(false); + manager.postAdded.called.should.equal(true); + + done(); + }).catch(done); + }); + + it('deletes posts that were unpublished', function (done) { + var manager = makeStubManager(), + fake = { + toJSON: sandbox.stub().returns({ + status: 'draft' + }), + get: sandbox.stub().returns('draft'), + updated: sandbox.stub().returns('published') + }; + + manager.init().then(function () { + manager.postAdded = sandbox.stub(); + manager.postDeleted = sandbox.stub(); + + manager.postEdited(fake); + + manager.posts.updateUrl.called.should.equal(false); + manager.postAdded.called.should.equal(false); + manager.postDeleted.called.should.equal(true); + + done(); + }).catch(done); + }); + + it('updates authors site map', function (done) { + var manager = makeStubManager(), + fake = { + toJSON: sandbox.stub().returns({}) + }; + + manager.init().then(function () { + manager.userAdded(fake); + manager.authors.addUrl.called.should.equal(true); + manager.userEdited(fake); + manager.authors.updateUrl.called.should.equal(true); + manager.userDeleted(fake); + manager.authors.removeUrl.called.should.equal(true); + + done(); + }).catch(done); + }); + + it('updates tags site map', function (done) { + var manager = makeStubManager(), + fake = { + toJSON: sandbox.stub().returns({}) + }; + + manager.init().then(function () { + manager.tagAdded(fake); + manager.tags.addUrl.called.should.equal(true); + manager.tagEdited(fake); + manager.tags.updateUrl.called.should.equal(true); + manager.tagDeleted(fake); + manager.tags.removeUrl.called.should.equal(true); + + done(); + }).catch(done); + }); + }); + + describe('Generators', function () { + var stubPermalinks = function (generator) { + sandbox.stub(generator, 'getPermalinksValue', function () { + return Promise.resolve({ + id: 13, + uuid: 'ac6d6bb2-0b64-4941-b5ef-e69000bb738a', + key: 'permalinks', + value: '/:slug/', + type: 'blog' + }); + }); + + return generator; + }, + stubUrl = function (generator) { + sandbox.stub(generator, 'getUrlForDatum', function (datum) { + return 'http://my-ghost-blog.com/url/' + datum.id; + }); + sandbox.stub(generator, 'getUrlForImage', function (image) { + return 'http://my-ghost-blog.com/images/' + image; + }); + + return generator; + }, + makeFakeDatum = function (id) { + return { + id: id, + created_at: (Date.UTC(2014, 11, 22, 12) - 360000) + id + }; + }; + + describe('BaseGenerator', function () { + it('can initialize with empty siteMapContent', function (done) { + var generator = new BaseGenerator(); + + stubPermalinks(generator); + + generator.init().then(function () { + should.exist(generator.siteMapContent); + + validator.contains(generator.siteMapContent, '').should.equal(false); + + done(); + }).catch(done); + }); + + it('can initialize with non-empty siteMapContent', function (done) { + var generator = new BaseGenerator(); + + stubPermalinks(generator); + stubUrl(generator); + + sandbox.stub(generator, 'getData', function () { + return Promise.resolve([ + makeFakeDatum(100), + makeFakeDatum(200), + makeFakeDatum(300) + ]); + }); + + generator.init().then(function () { + should.exist(generator.siteMapContent); + + // TODO: We should validate the contents against the XSD: + // xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + // xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd" + + validator.contains(generator.siteMapContent, + 'http://my-ghost-blog.com/url/100').should.equal(true); + validator.contains(generator.siteMapContent, + 'http://my-ghost-blog.com/url/200').should.equal(true); + validator.contains(generator.siteMapContent, + 'http://my-ghost-blog.com/url/300').should.equal(true); + + done(); + }).catch(done); + }); + }); + + describe('PostGenerator', function () { + it('uses 0.8 priority for all posts', function () { + var generator = new PostGenerator(); + + generator.getPriorityForDatum({}).should.equal(0.8); + }); + + it('adds an image:image element if post has a cover image', function () { + var generator = new PostGenerator(), + urlNode = generator.createUrlNodeFromDatum(_.extend(makeFakeDatum(100), { + image: 'post-100.jpg' + })), + hasImage; + + hasImage = _.any(urlNode.url, function (node) { + return !_.isUndefined(node['image:image']); + }); + + hasImage.should.equal(true); + }); + + it('can initialize with non-empty siteMapContent', function (done) { + var generator = new PostGenerator(); + + stubPermalinks(generator); + stubUrl(generator); + + sandbox.stub(generator, 'getData', function () { + return Promise.resolve([ + _.extend(makeFakeDatum(100), { + image: 'post-100.jpg' + }), + makeFakeDatum(200), + _.extend(makeFakeDatum(300), { + image: 'post-300.jpg' + }) + ]); + }); + + generator.init().then(function () { + should.exist(generator.siteMapContent); + + // TODO: We should validate the contents against the XSD: + // xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + // xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 + // http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd" + + validator.contains(generator.siteMapContent, + 'http://my-ghost-blog.com/url/100').should.equal(true); + validator.contains(generator.siteMapContent, + 'http://my-ghost-blog.com/url/200').should.equal(true); + validator.contains(generator.siteMapContent, + 'http://my-ghost-blog.com/url/300').should.equal(true); + + validator.contains(generator.siteMapContent, + 'http://my-ghost-blog.com/images/post-100.jpg') + .should.equal(true); + // This should NOT be present + validator.contains(generator.siteMapContent, + 'http://my-ghost-blog.com/images/post-200.jpg') + .should.equal(false); + validator.contains(generator.siteMapContent, + 'http://my-ghost-blog.com/images/post-300.jpg') + .should.equal(true); + + done(); + }).catch(done); + }); + }); + + describe('PageGenerator', function () { + it('uses 1 priority for home page', function () { + var generator = new PageGenerator(); + + generator.getPriorityForDatum({ + name: 'home' + }).should.equal(1); + }); + it('uses 0.8 priority for static pages', function () { + var generator = new PageGenerator(); + + generator.getPriorityForDatum({}).should.equal(0.8); + }); + }); + + describe('TagGenerator', function () { + it('uses 0.6 priority for all tags', function () { + var generator = new TagGenerator(); + + generator.getPriorityForDatum({}).should.equal(0.6); + }); + }); + + describe('UserGenerator', function () { + it('uses 0.6 priority for author links', function () { + var generator = new UserGenerator(); + + generator.getPriorityForDatum({}).should.equal(0.6); + }); + }); + }); +});