diff --git a/core/server/middleware/index.js b/core/server/middleware/index.js index 26bcebeac4..285e4a71b1 100644 --- a/core/server/middleware/index.js +++ b/core/server/middleware/index.js @@ -194,6 +194,41 @@ function checkSSL(req, res, next) { next(); } +// ### Robots Middleware +// Handle requests to robots.txt and cache file +function robots() { + var content, // file cache + filePath = path.join(config().paths.corePath, '/shared/robots.txt'); + + return function robots(req, res, next) { + if ('/robots.txt' === req.url) { + if (content) { + res.writeHead(200, content.headers); + res.end(content.body); + } else { + fs.readFile(filePath, function (err, buf) { + if (err) { + return next(err); + } + + content = { + headers: { + 'Content-Type': 'text/plain', + 'Content-Length': buf.length, + 'Cache-Control': 'public, max-age=' + ONE_YEAR_MS / 1000 + }, + body: buf + }; + res.writeHead(200, content.headers); + res.end(content.body); + }); + } + } else { + next(); + } + }; +} + module.exports = function (server, dbHash) { var logging = config().logging, subdir = config().paths.subdir, @@ -229,7 +264,6 @@ module.exports = function (server, dbHash) { // First determine whether we're serving admin or theme content expressServer.use(manageAdminAndTheme); - // Admin only config expressServer.use(subdir + '/ghost', middleware.whenEnabled('admin', express['static'](path.join(corePath, '/client/assets'), {maxAge: ONE_YEAR_MS}))); @@ -242,6 +276,9 @@ module.exports = function (server, dbHash) { // Theme only config expressServer.use(subdir, middleware.whenEnabled(expressServer.get('activeTheme'), middleware.staticTheme())); + // Serve robots.txt if not found in theme + expressServer.use(robots()); + // Add in all trailing slashes expressServer.use(slashes(true, {headers: {'Cache-Control': 'public, max-age=' + ONE_YEAR_S}})); diff --git a/core/test/functional/routes/frontend_test.js b/core/test/functional/routes/frontend_test.js index dddab36c58..351d06c650 100644 --- a/core/test/functional/routes/frontend_test.js +++ b/core/test/functional/routes/frontend_test.js @@ -282,6 +282,13 @@ describe('Frontend Routing', function () { .end(doEnd(done)); }); + it('should retrieve default robots.txt', function (done) { + request.get('/robots.txt') + .expect('Cache-Control', cacheRules.year) + .expect(200) + .end(doEnd(done)); + }) + // at the moment there is no image fixture to test // it('should retrieve image assets', function (done) { // request.get('/content/images/some.jpg')