From 2aaf4d60548f42b6921d5b12d404e3596c6493b6 Mon Sep 17 00:00:00 2001
From: Vikas Potluri <vikaspotluri123@users.noreply.github.com>
Date: Mon, 17 Sep 2018 04:29:47 -0500
Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9BDisallowed=20indexing=20of=20previe?=
 =?UTF-8?q?w=20pages=20(#9762)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

closes #9749

- disallow indexing of /p/ in robots.txt
- add meta robots tag to preview pages
- robots.txt wont' be applied for blogs not mounted to the root (i.e. https://example.com/blog/)
---
 core/server/helpers/ghost_head.js             |  7 ++++-
 core/server/public/robots.txt                 |  1 +
 .../services/routing/helpers/context.js       |  7 ++++-
 .../integration/helpers/ghost_head_spec.js    | 26 +++++++++++++++++++
 4 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/core/server/helpers/ghost_head.js b/core/server/helpers/ghost_head.js
index 847860057f..90a68a3dec 100644
--- a/core/server/helpers/ghost_head.js
+++ b/core/server/helpers/ghost_head.js
@@ -87,7 +87,7 @@ function getAjaxHelper(clientId, clientSecret) {
  *  }
  *
  * `blog`, `labs` and `config` are the templateOptions, search for `hbs.updateTemplateOptions` in the code base.
- *  Also see how the root object get's created, https://github.com/wycats/handlebars.js/blob/v4.0.6/lib/handlebars/runtime.js#L259
+ *  Also see how the root object gets created, https://github.com/wycats/handlebars.js/blob/v4.0.6/lib/handlebars/runtime.js#L259
  */
 // We use the name ghost_head to match the helper for consistency:
 module.exports = function ghost_head(options) { // eslint-disable-line camelcase
@@ -136,6 +136,11 @@ module.exports = function ghost_head(options) { // eslint-disable-line camelcase
                     escapeExpression(metaData.canonicalUrl) + '" />');
                 head.push('<meta name="referrer" content="' + referrerPolicy + '" />');
 
+                // don't allow indexing of preview URLs!
+                if (_.includes(context, 'preview')) {
+                    head.push(writeMetaTag('robots', 'noindex,nofollow', 'name'));
+                }
+
                 // show amp link in post when 1. we are not on the amp page and 2. amp is enabled
                 if (_.includes(context, 'post') && !_.includes(context, 'amp') && settingsCache.get('amp')) {
                     head.push('<link rel="amphtml" href="' +
diff --git a/core/server/public/robots.txt b/core/server/public/robots.txt
index b9f1b17caa..b93e0b9dee 100644
--- a/core/server/public/robots.txt
+++ b/core/server/public/robots.txt
@@ -1,3 +1,4 @@
 User-agent: *
 Sitemap: {{blog-url}}/sitemap.xml
 Disallow: /ghost/
+Disallow: /p/
diff --git a/core/server/services/routing/helpers/context.js b/core/server/services/routing/helpers/context.js
index ef284dde3a..512cc1f2fb 100644
--- a/core/server/services/routing/helpers/context.js
+++ b/core/server/services/routing/helpers/context.js
@@ -18,7 +18,8 @@ const labs = require('../../labs'),
     subscribePattern = new RegExp('^\\/subscribe\\/'),
     // routeKeywords.amp: 'amp'
     ampPattern = new RegExp('\\/amp\\/$'),
-    homePattern = new RegExp('^\\/$');
+    homePattern = new RegExp('^\\/$'),
+    previewPattern = new RegExp('^\\/p\\/');
 
 function setResponseContext(req, res, data) {
     var pageParam = req.params && req.params.page !== undefined ? parseInt(req.params.page, 10) : 1;
@@ -46,6 +47,10 @@ function setResponseContext(req, res, data) {
         res.locals.context.push('amp');
     }
 
+    if (previewPattern.test(res.locals.relativeUrl)) {
+        res.locals.context.push('preview');
+    }
+
     // Each page can only have at most one of these
     if (res.routerOptions && res.routerOptions.context) {
         res.locals.context = res.locals.context.concat(res.routerOptions.context);
diff --git a/core/test/integration/helpers/ghost_head_spec.js b/core/test/integration/helpers/ghost_head_spec.js
index 4d3e04f247..d5ba95894a 100644
--- a/core/test/integration/helpers/ghost_head_spec.js
+++ b/core/test/integration/helpers/ghost_head_spec.js
@@ -1218,6 +1218,32 @@ describe('{{ghost_head}} helper', function () {
             }).catch(done);
         });
 
+        it('disallows indexing for preview pages', function (done) {
+            helpers.ghost_head(testUtils.createHbsResponse({
+                locals: {
+                    context: ['preview', 'post']
+                }
+            })).then(function (rendered) {
+                should.exist(rendered);
+                rendered.string.should.match(/<meta name="robots" content="noindex,nofollow" \/>/);
+
+                done();
+            }).catch(done);
+        });
+
+        it('implicit indexing settings for non-preview pages', function (done) {
+            helpers.ghost_head(testUtils.createHbsResponse({
+                locals: {
+                    context: ['featured', 'paged', 'index', 'post', 'amp', 'home', 'unicorn']
+                }
+            })).then(function (rendered) {
+                should.exist(rendered);
+                rendered.string.should.not.match(/<meta name="robots" content="noindex,nofollow" \/>/);
+
+                done();
+            }).catch(done);
+        });
+
         it('outputs structured data but not schema for custom collection', function (done) {
             helpers.ghost_head(testUtils.createHbsResponse({
                 locals: {