Infrastructure/ghost
0
Fork 0
mirror of https://github.com/TryGhost/Ghost.git synced 2025-02-24 23:48:13 -05:00
Code Issues Activity

Added comments count endpoint to robots.txt disallow list

Browse source 
fix https://linear.app/tryghost/issue/ENG-771/add-comments-count-endpoint-to-robotstxt-ignorelist

- we've seen web scrapers hitting this endpoint a lot, but the value to
  be taken from it is minimal for SEO purposes
- adding it to robots.txt should encourage web scrapers to ignore it,
  and we should see less traffic as a result
This commit is contained in:
Daniel Lockyer 2024-03-20 14:29:34 +01:00 • committed by Daniel Lockyer
parent 7cc65c18cc
commit 27cc32ec25
2 changed files with 2 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
ghost/core/core/frontend/public/robots.txt
View file

@ -2,5 +2,6 @@ User-agent: *
Sitemap: {{blog-url}}/sitemap.xml Sitemap: {{blog-url}}/sitemap.xml
Disallow: /ghost/ Disallow: /ghost/
Disallow: /email/ Disallow: /email/
Disallow: /members/api/comments/counts/
Disallow: /r/ Disallow: /r/
Disallow: /webmentions/receive/ Disallow: /webmentions/receive/

1
ghost/core/test/e2e-frontend/default_routes.test.js
View file

@ -323,6 +323,7 @@ describe('Default Frontend routing', function () {
'User-agent: *\n' + 'User-agent: *\n' +
'Sitemap: http://127.0.0.1:2369/sitemap.xml\nDisallow: /ghost/\n' + 'Sitemap: http://127.0.0.1:2369/sitemap.xml\nDisallow: /ghost/\n' +
'Disallow: /email/\n' + 'Disallow: /email/\n' +
'Disallow: /members/api/comments/counts/\n' +
'Disallow: /r/\n' + 'Disallow: /r/\n' +
'Disallow: /webmentions/receive/\n' 'Disallow: /webmentions/receive/\n'
); );