mirror of
https://github.com/TryGhost/Ghost.git
synced 2025-02-24 23:48:13 -05:00
Added comments count endpoint to robots.txt disallow list
fix https://linear.app/tryghost/issue/ENG-771/add-comments-count-endpoint-to-robotstxt-ignorelist - we've seen web scrapers hitting this endpoint a lot, but the value to be taken from it is minimal for SEO purposes - adding it to robots.txt should encourage web scrapers to ignore it, and we should see less traffic as a result
This commit is contained in:
parent
7cc65c18cc
commit
27cc32ec25
2 changed files with 2 additions and 0 deletions
|
@ -2,5 +2,6 @@ User-agent: *
|
|||
Sitemap: {{blog-url}}/sitemap.xml
|
||||
Disallow: /ghost/
|
||||
Disallow: /email/
|
||||
Disallow: /members/api/comments/counts/
|
||||
Disallow: /r/
|
||||
Disallow: /webmentions/receive/
|
||||
|
|
|
@ -323,6 +323,7 @@ describe('Default Frontend routing', function () {
|
|||
'User-agent: *\n' +
|
||||
'Sitemap: http://127.0.0.1:2369/sitemap.xml\nDisallow: /ghost/\n' +
|
||||
'Disallow: /email/\n' +
|
||||
'Disallow: /members/api/comments/counts/\n' +
|
||||
'Disallow: /r/\n' +
|
||||
'Disallow: /webmentions/receive/\n'
|
||||
);
|
||||
|
|
Loading…
Add table
Reference in a new issue