fix: add noindex to server island headers (#12827)

* fix: add noindex to server island headers

* test: check if server island has noindex header

* chore: changeset

* refactor: set X-Robots-Tag: noindex in handler

* fix tests

---------

Co-authored-by: Emanuele Stoppa <my.burning@gmail.com>

.changeset/witty-lies-end.md

@@ -0,0 +1,5 @@
+---
+'astro': patch
+---
+
+Fixes an issue when crawlers try to index Server Islands thinking that Server Islands are pages

packages/astro/src/core/server-islands/endpoint.ts

@@ -104,6 +104,7 @@ export function createEndpoint(manifest: SSRManifest) {
 
 		// Get the request data from the body or search params
 		const data = await getRequestData(result.request);
+		// probably error
 		if (data instanceof Response) {
 			return data;
 		}
@@ -130,6 +131,9 @@ export function createEndpoint(manifest: SSRManifest) {
 			slots[prop] = createSlotValueFromString(data.slots[prop]);
 		}
 
+		// Prevent server islands from being indexed
+		result.response.headers.set('X-Robots-Tag', 'noindex');
+
 		// Wrap Astro components so we can set propagation to
 		// `self` which is needed to force the runtime to wait
 		// on the component before sending out the response headers.

packages/astro/test/server-islands.test.js

@@ -37,6 +37,18 @@ describe('Server islands', () => {
 				assert.equal(serverIslandEl.length, 0);
 			});
 
+			it('island is not indexed', async () => {
+				const res = await fixture.fetch('/_server-islands/Island', {
+					method: 'POST',
+					body: JSON.stringify({
+						componentExport: 'default',
+						encryptedProps: 'FC8337AF072BE5B1641501E1r8mLIhmIME1AV7UO9XmW9OLD',
+						slots: {},
+					}),
+				});
+				assert.equal(res.headers.get('x-robots-tag'), 'noindex');
+			});
+
 			it('island can set headers', async () => {
 				const res = await fixture.fetch('/_server-islands/Island', {
 					method: 'POST',
@@ -74,6 +86,23 @@ describe('Server islands', () => {
 				const serverIslandScript = $('script[data-island-id]');
 				assert.equal(serverIslandScript.length, 1, 'has the island script');
 			});
+
+			it('island is not indexed', async () => {
+				const app = await fixture.loadTestAdapterApp();
+				const request = new Request('http://example.com/_server-islands/Island', {
+					method: 'POST',
+					body: JSON.stringify({
+						componentExport: 'default',
+						encryptedProps: 'FC8337AF072BE5B1641501E1r8mLIhmIME1AV7UO9XmW9OLD',
+						slots: {},
+					}),
+					headers: {
+						origin: 'http://example.com',
+					},
+				});
+				const response = await app.render(request);
+				assert.equal(response.headers.get('x-robots-tag'), 'noindex');
+			});
 		});
 	});