feat: allow arrays of patterns for glob loader (#11952)

* feat: support pattern arrays with glob

* feat: allow arrays of patterns for content layer

* Apply suggestions from code review

Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

---------

Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

.changeset/honest-dingos-add.md

@@ -0,0 +1,23 @@
+---
+'astro': patch
+---
+
+Adds support for array patterns in the built-in `glob()` content collections loader
+
+The glob loader can now accept an array of multiple patterns as well as string patterns. This allows you to more easily combine multiple patterns into a single collection, and also means you can use negative matches to exclude files from the collection.
+
+```ts
+const probes = defineCollection({
+  // Load all markdown files in the space-probes directory, except for those that start with "voyager-"
+  loader: glob({ pattern: ['*.md', '!voyager-*'], base: 'src/data/space-probes' }),
+  schema: z.object({
+    name: z.string(),
+    type: z.enum(['Space Probe', 'Mars Rover', 'Comet Lander']),
+    launch_date: z.date(),
+    status: z.enum(['Active', 'Inactive', 'Decommissioned']),
+    destination: z.string(),
+    operator: z.string(),
+    notable_discoveries: z.array(z.string()),
+  }),
+});
+```

packages/astro/src/content/loaders/glob.ts

@@ -21,7 +21,7 @@ export interface GenerateIdOptions {
 
 export interface GlobOptions {
 	/** The glob pattern to match files, relative to the base directory */
-	pattern: string;
+	pattern: string | Array<string>;
 	/** The base directory to resolve the glob pattern from. Relative to the root directory, or an absolute file URL. Defaults to `.` */
 	base?: string | URL;
 	/**
@@ -44,17 +44,24 @@ function generateIdDefault({ entry, base, data }: GenerateIdOptions): string {
 	return slug;
 }
 
+function checkPrefix(pattern: string | Array<string>, prefix: string) {
+	if (Array.isArray(pattern)) {
+		return pattern.some((p) => p.startsWith(prefix));
+	}
+	return pattern.startsWith(prefix);
+}
+
 /**
  * Loads multiple entries, using a glob pattern to match files.
  * @param pattern A glob pattern to match files, relative to the content directory.
  */
 export function glob(globOptions: GlobOptions): Loader {
-	if (globOptions.pattern.startsWith('../')) {
+	if (checkPrefix(globOptions.pattern, '../')) {
 		throw new Error(
 			'Glob patterns cannot start with `../`. Set the `base` option to a parent directory instead.',
 		);
 	}
-	if (globOptions.pattern.startsWith('/')) {
+	if (checkPrefix(globOptions.pattern, '/')) {
 		throw new Error(
 			'Glob patterns cannot start with `/`. Set the `base` option to a parent directory or use a relative path instead.',
 		);
@@ -229,13 +236,17 @@ export function glob(globOptions: GlobOptions): Loader {
 			const skipCount = skippedFiles.length;
 
 			if (skipCount > 0) {
+				const patternList = Array.isArray(globOptions.pattern)
+					? globOptions.pattern.join(', ')
+					: globOptions.pattern;
+
 				logger.warn(`The glob() loader cannot be used for files in ${bold('src/content')}.`);
 				if (skipCount > 10) {
 					logger.warn(
-						`Skipped ${green(skippedFiles.length)} files that matched ${green(globOptions.pattern)}.`,
+						`Skipped ${green(skippedFiles.length)} files that matched ${green(patternList)}.`,
 					);
 				} else {
-					logger.warn(`Skipped the following files that matched ${green(globOptions.pattern)}:`);
+					logger.warn(`Skipped the following files that matched ${green(patternList)}:`);
 					skippedFiles.forEach((file) => logger.warn(`• ${green(file)}`));
 				}
 			}
@@ -247,9 +258,8 @@ export function glob(globOptions: GlobOptions): Loader {
 				return;
 			}
 
-			const matcher: RegExp = micromatch.makeRe(globOptions.pattern);
-
-			const matchesGlob = (entry: string) => !entry.startsWith('../') && matcher.test(entry);
+			const matchesGlob = (entry: string) =>
+				!entry.startsWith('../') && micromatch.isMatch(entry, globOptions.pattern);
 
 			const basePath = fileURLToPath(baseDir);
 

packages/astro/test/content-layer.test.js

@@ -83,6 +83,13 @@ describe('Content Layer', () => {
 			]);
 		});
 
+		it('handles negative matches in glob() loader', async () => {
+			assert.ok(json.hasOwnProperty('probes'));
+			assert.ok(Array.isArray(json.probes));
+			assert.equal(json.probes.length, 5);
+			assert.equal(json.probes.at(-1).id, 'philae-lander', 'Voyager probes should not be included');
+		});
+
 		it('Returns data entry by id', async () => {
 			assert.ok(json.hasOwnProperty('dataEntry'));
 			assert.equal(json.dataEntry.filePath?.split(sep).join(posixSep), 'src/data/dogs.json');

packages/astro/test/fixtures/content-layer/src/content/config.ts

@@ -66,7 +66,7 @@ const cats = defineCollection({
 });
 
 // Absolute paths should also work
-const absoluteRoot = new URL('../../content-outside-src', import.meta.url);
+const absoluteRoot = new URL('../../content/space', import.meta.url);
 
 const spacecraft = defineCollection({
 	loader: glob({ pattern: '*.md', base: absoluteRoot }),
@@ -78,10 +78,26 @@ const spacecraft = defineCollection({
 			tags: z.array(z.string()),
 			heroImage: image().optional(),
 			cat: reference('cats').optional(),
-			something: z.string().optional().transform(str => ({ type: 'test', content: str }))
+			something: z
+				.string()
+				.optional()
+				.transform((str) => ({ type: 'test', content: str })),
 		}),
 });
 
+const probes = defineCollection({
+	loader: glob({ pattern: ['*.md', '!voyager-*'], base: 'src/data/space-probes' }),
+	schema: z.object({
+		name: z.string(),
+		type: z.enum(['Space Probe', 'Mars Rover', 'Comet Lander']),
+		launch_date: z.date(),
+		status: z.enum(['Active', 'Inactive', 'Decommissioned']),
+		destination: z.string(),
+		operator: z.string(),
+		notable_discoveries: z.array(z.string()),
+	}),
+});
+
 const numbers = defineCollection({
 	loader: glob({ pattern: 'src/data/glob-data/*', base: '.' }),
 });
@@ -90,24 +106,25 @@ const images = defineCollection({
 	loader: () => [
 		{
 			id: '1',
-			image: '@images/shuttle.jpg'
+			image: '@images/shuttle.jpg',
 		},
 		{
 			id: '2',
-			image: 'https://images.unsplash.com/photo-1457364887197-9150188c107b?w=800&fm=jpg&fit=crop'
-		}
+			image: 'https://images.unsplash.com/photo-1457364887197-9150188c107b?w=800&fm=jpg&fit=crop',
+		},
 	],
-	schema: ({image}) => z.object({
-		id: z.string(),
-		image: image()
-	})
+	schema: ({ image }) =>
+		z.object({
+			id: z.string(),
+			image: image(),
+		}),
 });
 
 const increment = defineCollection({
 	loader: {
 		name: 'increment-loader',
 		load: async ({ store }) => {
-			const entry = store.get<{lastValue: number}>('value');
+			const entry = store.get<{ lastValue: number }>('value');
 			const lastValue = entry?.data.lastValue ?? 0;
 			store.set({
 				id: 'value',
@@ -118,12 +135,12 @@ const increment = defineCollection({
 			});
 		},
 		// Example of a loader that returns an async schema function
-		schema: async () => z.object({
-			lastValue: z.number(),
-			lastUpdated: z.date(),
-
-		}),
+		schema: async () =>
+			z.object({
+				lastValue: z.number(),
+				lastUpdated: z.date(),
+			}),
 	},
 });
 
-export const collections = { blog, dogs, cats, numbers, spacecraft, increment, images };
+export const collections = { blog, dogs, cats, numbers, spacecraft, increment, images, probes };

packages/astro/test/fixtures/content-layer/src/data/space-probes/cassini.md

@@ -0,0 +1,14 @@
+---
+name: Cassini
+type: Space Probe
+launch_date: 1997-10-15
+status: Decommissioned
+destination: Saturn
+operator: NASA/ESA/ASI
+notable_discoveries:
+  - Liquid methane seas on Titan
+  - Enceladus' subsurface ocean
+  - New Saturn rings and moons
+---
+
+The Cassini-Huygens mission was a collaboration between NASA, ESA, and ASI to study Saturn and its system. Launched in 1997, it arrived at Saturn in 2004 and operated until 2017. The mission dramatically improved our understanding of Saturn, its rings, and its moons, particularly Titan and Enceladus.

packages/astro/test/fixtures/content-layer/src/data/space-probes/curiosity-rover.md

@@ -0,0 +1,14 @@
+---
+name: Curiosity Rover
+type: Mars Rover
+launch_date: 2011-11-26
+status: Active
+destination: Mars
+operator: NASA
+notable_discoveries:
+  - Evidence of ancient streambeds
+  - Organic molecules in rocks
+  - Methane fluctuations in atmosphere
+---
+
+The Curiosity rover, part of NASA's Mars Science Laboratory mission, landed on Mars in 2012. Its primary goal is to determine if Mars could have supported microbial life. The rover has made significant discoveries about Mars' geology and climate, and continues to explore the Gale crater.

packages/astro/test/fixtures/content-layer/src/data/space-probes/juno.md

@@ -0,0 +1,14 @@
+---
+name: Juno
+type: Space Probe
+launch_date: 2011-08-05
+status: Active
+destination: Jupiter
+operator: NASA
+notable_discoveries:
+  - Jupiter's deep atmospheric dynamics
+  - Complex magnetic field structure
+  - Insights into Jupiter's core structure
+---
+
+Juno is a NASA space probe orbiting Jupiter. It was launched in 2011 and entered Jupiter's orbit in 2016. The spacecraft's mission is to measure Jupiter's composition, gravity field, magnetic field, and polar magnetosphere. Juno has provided new insights into Jupiter's interior structure and the processes that drive its intense magnetic fields and aurorae.

packages/astro/test/fixtures/content-layer/src/data/space-probes/new-horizons.md

@@ -0,0 +1,14 @@
+---
+name: New Horizons
+type: Space Probe
+launch_date: 2006-01-19
+status: Active
+destination: Pluto and Kuiper Belt
+operator: NASA
+notable_discoveries:
+  - Pluto's heart-shaped glacier
+  - Pluto's thin atmosphere
+  - Kuiper Belt Object Arrokoth's unusual shape
+---
+
+New Horizons is an interplanetary space probe launched as part of NASA's New Frontiers program. It performed the first flyby of Pluto in 2015, providing unprecedented data about the dwarf planet. After its Pluto mission, New Horizons continued into the Kuiper Belt, where it encountered the object Arrokoth in 2019, the most distant object in the Solar System visited by a spacecraft.

packages/astro/test/fixtures/content-layer/src/data/space-probes/philae-lander.md

@@ -0,0 +1,14 @@
+---
+name: Philae Lander
+type: Comet Lander
+launch_date: 2004-03-02
+status: Inactive
+destination: Comet 67P/Churyumov-Gerasimenko
+operator: ESA
+notable_discoveries:
+  - Organic molecules on the comet's surface
+  - Comet's surface hardness
+  - Presence of water ice
+---
+
+Philae was a robotic European Space Agency lander that accompanied the Rosetta spacecraft. It achieved the first-ever soft landing on a comet nucleus when it touched down on comet 67P/Churyumov-Gerasimenko in November 2014. Despite its short operational life, Philae provided unique data about the comet's composition and structure.

packages/astro/test/fixtures/content-layer/src/data/space-probes/voyager-1.md

@@ -0,0 +1,14 @@
+---
+name: Voyager 1
+type: Space Probe
+launch_date: 1977-09-05
+status: Active
+destination: Interstellar space
+operator: NASA
+notable_discoveries:
+  - Jupiter's complex cloud structures
+  - Active volcanoes on Io
+  - Saturn's ring structure
+---
+
+Voyager 1 is NASA's farthest and fastest-traveling space probe. Launched in 1977, it has been operating for over 45 years and entered interstellar space in 2012. The probe has provided invaluable data about the outer planets and the boundary between the Sun's influence and interstellar space.

packages/astro/test/fixtures/content-layer/src/data/space-probes/voyager-2.md

@@ -0,0 +1,16 @@
+---
+name: Voyager 2
+type: Space Probe
+launch_date: 1977-08-20
+status: Active
+destination: Interstellar space
+operator: NASA
+notable_discoveries:
+  - Neptune's Great Dark Spot
+  - Uranus' tilted magnetic field
+  - Active geysers on Neptune's moon Triton
+  - Jupiter's complex storm systems
+  - Saturn's intricate ring structure
+---
+
+Voyager 2 is a space probe launched by NASA as part of the Voyager program to study the outer Solar System and interstellar space. Despite being launched 16 days before Voyager 1, it's named Voyager 2 due to its slower trajectory. It's the only spacecraft to have visited all four gas giant planets: Jupiter, Saturn, Uranus, and Neptune. After completing its planetary mission, Voyager 2 continued on to study the outer reaches of the Solar System and entered interstellar space in 2018, becoming the second human-made object to do so after Voyager 1.

packages/astro/test/fixtures/content-layer/src/pages/collections.json.js

@@ -17,6 +17,8 @@ export async function GET() {
 	const increment = await getEntry('increment', 'value');
 
 	const images = await getCollection('images');
+
+	const probes = await getCollection('probes');
 	return new Response(
 		devalue.stringify({
 			customLoader,
@@ -26,7 +28,8 @@ export async function GET() {
 			entryWithReference,
 			referencedEntry,
 			increment,
-			images
+			images, 
+			probes
 		})
 	);
 }