Infrastructure/astro
0
Fork 0
mirror of https://github.com/withastro/astro.git synced 2025-01-27 22:19:04 -05:00
Code Issues Activity

feat: allow integrations to refresh content layer data (#11878)

Browse source 
* Allow integrations to refresh content layer data

This reverts commit 90a862fc94.

* Add test

* Add changeset

* Add wait in test

* Dispose of queue

* Skip if no content layer collections

* Use spaces in markdown

* fix: don't keep data store in node_modules during dev

* Lint

* Fix test

* Apply suggestions from code review

Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

* Re-order tests

* Wait for data store

* Lint

* Handle case where Vite already knows about save

---------

Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>
This commit is contained in:
Matt Kane 2024-09-03 12:44:57 +01:00 • committed by GitHub
parent 4563ffcd89
commit 334948ced2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 209 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
.changeset/curvy-walls-kneel.md Normal file
View file

@ -0,0 +1,45 @@
---
'astro': patch
---
Adds a new function `refreshContent` to the `astro:server:setup` hook that allows integrations to refresh the content layer. This can be used, for example, to register a webhook endpoint during dev, or to open a socket to a CMS to listen for changes.
By default, `refreshContent` will refresh all collections. You can optionally pass a `loaders` property, which is an array of loader names. If provided, only collections that use those loaders will be refreshed. For example, A CMS integration could use this property to only refresh its own collections.
You can also pass a `context` object to the loaders. This can be used to pass arbitrary data, such as the webhook body, or an event from the websocket.
```ts
{
name: 'my-integration',
hooks: {
'astro:server:setup': async ({ server, refreshContent }) => {
server.middlewares.use('/_refresh', async (req, res) => {
if(req.method !== 'POST') {
res.statusCode = 405
res.end('Method Not Allowed');
return
}
let body = '';
req.on('data', chunk => {
body += chunk.toString();
});
req.on('end', async () => {
try {
const webhookBody = JSON.parse(body);
await refreshContent({
context: { webhookBody },
loaders: ['my-loader']
});
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ message: 'Content refreshed successfully' }));
} catch (error) {
res.writeHead(500, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: 'Failed to refresh content: ' + error.message }));
}
});
});
}
}
}
```

18
packages/astro/src/content/content-layer.ts
View file

@ -73,6 +73,11 @@ export class ContentLayer {
this.#unsubscribe?.();
}
dispose() {
this.#queue.kill();
this.#unsubscribe?.();
}
async #getGenerateDigest() {
if (this.#generateDigest) {
return this.#generateDigest;
@ -224,7 +229,7 @@ export class ContentLayer {
if (!existsSync(this.#settings.config.cacheDir)) {
await fs.mkdir(this.#settings.config.cacheDir, { recursive: true });
}
const cacheFile = new URL(DATA_STORE_FILE, this.#settings.config.cacheDir);
const cacheFile = getDataStoreFile(this.#settings);
await this.#store.writeToDisk(cacheFile);
if (!existsSync(this.#settings.dotAstroDir)) {
await fs.mkdir(this.#settings.dotAstroDir, { recursive: true });
@ -285,17 +290,24 @@ export async function simpleLoader<TData extends { id: string }>(
}
}
export function getDataStoreFile(settings: AstroSettings) {
return new URL(
DATA_STORE_FILE,
process?.env.NODE_ENV === 'development' ? settings.dotAstroDir : settings.config.cacheDir,
);
}
function contentLayerSingleton() {
let instance: ContentLayer | null = null;
return {
init: (options: ContentLayerOptions) => {
instance?.unwatchContentConfig();
instance?.dispose();
instance = new ContentLayer(options);
return instance;
},
get: () => instance,
dispose: () => {
instance?.unwatchContentConfig();
instance?.dispose();
instance = null;
},
};

3
packages/astro/src/content/data-store.ts
View file

@ -91,6 +91,9 @@ export class DataStore {
try {
// @ts-expect-error - this is a virtual module
const data = await import('astro:data-layer-content');
if (data.default instanceof Map) {
return DataStore.fromMap(data.default);
}
const map = devalue.unflatten(data.default);
return DataStore.fromMap(map);
} catch {}

1
packages/astro/src/content/loaders/types.ts
View file

@ -33,6 +33,7 @@ export interface LoaderContext {
/** When running in dev, this is a filesystem watcher that can be used to trigger updates */
watcher?: FSWatcher;
/** If the loader has been triggered by an integration, this may optionally contain extra data set by that integration */
refreshContextData?: Record<string, unknown>;
entryTypes: Map<string, ContentEntryType>;
}

42
packages/astro/src/content/vite-plugin-content-virtual-mod.ts
View file

@ -54,12 +54,16 @@ export function astroContentVirtualModPlugin({
}: AstroContentVirtualModPluginParams): Plugin {
let IS_DEV = false;
const IS_SERVER = isServerLikeOutput(settings.config);
const dataStoreFile = new URL(DATA_STORE_FILE, settings.config.cacheDir);
let dataStoreFile: URL;
return {
name: 'astro-content-virtual-mod-plugin',
enforce: 'pre',
configResolved(config) {
IS_DEV = config.mode === 'development';
dataStoreFile = new URL(
DATA_STORE_FILE,
IS_DEV ? settings.dotAstroDir : settings.config.cacheDir,
);
},
async resolveId(id) {
if (id === VIRTUAL_MODULE_ID) {
@ -180,25 +184,31 @@ export function astroContentVirtualModPlugin({
configureServer(server) {
const dataStorePath = fileURLToPath(dataStoreFile);
// Watch for changes to the data store file
if (Array.isArray(server.watcher.options.ignored)) {
// The data store file is in node_modules, so is ignored by default,
// so we need to un-ignore it.
server.watcher.options.ignored.push(`!${dataStorePath}`);
}
server.watcher.add(dataStorePath);
function invalidateDataStore() {
const module = server.moduleGraph.getModuleById(RESOLVED_DATA_STORE_VIRTUAL_ID);
if (module) {
server.moduleGraph.invalidateModule(module);
}
server.ws.send({
type: 'full-reload',
path: '*',
});
}
// If the datastore file changes, invalidate the virtual module
server.watcher.on('add', (addedPath) => {
if (addedPath === dataStorePath) {
invalidateDataStore();
}
});
server.watcher.on('change', (changedPath) => {
// If the datastore file changes, invalidate the virtual module
if (changedPath === dataStorePath) {
const module = server.moduleGraph.getModuleById(RESOLVED_DATA_STORE_VIRTUAL_ID);
if (module) {
server.moduleGraph.invalidateModule(module);
}
server.ws.send({
type: 'full-reload',
path: '*',
});
invalidateDataStore();
}
});
},

7
packages/astro/src/core/build/index.ts
View file

@ -56,6 +56,9 @@ export default async function build(
const logger = createNodeLogger(inlineConfig);
const { userConfig, astroConfig } = await resolveConfig(inlineConfig, 'build');
telemetry.record(eventCliSession('build', userConfig));
const settings = await createSettings(astroConfig, fileURLToPath(astroConfig.root));
if (inlineConfig.force) {
if (astroConfig.experimental.contentCollectionCache) {
const contentCacheDir = new URL('./content/', astroConfig.cacheDir);
@ -65,11 +68,9 @@ export default async function build(
logger.warn('content', 'content cache cleared (force)');
}
}
await clearContentLayerCache({ astroConfig, logger, fs });
await clearContentLayerCache({ settings, logger, fs });
}
const settings = await createSettings(astroConfig, fileURLToPath(astroConfig.root));
const builder = new AstroBuilder(settings, {
...options,
logger,

5
packages/astro/src/core/dev/dev.ts
View file

@ -5,8 +5,7 @@ import { performance } from 'node:perf_hooks';
import { green } from 'kleur/colors';
import { gt, major, minor, patch } from 'semver';
import type * as vite from 'vite';
import { DATA_STORE_FILE } from '../../content/consts.js';
import { globalContentLayer } from '../../content/content-layer.js';
import { getDataStoreFile, globalContentLayer } from '../../content/content-layer.js';
import { attachContentServerListeners } from '../../content/index.js';
import { MutableDataStore } from '../../content/mutable-data-store.js';
import { globalContentConfigObserver } from '../../content/utils.js';
@ -108,7 +107,7 @@ export default async function dev(inlineConfig: AstroInlineConfig): Promise<DevS
let store: MutableDataStore | undefined;
try {
const dataStoreFile = new URL(DATA_STORE_FILE, restart.container.settings.config.cacheDir);
const dataStoreFile = getDataStoreFile(restart.container.settings);
if (existsSync(dataStoreFile)) {
store = await MutableDataStore.fromFile(dataStoreFile);
}

16
packages/astro/src/core/sync/index.ts
View file

@ -3,8 +3,8 @@ import { performance } from 'node:perf_hooks';
import { fileURLToPath } from 'node:url';
import { dim } from 'kleur/colors';
import { type HMRPayload, createServer } from 'vite';
import { CONTENT_TYPES_FILE, DATA_STORE_FILE } from '../../content/consts.js';
import { globalContentLayer } from '../../content/content-layer.js';
import { CONTENT_TYPES_FILE } from '../../content/consts.js';
import { getDataStoreFile, globalContentLayer } from '../../content/content-layer.js';
import { createContentTypesGenerator } from '../../content/index.js';
import { MutableDataStore } from '../../content/mutable-data-store.js';
import { getContentPaths, globalContentConfigObserver } from '../../content/utils.js';
@ -13,7 +13,7 @@ import { telemetry } from '../../events/index.js';
import { eventCliSession } from '../../events/session.js';
import { runHookConfigDone, runHookConfigSetup } from '../../integrations/hooks.js';
import type { AstroSettings } from '../../types/astro.js';
import type { AstroConfig, AstroInlineConfig } from '../../types/public/config.js';
import type { AstroInlineConfig } from '../../types/public/config.js';
import { getTimeStat } from '../build/util.js';
import { resolveConfig } from '../config/config.js';
import { createNodeLogger } from '../config/logging.js';
@ -70,11 +70,11 @@ export default async function sync(
* Clears the content layer and content collection cache, forcing a full rebuild.
*/
export async function clearContentLayerCache({
astroConfig,
settings,
logger,
fs = fsMod,
}: { astroConfig: AstroConfig; logger: Logger; fs?: typeof fsMod }) {
const dataStore = new URL(DATA_STORE_FILE, astroConfig.cacheDir);
}: { settings: AstroSettings; logger: Logger; fs?: typeof fsMod }) {
const dataStore = getDataStoreFile(settings);
if (fs.existsSync(dataStore)) {
logger.debug('content', 'clearing data store');
await fs.promises.rm(dataStore, { force: true });
@ -96,7 +96,7 @@ export async function syncInternal({
force,
}: SyncOptions): Promise<void> {
if (force) {
await clearContentLayerCache({ astroConfig: settings.config, logger, fs });
await clearContentLayerCache({ settings, logger, fs });
}
const timerStart = performance.now();
@ -107,7 +107,7 @@ export async function syncInternal({
settings.timer.start('Sync content layer');
let store: MutableDataStore | undefined;
try {
const dataStoreFile = new URL(DATA_STORE_FILE, settings.config.cacheDir);
const dataStoreFile = getDataStoreFile(settings);
if (existsSync(dataStoreFile)) {
store = await MutableDataStore.fromFile(dataStoreFile);
}

28
packages/astro/src/integrations/hooks.ts
View file

@ -5,6 +5,9 @@ import { bold } from 'kleur/colors';
import type { InlineConfig, ViteDevServer } from 'vite';
import astroIntegrationActionsRouteHandler from '../actions/integration.js';
import { isActionsFilePresent } from '../actions/utils.js';
import { CONTENT_LAYER_TYPE } from '../content/consts.js';
import { globalContentLayer } from '../content/content-layer.js';
import { globalContentConfigObserver } from '../content/utils.js';
import type { SerializedSSRManifest } from '../core/app/types.js';
import type { PageBuildData } from '../core/build/types.js';
import { buildClientDirectiveEntrypoint } from '../core/client-directive/index.js';
@ -13,7 +16,11 @@ import type { AstroIntegrationLogger, Logger } from '../core/logger/core.js';
import { isServerLikeOutput } from '../core/util.js';
import type { AstroSettings } from '../types/astro.js';
import type { AstroConfig } from '../types/public/config.js';
import type { ContentEntryType, DataEntryType } from '../types/public/content.js';
import type {
ContentEntryType,
DataEntryType,
RefreshContentOptions,
} from '../types/public/content.js';
import type {
AstroIntegration,
AstroRenderer,
@ -367,6 +374,24 @@ export async function runHookServerSetup({
server: ViteDevServer;
logger: Logger;
}) {
let refreshContent: undefined | ((options: RefreshContentOptions) => Promise<void>);
if (config.experimental?.contentLayer) {
refreshContent = async (options: RefreshContentOptions) => {
const contentConfig = globalContentConfigObserver.get();
if (
contentConfig.status !== 'loaded' ||
!Object.values(contentConfig.config.collections).some(
(collection) => collection.type === CONTENT_LAYER_TYPE,
)
) {
return;
}
const contentLayer = await globalContentLayer.get();
await contentLayer?.sync(options);
};
}
for (const integration of config.integrations) {
if (integration?.hooks?.['astro:server:setup']) {
await withTakingALongTimeMsg({
@ -376,6 +401,7 @@ export async function runHookServerSetup({
server,
logger: getLogger(integration, logger),
toolbar: getToolbarServerCommunicationHelpers(server),
refreshContent,
}),
logger,
});

2
packages/astro/src/types/public/integrations.ts
View file

@ -6,6 +6,7 @@ import type { AstroIntegrationLogger } from '../../core/logger/core.js';
import type { getToolbarServerCommunicationHelpers } from '../../integrations/hooks.js';
import type { DeepPartial } from '../../type-utils.js';
import type { AstroConfig } from './config.js';
import type { RefreshContentOptions } from './content.js';
import type { RouteData } from './internal.js';
import type { DevToolbarAppEntry } from './toolbar.js';
@ -187,6 +188,7 @@ export interface BaseIntegrationHooks {
server: ViteDevServer;
logger: AstroIntegrationLogger;
toolbar: ReturnType<typeof getToolbarServerCommunicationHelpers>;
refreshContent?: (options: RefreshContentOptions) => Promise<void>;
}) => void | Promise<void>;
'astro:server:start': (options: {
address: AddressInfo;

26
packages/astro/test/content-layer.test.js
View file

@ -196,7 +196,11 @@ describe('Content Layer', () => {
let devServer;
let json;
before(async () => {
devServer = await fixture.startDevServer();
devServer = await fixture.startDevServer({ force: true });
// Vite may not have noticed the saved data store yet. Wait a little just in case.
await fixture.onNextDataStoreChange(1000).catch(() => {
// Ignore timeout, because it may have saved before we get here.
})
const rawJsonResponse = await fixture.fetch('/collections.json');
const rawJson = await rawJsonResponse.text();
json = devalue.parse(rawJson);
@ -275,6 +279,22 @@ describe('Content Layer', () => {
});
});
it('reloads data when an integration triggers a content refresh', async () => {
const rawJsonResponse = await fixture.fetch('/collections.json');
const initialJson = devalue.parse(await rawJsonResponse.text());
assert.equal(initialJson.increment.data.lastValue, 1);
const refreshResponse = await fixture.fetch('/_refresh', {
method: 'POST',
body: JSON.stringify({}),
});
const refreshData = await refreshResponse.json();
assert.equal(refreshData.message, 'Content refreshed successfully');
const updatedJsonResponse = await fixture.fetch('/collections.json');
const updated = devalue.parse(await updatedJsonResponse.text());
assert.equal(updated.increment.data.lastValue, 2);
});
it('updates collection when data file is changed', async () => {
const rawJsonResponse = await fixture.fetch('/collections.json');
const initialJson = devalue.parse(await rawJsonResponse.text());
@ -286,9 +306,7 @@ describe('Content Layer', () => {
return JSON.stringify(data, null, 2);
});
// Writes are debounced to 500ms
await new Promise((r) => setTimeout(r, 700));
await fixture.onNextDataStoreChange();
const updatedJsonResponse = await fixture.fetch('/collections.json');
const updated = devalue.parse(await updatedJsonResponse.text());
assert.ok(updated.fileLoader[0].data.temperament.includes('Bouncy'));

33
packages/astro/test/fixtures/content-layer/astro.config.mjs vendored
View file

@ -3,7 +3,38 @@ import { defineConfig } from 'astro/config';
import { fileURLToPath } from 'node:url';
export default defineConfig({
integrations: [mdx()],
integrations: [mdx(), {
name: '@astrojs/my-integration',
hooks: {
'astro:server:setup': async ({ server, refreshContent }) => {
server.middlewares.use('/_refresh', async (req, res) => {
if(req.method !== 'POST') {
res.statusCode = 405
res.end('Method Not Allowed');
return
}
let body = '';
req.on('data', chunk => {
body += chunk.toString();
});
req.on('end', async () => {
try {
const webhookBody = JSON.parse(body);
await refreshContent({
context: { webhookBody },
loaders: ['increment-loader']
});
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ message: 'Content refreshed successfully' }));
} catch (error) {
res.writeHead(500, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: 'Failed to refresh content: ' + error.message }));
}
});
});
}
}
}],
vite: {
resolve: {
alias: {

22
packages/astro/test/test-utils.js
View file

@ -45,6 +45,7 @@ process.env.ASTRO_TELEMETRY_DISABLED = true;
* @property {() => Promise<App>} loadTestAdapterApp
* @property {() => Promise<(req: NodeRequest, res: NodeResponse) => void>} loadNodeAdapterHandler
* @property {() => Promise<void>} onNextChange
* @property {(timeout?: number) => Promise<void>} onNextDataStoreChange
* @property {typeof check} check
* @property {typeof sync} sync
* @property {AstroConfig} config
@ -180,6 +181,27 @@ export async function loadFixture(inlineConfig) {
config.server.port = devServer.address.port; // update port
return devServer;
},
onNextDataStoreChange: (timeout = 5000) => {
if (!devServer) {
return Promise.reject(new Error('No dev server running'));
}
const dataStoreFile = path.join(root, '.astro', 'data-store.json');
return new Promise((resolve, reject) => {
const changeHandler = (fileName) => {
if (fileName === dataStoreFile) {
devServer.watcher.removeListener('change', changeHandler);
resolve();
}
};
devServer.watcher.on('change', changeHandler);
setTimeout(() => {
devServer.watcher.removeListener('change', changeHandler);
reject(new Error('Data store did not update within timeout'));
}, timeout);
});
},
config,
resolveUrl,
fetch: async (url, init) => {