0
Fork 0
mirror of https://github.com/withastro/astro.git synced 2025-03-24 23:21:57 -05:00

Add integrations API for db config/seed files (#10321)

* Add integrations API for adding db config/seed files

* Fix seeding when user seed file is present

* Add basic test and fixture for integrations API

* Freeze that lockfile

* Test to see if this is a Windows fix

* Don’t import.meta.glob integration seed files

* Make integration seed files export a default function

* style: rejiggle

* Fix temporary file conflicts

* Remove changes to Astro’s core types, type utility method instead

* Use `astro:db` instead of `@astrojs/db`

* Revert unnecessarily cautious temporary path name

This reverts commit ef2156e41b.

* Add changeset

* Fix entrypoints and `asDrizzleTable` usage in changeset

* Getting Nate in on the co-author action

Co-authored-by: Nate Moore <natemoo-re@users.noreply.github.com>

* Fix user seed file in integrations fixture

* Update `seedLocal()` after merge

* Provide empty `seedFiles` array in `db execute`

---------

Co-authored-by: Nate Moore <natemoo-re@users.noreply.github.com>
This commit is contained in:
Chris Swithinbank 2024-03-07 20:19:17 +01:00 committed by GitHub
parent e086a9f8c8
commit 2e4958c8a7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 401 additions and 38 deletions

View file

@ -0,0 +1,58 @@
---
"@astrojs/db": minor
---
Adds support for integrations providing `astro:db` configuration and seed files, using the new `astro:db:setup` hook.
To get TypeScript support for the `astro:db:setup` hook, wrap your integration object in the `defineDbIntegration()` utility:
```js
import { defineDbIntegration } from '@astrojs/db/utils';
export default function MyDbIntegration() {
return defineDbIntegration({
name: 'my-astro-db-powered-integration',
hooks: {
'astro:db:setup': ({ extendDb }) => {
extendDb({
configEntrypoint: '@astronaut/my-package/config',
seedEntrypoint: '@astronaut/my-package/seed',
});
},
},
});
}
```
Use the `extendDb` method to register additional `astro:db` config and seed files.
Integration config and seed files follow the same format as their user-defined equivalents. However, often while working on integrations, you may not be able to benefit from Astros generated table types exported from `astro:db`. For full type safety and autocompletion support, use the `asDrizzleTable()` utility to wrap your table definitions in the seed file.
```js
// config.ts
import { defineTable, column } from 'astro:db';
export const Pets = defineTable({
columns: {
name: column.text(),
age: column.number(),
},
});
```
```js
// seed.ts
import { asDrizzleTable } from '@astrojs/db/utils';
import { db } from 'astro:db';
import { Pets } from './config';
export default async function() {
// Convert the Pets table into a format ready for querying.
const typeSafePets = asDrizzleTable('Pets', Pets);
await db.insert(typeSafePets).values([
{ name: 'Palomita', age: 7 },
{ name: 'Pan', age: 3.5 },
]);
}
```

View file

@ -43,6 +43,7 @@ export async function cmd({
tables: dbConfig.tables ?? {},
root: astroConfig.root,
shouldSeed: false,
seedFiles: [],
});
}
const { code } = await bundleFile({ virtualModContents, root: astroConfig.root, fileUrl });

View file

@ -1,7 +1,6 @@
import type { AstroConfig } from 'astro';
import type { Arguments } from 'yargs-parser';
import { loadDbConfigFile } from '../load-file.js';
import { dbConfigSchema } from '../types.js';
import { resolveDbConfig } from '../load-file.js';
export async function cli({
flags,
@ -14,9 +13,7 @@ export async function cli({
// Most commands are `astro db foo`, but for now login/logout
// are also handled by this package, so first check if this is a db command.
const command = args[2] === 'db' ? args[3] : args[2];
const { mod } = await loadDbConfigFile(astroConfig.root);
// TODO: parseConfigOrExit()
const dbConfig = dbConfigSchema.parse(mod?.default ?? {});
const { dbConfig } = await resolveDbConfig(astroConfig);
switch (command) {
case 'shell': {

View file

@ -71,3 +71,13 @@ export const FOREIGN_KEY_REFERENCES_EMPTY_ERROR = (tableName: string) => {
tableName
)} is misconfigured. \`references\` array cannot be empty.`;
};
export const INTEGRATION_TABLE_CONFLICT_ERROR = (
integrationName: string,
tableName: string,
isUserConflict: boolean
) => {
return red('▶ Conflicting table name in integration ' + bold(integrationName)) + isUserConflict
? `\n A user-defined table named ${bold(tableName)} already exists`
: `\n Another integration already added a table named ${bold(tableName)}`;
};

View file

@ -6,14 +6,12 @@ import { mkdir, rm, writeFile } from 'fs/promises';
import { blue, yellow } from 'kleur/colors';
import parseArgs from 'yargs-parser';
import { CONFIG_FILE_NAMES, DB_PATH } from '../consts.js';
import { loadDbConfigFile } from '../load-file.js';
import { resolveDbConfig } from '../load-file.js';
import { type ManagedAppToken, getManagedAppTokenOrExit } from '../tokens.js';
import { type DBConfig, dbConfigSchema } from '../types.js';
import { type VitePlugin, getDbDirectoryUrl } from '../utils.js';
import { errorMap } from './error-map.js';
import { fileURLIntegration } from './file-url.js';
import { typegen } from './typegen.js';
import { type LateTables, vitePluginDb } from './vite-plugin-db.js';
import { type LateTables, vitePluginDb, type LateSeedFiles } from './vite-plugin-db.js';
import { vitePluginInjectEnvTs } from './vite-plugin-inject-env-ts.js';
function astroDBIntegration(): AstroIntegration {
@ -21,7 +19,6 @@ function astroDBIntegration(): AstroIntegration {
let configFileDependencies: string[] = [];
let root: URL;
let appToken: ManagedAppToken | undefined;
let dbConfig: DBConfig;
// Make table loading "late" to pass to plugins from `config:setup`,
// but load during `config:done` to wait for integrations to settle.
@ -30,6 +27,11 @@ function astroDBIntegration(): AstroIntegration {
throw new Error('[astro:db] INTERNAL Tables not loaded yet');
},
};
let seedFiles: LateSeedFiles = {
get() {
throw new Error('[astro:db] INTERNAL Seed files not loaded yet');
},
};
let command: 'dev' | 'build' | 'preview';
return {
name: 'astro:db',
@ -57,6 +59,7 @@ function astroDBIntegration(): AstroIntegration {
dbPlugin = vitePluginDb({
connectToStudio: false,
tables,
seedFiles,
root: config.root,
srcDir: config.srcDir,
});
@ -74,13 +77,10 @@ function astroDBIntegration(): AstroIntegration {
// TODO: refine where we load tables
// @matthewp: may want to load tables by path at runtime
const { mod, dependencies } = await loadDbConfigFile(config.root);
const { dbConfig, dependencies, integrationSeedPaths } = await resolveDbConfig(config);
tables.get = () => dbConfig.tables;
seedFiles.get = () => integrationSeedPaths;
configFileDependencies = dependencies;
dbConfig = dbConfigSchema.parse(mod?.default ?? {}, {
errorMap,
});
// TODO: resolve integrations here?
tables.get = () => dbConfig.tables ?? {};
if (!connectToStudio) {
const dbUrl = new URL(DB_PATH, config.root);

View file

@ -1,3 +1,4 @@
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { type SQL, sql } from 'drizzle-orm';
import { SQLiteAsyncDialect } from 'drizzle-orm/sqlite-core';
@ -23,11 +24,15 @@ const resolved = {
export type LateTables = {
get: () => DBTables;
};
export type LateSeedFiles = {
get: () => Array<string | URL>;
};
type VitePluginDBParams =
| {
connectToStudio: false;
tables: LateTables;
seedFiles: LateSeedFiles;
srcDir: URL;
root: URL;
}
@ -81,6 +86,7 @@ export function vitePluginDb(params: VitePluginDBParams): VitePlugin {
return getLocalVirtualModContents({
root: params.root,
tables: params.tables.get(),
seedFiles: params.seedFiles.get(),
shouldSeed: id === resolved.seedVirtual,
});
},
@ -94,17 +100,26 @@ export function getConfigVirtualModContents() {
export function getLocalVirtualModContents({
tables,
root,
seedFiles,
shouldSeed,
}: {
tables: DBTables;
seedFiles: Array<string | URL>;
root: URL;
shouldSeed: boolean;
}) {
const seedFilePaths = SEED_DEV_FILE_NAME.map(
const userSeedFilePaths = SEED_DEV_FILE_NAME.map(
// Format as /db/[name].ts
// for Vite import.meta.glob
(name) => new URL(name, getDbDirectoryUrl('file:///')).pathname
);
const resolveId = (id: string) => (id.startsWith('.') ? resolve(fileURLToPath(root), id) : id);
const integrationSeedFilePaths = seedFiles.map((pathOrUrl) =>
typeof pathOrUrl === 'string' ? resolveId(pathOrUrl) : pathOrUrl.pathname
);
const integrationSeedImports = integrationSeedFilePaths.map(
(filePath) => `() => import(${JSON.stringify(filePath)})`
);
const dbUrl = new URL(DB_PATH, root);
return `
@ -117,7 +132,8 @@ export const db = createLocalDatabaseClient({ dbUrl });
${
shouldSeed
? `await seedLocal({
fileGlob: import.meta.glob(${JSON.stringify(seedFilePaths)}, { eager: true }),
userSeedGlob: import.meta.glob(${JSON.stringify(userSeedFilePaths)}, { eager: true }),
integrationSeedImports: [${integrationSeedImports.join(',')}],
});`
: ''
}

View file

@ -1,12 +1,74 @@
import type { AstroConfig, AstroIntegration } from 'astro';
import { build as esbuild } from 'esbuild';
import { existsSync } from 'node:fs';
import { unlink, writeFile } from 'node:fs/promises';
import { fileURLToPath } from 'node:url';
import { build as esbuild } from 'esbuild';
import { createRequire } from 'node:module';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { CONFIG_FILE_NAMES, VIRTUAL_MODULE_ID } from './consts.js';
import { INTEGRATION_TABLE_CONFLICT_ERROR } from './errors.js';
import { errorMap } from './integration/error-map.js';
import { getConfigVirtualModContents } from './integration/vite-plugin-db.js';
import { dbConfigSchema, type AstroDbIntegration } from './types.js';
import { getDbDirectoryUrl } from './utils.js';
export async function loadDbConfigFile(
const isDbIntegration = (integration: AstroIntegration): integration is AstroDbIntegration =>
'astro:db:setup' in integration.hooks;
/**
* Load a users `astro:db` configuration file and additional configuration files provided by integrations.
*/
export async function resolveDbConfig({ root, integrations }: AstroConfig) {
const { mod, dependencies } = await loadUserConfigFile(root);
const userDbConfig = dbConfigSchema.parse(mod?.default ?? {}, { errorMap });
/** Resolved `astro:db` config including tables provided by integrations. */
const dbConfig = { tables: userDbConfig.tables ?? {} };
// Collect additional config and seed files from integrations.
const integrationDbConfigPaths: Array<{ name: string; configEntrypoint: string | URL }> = [];
const integrationSeedPaths: Array<string | URL> = [];
for (const integration of integrations) {
if (!isDbIntegration(integration)) continue;
const { name, hooks } = integration;
if (hooks['astro:db:setup']) {
hooks['astro:db:setup']({
extendDb({ configEntrypoint, seedEntrypoint }) {
if (configEntrypoint) {
integrationDbConfigPaths.push({ name, configEntrypoint });
}
if (seedEntrypoint) {
integrationSeedPaths.push(seedEntrypoint);
}
},
});
}
}
for (const { name, configEntrypoint } of integrationDbConfigPaths) {
// TODO: config file dependencies are not tracked for integrations for now.
const loadedConfig = await loadIntegrationConfigFile(root, configEntrypoint);
const integrationDbConfig = dbConfigSchema.parse(loadedConfig.mod?.default ?? {}, {
errorMap,
});
for (const key in integrationDbConfig.tables) {
if (key in dbConfig.tables) {
const isUserConflict = key in (userDbConfig.tables ?? {});
throw new Error(INTEGRATION_TABLE_CONFLICT_ERROR(name, key, isUserConflict));
} else {
dbConfig.tables[key] = integrationDbConfig.tables[key];
}
}
}
return {
/** Resolved `astro:db` config, including tables added by integrations. */
dbConfig,
/** Dependencies imported into the user config file. */
dependencies,
/** Additional `astro:db` seed file paths provided by integrations. */
integrationSeedPaths,
};
}
async function loadUserConfigFile(
root: URL
): Promise<{ mod: { default?: unknown } | undefined; dependencies: string[] }> {
let configFileUrl: URL | undefined;
@ -16,13 +78,35 @@ export async function loadDbConfigFile(
configFileUrl = fileUrl;
}
}
if (!configFileUrl) {
return await loadAndBundleDbConfigFile({ root, fileUrl: configFileUrl });
}
async function loadIntegrationConfigFile(root: URL, filePathOrUrl: string | URL) {
let fileUrl: URL;
if (typeof filePathOrUrl === 'string') {
const { resolve } = createRequire(root);
const resolvedFilePath = resolve(filePathOrUrl);
fileUrl = pathToFileURL(resolvedFilePath);
} else {
fileUrl = filePathOrUrl;
}
return await loadAndBundleDbConfigFile({ root, fileUrl });
}
async function loadAndBundleDbConfigFile({
root,
fileUrl,
}: {
root: URL;
fileUrl: URL | undefined;
}): Promise<{ mod: { default?: unknown } | undefined; dependencies: string[] }> {
if (!fileUrl) {
return { mod: undefined, dependencies: [] };
}
const { code, dependencies } = await bundleFile({
virtualModContents: getConfigVirtualModContents(),
root,
fileUrl: configFileUrl,
fileUrl,
});
return {
mod: await importBundledFile({ code, root }),

View file

@ -3,6 +3,7 @@ import { SQLiteAsyncDialect } from 'drizzle-orm/sqlite-core';
import { type ZodTypeDef, z } from 'zod';
import { SERIALIZED_SQL_KEY, type SerializedSQL } from '../runtime/types.js';
import { errorMap } from './integration/error-map.js';
import type { AstroIntegration } from 'astro';
export type MaybePromise<T> = T | Promise<T>;
export type MaybeArray<T> = T | T[];
@ -271,3 +272,14 @@ export type ResolvedCollectionConfig<TColumns extends ColumnsConfig = ColumnsCon
// since Omit collapses our union type on primary key.
export type NumberColumnOpts = z.input<typeof numberColumnOptsSchema>;
export type TextColumnOpts = z.input<typeof textColumnOptsSchema>;
export type AstroDbIntegration = AstroIntegration & {
hooks: {
'astro:db:setup'?: (options: {
extendDb: (options: {
configEntrypoint?: URL | string;
seedEntrypoint?: URL | string;
}) => void;
}) => void | Promise<void>;
};
};

View file

@ -1,5 +1,6 @@
import type { AstroConfig } from 'astro';
import type { AstroConfig, AstroIntegration } from 'astro';
import { loadEnv } from 'vite';
import type { AstroDbIntegration } from './types.js';
export type VitePlugin = Required<AstroConfig['vite']>['plugins'][number];
@ -21,3 +22,7 @@ export function getAstroStudioUrl(): string {
export function getDbDirectoryUrl(root: URL | string) {
return new URL('db/', root);
}
export function defineDbIntegration(integration: AstroDbIntegration): AstroIntegration {
return integration;
}

View file

@ -21,24 +21,36 @@ export { createRemoteDatabaseClient, createLocalDatabaseClient } from './db-clie
export async function seedLocal({
// Glob all potential seed files to catch renames and deletions.
fileGlob,
userSeedGlob,
integrationSeedImports,
}: {
fileGlob: Record<string, { default?: () => Promise<void> }>;
userSeedGlob: Record<string, { default?: () => Promise<void> }>;
integrationSeedImports: Array<() => Promise<{ default: () => Promise<void> }>>;
}) {
const seedFilePath = Object.keys(fileGlob)[0];
if (!seedFilePath) return;
const mod = fileGlob[seedFilePath];
const seedFilePath = Object.keys(userSeedGlob)[0];
if (seedFilePath) {
const mod = userSeedGlob[seedFilePath];
if (!mod.default) {
throw new Error(SEED_DEFAULT_EXPORT_ERROR(seedFilePath));
}
try {
await mod.default();
} catch (e) {
if (e instanceof LibsqlError) {
throw new Error(SEED_ERROR(e.message));
if (!mod.default) {
throw new Error(SEED_DEFAULT_EXPORT_ERROR(seedFilePath));
}
throw e;
try {
await mod.default();
} catch (e) {
if (e instanceof LibsqlError) {
throw new Error(SEED_ERROR(e.message));
}
throw e;
}
}
for (const importModule of integrationSeedImports) {
const mod = await importModule();
await mod.default().catch((e) => {
if (e instanceof LibsqlError) {
throw new Error(SEED_ERROR(e.message));
}
throw e;
});
}
}

View file

@ -1 +1,2 @@
export { defineDbIntegration } from './core/utils.js';
export { asDrizzleTable } from './runtime/index.js';

View file

@ -0,0 +1,8 @@
import db from '@astrojs/db';
import { defineConfig } from 'astro/config';
import testIntegration from './integration';
// https://astro.build/config
export default defineConfig({
integrations: [db(), testIntegration()],
});

View file

@ -0,0 +1,12 @@
import { column, defineDB, defineTable } from 'astro:db';
const Author = defineTable({
columns: {
name: column.text(),
age2: column.number({ optional: true }),
},
});
export default defineDB({
tables: { Author },
});

View file

@ -0,0 +1,13 @@
import { Author, db } from 'astro:db';
export default async () => {
await db
.insert(Author)
.values([
{ name: 'Ben' },
{ name: 'Nate' },
{ name: 'Erika' },
{ name: 'Bjorn' },
{ name: 'Sarah' },
]);
};

View file

@ -0,0 +1,8 @@
import { defineDB } from 'astro:db';
import { menu } from './shared';
export default defineDB({
tables: {
menu,
},
});

View file

@ -0,0 +1,15 @@
import { defineDbIntegration } from '@astrojs/db/utils';
export default function testIntegration() {
return defineDbIntegration({
name: 'db-test-integration',
hooks: {
'astro:db:setup'({ extendDb }) {
extendDb({
configEntrypoint: './integration/config.ts',
seedEntrypoint: './integration/seed.ts',
});
},
},
});
}

View file

@ -0,0 +1,14 @@
import { asDrizzleTable } from '@astrojs/db/utils';
import { db } from 'astro:db';
import { menu } from './shared';
export default async function () {
const table = asDrizzleTable('menu', menu);
await db.insert(table).values([
{ name: 'Pancakes', price: 9.5, type: 'Breakfast' },
{ name: 'French Toast', price: 11.25, type: 'Breakfast' },
{ name: 'Coffee', price: 3, type: 'Beverages' },
{ name: 'Cappuccino', price: 4.5, type: 'Beverages' },
]);
}

View file

@ -0,0 +1,9 @@
import { defineTable, column } from 'astro:db';
export const menu = defineTable({
columns: {
name: column.text(),
type: column.text(),
price: column.number(),
},
});

View file

@ -0,0 +1,14 @@
{
"name": "@test/db-integration",
"version": "0.0.0",
"private": true,
"scripts": {
"dev": "astro dev",
"build": "astro build",
"preview": "astro preview"
},
"dependencies": {
"@astrojs/db": "workspace:*",
"astro": "workspace:*"
}
}

View file

@ -0,0 +1,17 @@
---
/// <reference path="../../.astro/db-types.d.ts" />
import { Author, db, menu } from 'astro:db';
const authors = await db.select().from(Author);
const menuItems = await db.select().from(menu);
---
<h2>Authors</h2>
<ul class="authors-list">
{authors.map((author) => <li>{author.name}</li>)}
</ul>
<h2>Menu</h2>
<ul class="menu">
{menuItems.map((item) => <li>{item.name}</li>)}
</ul>

View file

@ -0,0 +1,48 @@
import { expect } from 'chai';
import { load as cheerioLoad } from 'cheerio';
import { loadFixture } from '../../astro/test/test-utils.js';
describe('astro:db with integrations', () => {
let fixture;
before(async () => {
fixture = await loadFixture({
root: new URL('./fixtures/integrations/', import.meta.url),
});
});
// Note(bholmesdev): Use in-memory db to avoid
// Multiple dev servers trying to unlink and remount
// the same database file.
process.env.TEST_IN_MEMORY_DB = 'true';
describe('development', () => {
let devServer;
before(async () => {
console.log('starting dev server');
devServer = await fixture.startDevServer();
});
after(async () => {
await devServer.stop();
process.env.TEST_IN_MEMORY_DB = undefined;
});
it('Prints the list of authors from user-defined table', async () => {
const html = await fixture.fetch('/').then((res) => res.text());
const $ = cheerioLoad(html);
const ul = $('.authors-list');
expect(ul.children()).to.have.a.lengthOf(5);
expect(ul.children().eq(0).text()).to.equal('Ben');
});
it('Prints the list of menu items from integration-defined table', async () => {
const html = await fixture.fetch('/').then((res) => res.text());
const $ = cheerioLoad(html);
const ul = $('ul.menu');
expect(ul.children()).to.have.a.lengthOf(4);
expect(ul.children().eq(0).text()).to.equal('Pancakes');
});
});
});

9
pnpm-lock.yaml generated
View file

@ -3900,6 +3900,15 @@ importers:
specifier: workspace:*
version: link:../../../../astro
packages/db/test/fixtures/integrations:
dependencies:
'@astrojs/db':
specifier: workspace:*
version: link:../../..
astro:
specifier: workspace:*
version: link:../../../../astro
packages/db/test/fixtures/recipes:
dependencies:
'@astrojs/db':