Fix problems with local libSQL DB (#12089)

Co-authored-by:  Matthew Phillips <361671+matthewp@users.noreply.github.com>
Co-authored-by: Emanuele Stoppa <602478+ematipico@users.noreply.github.com>

.changeset/mighty-lions-give.md

@@ -0,0 +1,5 @@
+---
+'@astrojs/db': patch
+---
+
+Fixes initial schema push for local file and in-memory libSQL DB

.changeset/shy-knives-pretend.md

@@ -0,0 +1,5 @@
+---
+'@astrojs/db': patch
+---
+
+Fixes relative local libSQL db URL

packages/db/src/core/cli/migration-queries.ts

@@ -36,6 +36,7 @@ import type {
 	TextColumn,
 } from '../types.js';
 import type { RemoteDatabaseInfo, Result } from '../utils.js';
+import { LibsqlError } from '@libsql/client';
 
 const sqlite = new SQLiteAsyncDialect();
 const genTempTableName = customAlphabet('abcdefghijklmnopqrstuvwxyz', 10);
@@ -450,10 +451,19 @@ async function getDbCurrentSnapshot(
 		);
 
 		return JSON.parse(res.snapshot);
-	} catch (error: any) {
+	} catch (error) {
-		if (error.code === 'SQLITE_UNKNOWN') {
+		// Don't handle errors that are not from libSQL
+		if (error instanceof LibsqlError &&
 			// If the schema was never pushed to the database yet the table won't exist.
 			// Treat a missing snapshot table as an empty table.
+			(
+				// When connecting to a remote database in that condition
+				// the query will fail with the following error code and message.
+				(error.code === 'SQLITE_UNKNOWN' && error.message === 'SQLITE_UNKNOWN: SQLite error: no such table: _astro_db_snapshot') ||
+				// When connecting to a local or in-memory database that does not have a snapshot table yet
+				// the query will fail with the following error code and message.
+				(error.code === 'SQLITE_ERROR' && error.message === 'SQLITE_ERROR: no such table: _astro_db_snapshot'))
+		) {
 			return;
 		}
 

packages/db/src/core/integration/vite-plugin-db.ts

@@ -168,10 +168,14 @@ export function getStudioVirtualModContents({
 	function dbUrlArg() {
 		const dbStr = JSON.stringify(dbInfo.url);
 
-		// Allow overriding, mostly for testing
+		if (isBuild) {
-		return dbInfo.type === 'studio'
+			// Allow overriding, mostly for testing
-			? `import.meta.env.ASTRO_STUDIO_REMOTE_DB_URL ?? ${dbStr}`
+			return dbInfo.type === 'studio'
-			: `import.meta.env.ASTRO_DB_REMOTE_URL ?? ${dbStr}`;
+				? `import.meta.env.ASTRO_STUDIO_REMOTE_DB_URL ?? ${dbStr}`
+				: `import.meta.env.ASTRO_DB_REMOTE_URL ?? ${dbStr}`;
+		} else {
+			return dbStr;
+		}
 	}
 
 	return `

packages/db/src/runtime/db-client.ts

@@ -53,17 +53,38 @@ export function createRemoteDatabaseClient(options: RemoteDbClientOptions) {
 
 	return options.dbType === 'studio'
 		? createStudioDatabaseClient(options.appToken, remoteUrl)
-		: createRemoteLibSQLClient(options.appToken, remoteUrl);
+		: createRemoteLibSQLClient(options.appToken, remoteUrl, options.remoteUrl.toString());
 }
 
-function createRemoteLibSQLClient(appToken: string, remoteDbURL: URL) {
+function createRemoteLibSQLClient(appToken: string, remoteDbURL: URL, rawUrl: string) {
 	const options: Partial<LibSQLConfig> = Object.fromEntries(remoteDbURL.searchParams.entries());
 	remoteDbURL.search = '';
 
+	let url = remoteDbURL.toString();
+	if (remoteDbURL.protocol === 'memory:') {
+		// libSQL expects a special string in place of a URL
+		// for in-memory DBs.
+		url = ':memory:';
+	} else if (
+		remoteDbURL.protocol === 'file:' &&
+		remoteDbURL.pathname.startsWith('/') &&
+		!rawUrl.startsWith('file:/')
+	) {
+		// libSQL accepts relative and absolute file URLs
+		// for local DBs. This doesn't match the URL specification.
+		// Parsing `file:some.db` and `file:/some.db` should yield
+		// the same result, but libSQL interprets the former as
+		// a relative path, and the latter as an absolute path.
+		// This detects when such a conversion happened during parsing
+		// and undoes it so that the URL given to libSQL is the
+		// same as given by the user.
+		url = 'file:' + remoteDbURL.pathname.substring(1);
+	}
+
 	const client = createClient({
 		...options,
 		authToken: appToken,
-		url: remoteDbURL.protocol === 'memory:' ? ':memory:' : remoteDbURL.toString(),
+		url,
 	});
 	return drizzleLibsql(client);
 }

packages/db/test/basics.test.js

@@ -15,7 +15,7 @@ describe('astro:db', () => {
 		});
 	});
 
-	describe('development', () => {
+	describe({ skip: process.platform === 'darwin' }, 'development', () => {
 		let devServer;
 
 		before(async () => {
@@ -94,7 +94,7 @@ describe('astro:db', () => {
 		});
 	});
 
-	describe('development --remote', () => {
+	describe({ skip: process.platform === 'darwin' }, 'development --remote', () => {
 		let devServer;
 		let remoteDbServer;
 

packages/db/test/fixtures/libsql-remote/astro.config.ts

@@ -0,0 +1,10 @@
+import db from '@astrojs/db';
+import { defineConfig } from 'astro/config';
+
+// https://astro.build/config
+export default defineConfig({
+	integrations: [db()],
+	devToolbar: {
+		enabled: false,
+	},
+});

packages/db/test/fixtures/libsql-remote/db/config.ts

@@ -0,0 +1,13 @@
+import { column, defineDb, defineTable } from 'astro:db';
+
+const User = defineTable({
+	columns: {
+		id: column.text({ primaryKey: true, optional: false }),
+		username: column.text({ optional: false, unique: true }),
+		password: column.text({ optional: false }),
+	},
+});
+
+export default defineDb({
+	tables: { User },
+});

packages/db/test/fixtures/libsql-remote/db/seed.ts

@@ -0,0 +1,7 @@
+import { User, db } from 'astro:db';
+
+export default async function () {
+	await db.batch([
+		db.insert(User).values([{ id: 'mario', username: 'Mario', password: 'itsame' }]),
+	]);
+}

packages/db/test/fixtures/libsql-remote/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "@test/db-libsql-remote",
+  "version": "0.0.0",
+  "private": true,
+  "scripts": {
+    "dev": "astro dev",
+    "build": "astro build",
+    "preview": "astro preview"
+  },
+  "dependencies": {
+    "@astrojs/db": "workspace:*",
+    "astro": "workspace:*"
+  }
+}

packages/db/test/fixtures/libsql-remote/src/pages/index.astro

@@ -0,0 +1,11 @@
+---
+/// <reference path="../../.astro/db-types.d.ts" />
+import { User, db } from 'astro:db';
+
+const users = await db.select().from(User);
+---
+
+<h2>Users</h2>
+<ul class="users-list">
+	{users.map((user) => <li>{user.name}</li>)}
+</ul>

packages/db/test/libsql-remote.test.js

@@ -0,0 +1,77 @@
+import assert from 'node:assert/strict';
+import { relative } from 'node:path';
+import { rm } from 'node:fs/promises';
+import { after, before, describe, it } from 'node:test';
+import { fileURLToPath } from 'node:url';
+import testAdapter from '../../astro/test/test-adapter.js';
+import { loadFixture } from '../../astro/test/test-utils.js';
+import { clearEnvironment, initializeRemoteDb } from './test-utils.js';
+
+describe('astro:db local database', () => {
+	let fixture;
+	before(async () => {
+		fixture = await loadFixture({
+			root: new URL('./fixtures/libsql-remote/', import.meta.url),
+			output: 'server',
+			adapter: testAdapter(),
+		});
+	});
+
+	describe('build --remote with local libSQL file (absolute path)', () => {
+		before(async () => {
+			clearEnvironment();
+
+			const absoluteFileUrl = new URL('./fixtures/libsql-remote/dist/absolute.db', import.meta.url);
+			// Remove the file if it exists to avoid conflict between test runs
+			await rm(absoluteFileUrl, { force: true });
+
+			process.env.ASTRO_INTERNAL_TEST_REMOTE = true;
+			process.env.ASTRO_DB_REMOTE_URL = absoluteFileUrl.toString();
+			await fixture.build();
+			await initializeRemoteDb(fixture.config);
+		});
+
+		after(async () => {
+			delete process.env.ASTRO_INTERNAL_TEST_REMOTE;
+			delete process.env.ASTRO_DB_REMOTE_URL;
+		});
+
+		it('Can render page', async () => {
+			const app = await fixture.loadTestAdapterApp();
+			const request = new Request('http://example.com/');
+			const response = await app.render(request);
+			assert.equal(response.status, 200);
+		});
+	});
+
+	describe('build --remote with local libSQL file (relative path)', () => {
+		before(async () => {
+			clearEnvironment();
+
+			const absoluteFileUrl = new URL('./fixtures/libsql-remote/dist/relative.db', import.meta.url);
+			const prodDbPath = relative(
+				fileURLToPath(fixture.config.root),
+				fileURLToPath(absoluteFileUrl),
+			);
+			// Remove the file if it exists to avoid conflict between test runs
+			await rm(prodDbPath, { force: true });
+
+			process.env.ASTRO_INTERNAL_TEST_REMOTE = true;
+			process.env.ASTRO_DB_REMOTE_URL = `file:${prodDbPath}`;
+			await fixture.build();
+			await initializeRemoteDb(fixture.config);
+		});
+
+		after(async () => {
+			delete process.env.ASTRO_INTERNAL_TEST_REMOTE;
+			delete process.env.ASTRO_DB_REMOTE_URL;
+		});
+
+		it('Can render page', async () => {
+			const app = await fixture.loadTestAdapterApp();
+			const request = new Request('http://example.com/');
+			const response = await app.render(request);
+			assert.equal(response.status, 200);
+		});
+	});
+});

packages/db/test/test-utils.js

@@ -64,6 +64,23 @@ export async function setupRemoteDbServer(astroConfig) {
 	};
 }
 
+export async function initializeRemoteDb(astroConfig) {
+	await cli({
+		config: astroConfig,
+		flags: {
+			_: [undefined, 'astro', 'db', 'push'],
+			remote: true,
+		},
+	});
+	await cli({
+		config: astroConfig,
+		flags: {
+			_: [undefined, 'astro', 'db', 'execute', 'db/seed.ts'],
+			remote: true,
+		},
+	});
+}
+
 /**
  * Clears the environment variables related to Astro DB and Astro Studio.
  */

pnpm-lock.yaml

@@ -4408,6 +4408,15 @@ importers:
         specifier: workspace:*
         version: link:../../../../astro
 
+  packages/db/test/fixtures/libsql-remote:
+    dependencies:
+      '@astrojs/db':
+        specifier: workspace:*
+        version: link:../../..
+      astro:
+        specifier: workspace:*
+        version: link:../../../../astro
+
   packages/db/test/fixtures/local-prod:
     dependencies:
       '@astrojs/db':