From f85e8cd8979b4e3efa3b97458915b53932d138df Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" Date: Thu, 18 Jan 2024 22:15:39 -0800 Subject: [PATCH] migrate seed data to new db push command --- packages/db/src/cli/commands/push/index.ts | 84 +++++++++++++++++----- packages/db/src/migrations.ts | 9 +-- packages/db/src/utils.ts | 29 -------- 3 files changed, 73 insertions(+), 49 deletions(-) diff --git a/packages/db/src/cli/commands/push/index.ts b/packages/db/src/cli/commands/push/index.ts index 5bdf564010..4d89daa8ce 100644 --- a/packages/db/src/cli/commands/push/index.ts +++ b/packages/db/src/cli/commands/push/index.ts @@ -1,24 +1,25 @@ +import type { InArgs, InStatement } from '@libsql/client'; import type { AstroConfig } from 'astro'; import deepDiff from 'deep-diff'; -import { eq, sql } from 'drizzle-orm'; -import { readFile } from 'fs/promises'; +import { eq, sql, type Query } from 'drizzle-orm'; import type { Arguments } from 'yargs-parser'; import { appTokenError } from '../../../errors.js'; -import { - getMigrations, - initializeFromMigrations, -} from '../../../migrations.js'; +import { collectionToTable, createLocalDatabaseClient } from '../../../internal.js'; +import { getMigrations, initializeFromMigrations, loadMigration } from '../../../migrations.js'; +import type { DBCollections } from '../../../types.js'; import { STUDIO_ADMIN_TABLE_ROW_ID, adminTable, createRemoteDatabaseClient, - getAstroStudioEnv + getAstroStudioEnv, + getRemoteDatabaseUrl, } from '../../../utils.js'; const { diff } = deepDiff; +export async function cmd({ config, flags }: { config: AstroConfig; flags: Arguments }) { + const isSeedData = flags.seed; + const isDryRun = flags.dryRun; - -export async function cmd({ config }: { config: AstroConfig, flags: Arguments }) { const currentSnapshot = JSON.parse(JSON.stringify(config.db?.collections ?? {})); const allMigrationFiles = await getMigrations(); if (allMigrationFiles.length === 0) { @@ -50,23 +51,74 @@ export async function cmd({ config }: { config: AstroConfig, flags: Arguments }) const missingMigrations = allLocalMigrations.filter((migration) => { return !allRemoteMigrations.find((m: any) => m.name === migration); }); + + console.log(`Pushing ${missingMigrations.length} migrations...`); + // load all missing migrations - const missingMigrationContents = await Promise.all( - missingMigrations.map(async (migration) => { - return JSON.parse(await readFile(`./migrations/${migration}`, 'utf-8')); - }) - ); + const missingMigrationContents = await Promise.all(missingMigrations.map(loadMigration)); // combine all missing migrations into a single batch const missingMigrationBatch = missingMigrationContents.reduce((acc, curr) => { - return [...acc, ...curr.diff]; - }); + return [...acc, ...curr.db]; + }, [] as string[]); // apply the batch to the DB // TODO: How to do this with Drizzle ORM & proxy implementation? Unclear. // @ts-expect-error await db.batch(missingMigrationBatch); + + // TODO: Update the migrations table to set all to "applied" + // update the config schema in the admin table db.update(adminTable) .set({ collections: JSON.stringify(currentSnapshot) }) .where(eq(adminTable.id, STUDIO_ADMIN_TABLE_ROW_ID)); + + if (isSeedData) { + console.info('Pushing data...'); + await tempDataPush({ currentSnapshot, appToken, isDryRun }); + } + console.info('Push complete!'); } +/** TODO: refine with migration changes */ +async function tempDataPush({ + currentSnapshot, + appToken, + isDryRun, +}: { + currentSnapshot: DBCollections; + appToken: string; + isDryRun?: boolean; +}) { + const db = await createLocalDatabaseClient({ + collections: currentSnapshot, + dbUrl: ':memory:', + seeding: true, + }); + const queries: Query[] = []; + + for (const [name, collection] of Object.entries(currentSnapshot)) { + if (collection.writable || !collection.data) continue; + const table = collectionToTable(name, collection); + const insert = db.insert(table).values(await collection.data()); + + queries.push(insert.toSQL()); + } + const url = new URL('/db/query', getRemoteDatabaseUrl()); + const requestBody: InStatement[] = queries.map((q) => ({ + sql: q.sql, + args: q.params as InArgs, + })); + + if (isDryRun) { + console.info('[DRY RUN] Batch data seed:', JSON.stringify(requestBody, null, 2)); + return new Response(null, { status: 200 }); + } + + return await fetch(url, { + method: 'POST', + headers: new Headers({ + Authorization: `Bearer ${appToken}`, + }), + body: JSON.stringify(requestBody), + }); +} diff --git a/packages/db/src/migrations.ts b/packages/db/src/migrations.ts index d66c0401ff..f6bc51ef5c 100644 --- a/packages/db/src/migrations.ts +++ b/packages/db/src/migrations.ts @@ -1,6 +1,7 @@ import deepDiff from 'deep-diff'; import { mkdir, readFile, readdir, writeFile } from 'fs/promises'; -const { diff, applyChange } = deepDiff; +import type { DBCollections } from './types.js'; +const { applyChange } = deepDiff; export async function getMigrations(): Promise { const migrationFiles = await readdir('./migrations').catch((err) => { @@ -12,11 +13,11 @@ export async function getMigrations(): Promise { return migrationFiles; } -export async function loadMigration(migration: string): Promise<{ diff: any[]; db: any[] }> { +export async function loadMigration(migration: string): Promise<{ diff: any[]; db: string[] }> { return JSON.parse(await readFile(`./migrations/${migration}`, 'utf-8')); } -export async function loadInitialSnapshot(): Promise { +export async function loadInitialSnapshot(): Promise { return JSON.parse(await readFile('./migrations/0000_snapshot.json', 'utf-8')); } @@ -25,7 +26,7 @@ export async function initializeMigrationsDirectory(currentSnapshot: unknown) { await writeFile('./migrations/0000_snapshot.json', JSON.stringify(currentSnapshot, undefined, 2)); } -export async function initializeFromMigrations(allMigrationFiles: string[]) { +export async function initializeFromMigrations(allMigrationFiles: string[]): Promise { const prevSnapshot = await loadInitialSnapshot(); for (const migration of allMigrationFiles) { if (migration === '0000_snapshot.json') continue; diff --git a/packages/db/src/utils.ts b/packages/db/src/utils.ts index e0830beac0..3a3ff04d3f 100644 --- a/packages/db/src/utils.ts +++ b/packages/db/src/utils.ts @@ -2,7 +2,6 @@ import type { InStatement } from "@libsql/client"; import type { AstroConfig } from 'astro'; import { sqliteTable, text } from 'drizzle-orm/sqlite-core'; import { drizzle } from 'drizzle-orm/sqlite-proxy'; -import { red, yellow } from 'kleur/colors'; import { loadEnv } from 'vite'; import { z } from 'zod'; @@ -22,34 +21,6 @@ export function getAstroStudioEnv(envMode = ''): Record<`ASTRO_STUDIO_${string}` return env; } -export async function isAppTokenValid({ - remoteDbUrl, - appToken, -}: { - remoteDbUrl: string; - appToken: string; -}): Promise { - const { status } = await fetch(new URL('/authorize', remoteDbUrl), { - headers: { - Authorization: `Bearer ${appToken}`, - }, - }); - - if (status === 200) { - return true; - } else if (status === 401) { - // eslint-disable-next-line no-console - console.warn(yellow(`⚠️ App token is invalid or revoked.`)); - return false; - } else { - // eslint-disable-next-line no-console - console.error( - `${red('⚠️ Unexpected error connecting to Astro Studio.')} Please try again later.`, - ); - process.exit(1); - } -} - export function getStudioUrl(): string { const env = getAstroStudioEnv(); return env.ASTRO_STUDIO_BASE_URL;