Bundling! 🤘 (#36)

* Bundling! 🤘

* Finalize build changes

examples/snowpack/astro.config.mjs

@@ -2,6 +2,7 @@ export default {
   projectRoot: '.',
   astroRoot: './astro',
   dist: './_site',
+  public: './public',
   extensions: {
     '.jsx': 'preact',
   },

examples/snowpack/astro/pages/guides.astro

@@ -73,6 +73,7 @@ let communityGuides;
 
     <div class="card-grid card-grid-4">
       {communityGuides.map((post) => <Card item={post} />)}
+
       <Card item={{
         url: 'https://www.snowpack.dev/posts/2021-01-13-snowpack-3-0',
         img: 'https://www.snowpack.dev/img/social-snowpackv3.jpg',

package-lock.json

@@ -3169,9 +3169,9 @@
       }
     },
     "rollup": {
-      "version": "2.42.3",
-      "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.42.3.tgz",
-      "integrity": "sha512-JjaT9WaUS5vmjy6xUrnPOskjkQg2cN4WSACNCwbOvBz8VDmbiKVdmTFUoMPRqTud0tsex8Xy9/boLbDW9HKD1w==",
+      "version": "2.43.1",
+      "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.43.1.tgz",
+      "integrity": "sha512-kvRE6VJbiv4d8m2nGeccc3qRpzOMghAhu2KeITjyZVCjneIFLPQ3zm2Wmqnl0LcUg3FvDaV0MfKnG4NCMbiSfw==",
       "requires": {
         "fsevents": "~2.3.1"
       }

package.json

@@ -59,6 +59,7 @@
     "postcss-modules": "^4.0.0",
     "react": "^17.0.1",
     "react-dom": "^17.0.1",
+    "rollup": "^2.43.1",
     "sass": "^1.32.8",
     "snowpack": "^3.1.2",
     "svelte": "^3.35.0",

src/@types/astro.ts

@@ -2,6 +2,7 @@ export interface AstroConfigRaw {
   dist: string;
   projectRoot: string;
   astroRoot: string;
+  public: string;
   jsx?: string;
 }
 
@@ -11,6 +12,7 @@ export interface AstroConfig {
   dist: string;
   projectRoot: URL;
   astroRoot: URL;
+  public: URL;
   extensions?: Record<string, ValidExtensionPlugins>;
 }
 

src/build.ts

@@ -1,34 +1,65 @@
 import type { AstroConfig } from './@types/astro';
-import { defaultLogOptions, LogOptions } from './logger';
+import type { LogOptions } from './logger';
+import type { LoadResult } from './runtime';
 
-import { loadConfiguration, startServer as startSnowpackServer, build as snowpackBuild } from 'snowpack';
-import { promises as fsPromises } from 'fs';
+import { existsSync, promises as fsPromises } from 'fs';
 import { relative as pathRelative } from 'path';
+import {fdir} from 'fdir';
 import { defaultLogDestination, error } from './logger.js';
 import { createRuntime } from './runtime.js';
+import { bundle, collectDynamicImports } from './build/bundle.js';
+import { collectStatics } from './build/static.js';
 
-const { mkdir, readdir, stat, writeFile } = fsPromises;
+const { mkdir, readdir, readFile, stat, writeFile } = fsPromises;
 
 const logging: LogOptions = {
   level: 'debug',
   dest: defaultLogDestination,
 };
 
-async function* allPages(root: URL): AsyncGenerator<URL, void, unknown> {
+async function* recurseFiles(root: URL): AsyncGenerator<URL, void, unknown> {
   for (const filename of await readdir(root)) {
     const fullpath = new URL(filename, root);
     const info = await stat(fullpath);
 
     if (info.isDirectory()) {
-      yield* allPages(new URL(fullpath + '/'));
+      yield* recurseFiles(new URL(fullpath + '/'));
     } else {
-      if (/\.(astro|md)$/.test(fullpath.pathname)) {
-        yield fullpath;
-      }
+      yield fullpath;
     }
   }
 }
 
+async function allPages(root: URL) {
+  const api = new fdir().filter(p => /\.(astro|md)$/.test(p))
+    .withFullPaths().crawl(root.pathname);
+  const files = await api.withPromise();
+  return files as string[];
+}
+
+function mergeSet(a: Set<string>, b: Set<string>) {
+  for(let str of b) {
+    a.add(str);
+  }
+  return a;
+}
+
+async function writeFilep(outPath: URL, bytes: string | Buffer, encoding: 'utf-8' | null) {
+  const outFolder = new URL('./', outPath);
+  await mkdir(outFolder, { recursive: true });
+  await writeFile(outPath, bytes, encoding || 'binary');
+}
+
+async function writeResult(result: LoadResult, outPath: URL, encoding: null | 'utf-8') {
+  if(result.statusCode !== 200) {
+    error(logging, 'build', result.error || result.statusCode);
+    //return 1;
+  } else {
+    const bytes = result.contents;
+    await writeFilep(outPath, bytes, encoding);
+  }
+}
+
 export async function build(astroConfig: AstroConfig): Promise<0 | 1> {
   const { projectRoot, astroRoot } = astroConfig;
   const pageRoot = new URL('./pages/', astroRoot);
@@ -39,39 +70,60 @@ export async function build(astroConfig: AstroConfig): Promise<0 | 1> {
     dest: defaultLogDestination,
   };
 
-  const runtime = await createRuntime(astroConfig, { logging: runtimeLogging, env: 'build' });
-  const { snowpackConfig } = runtime.runtimeConfig;
+  const runtime = await createRuntime(astroConfig, { logging: runtimeLogging });
+  const { runtimeConfig } = runtime;
+  const { snowpack } = runtimeConfig;
+  const resolve = (pkgName: string) => snowpack.getUrlForPackage(pkgName)
 
-  try {
-    const result = await snowpackBuild({
-      config: snowpackConfig,
-      lockfile: null,
-    });
-  } catch (err) {
-    error(logging, 'build', err);
-    return 1;
-  }
+  const imports = new Set<string>();
+  const statics = new Set<string>();
 
-  for await (const filepath of allPages(pageRoot)) {
+  for (const pathname of await allPages(pageRoot)) {
+    const filepath = new URL(`file://${pathname}`);
     const rel = pathRelative(astroRoot.pathname + '/pages', filepath.pathname); // pages/index.astro
     const pagePath = `/${rel.replace(/\.(astro|md)/, '')}`;
 
     try {
-      const outPath = new URL('./' + rel.replace(/\.(astro|md)/, '.html'), dist);
-      const outFolder = new URL('./', outPath);
+      let relPath = './' + rel.replace(/\.(astro|md)$/, '.html');
+      if(!relPath.endsWith('index.html')) {
+        relPath = relPath.replace(/\.html$/, '/index.html');
+      }
+
+      const outPath = new URL(relPath, dist);
       const result = await runtime.load(pagePath);
 
-      if (result.statusCode !== 200) {
-        error(logging, 'generate', result.error || result.statusCode);
-        //return 1;
-      } else {
-        await mkdir(outFolder, { recursive: true });
-        await writeFile(outPath, result.contents, 'utf-8');
+      await writeResult(result, outPath, 'utf-8');
+      if(result.statusCode === 200) {
+        mergeSet(statics, collectStatics(result.contents.toString('utf-8')));
       }
     } catch (err) {
       error(logging, 'generate', err);
       return 1;
     }
+
+    mergeSet(imports, await collectDynamicImports(filepath, astroConfig, resolve));
+  }
+
+  await bundle(imports, {dist, runtime, astroConfig});
+
+  for(let url of statics) {
+    const outPath = new URL('.' + url, dist);
+    const result = await runtime.load(url);
+
+    await writeResult(result, outPath, null);
+  }
+
+  if(existsSync(astroConfig.public)) {
+    const pub = astroConfig.public;
+    const publicFiles = (await new fdir().withFullPaths().crawl(pub.pathname).withPromise()) as string[];
+    for(const filepath of publicFiles) {
+      const fileUrl = new URL(`file://${filepath}`)
+      const rel = pathRelative(pub.pathname, fileUrl.pathname);
+      const outUrl = new URL('./' + rel, dist);
+
+      const bytes = await readFile(fileUrl);
+      await writeFilep(outUrl, bytes, null);
+    }
   }
 
   await runtime.shutdown();

src/build/bundle.ts

@@ -0,0 +1,244 @@
+import type { AstroConfig, ValidExtensionPlugins } from '../@types/astro';
+import type { ImportDeclaration, ExportNamedDeclaration, VariableDeclarator, Identifier, VariableDeclaration } from '@babel/types';
+import type { InputOptions, OutputOptions } from 'rollup';
+import type { AstroRuntime } from '../runtime';
+
+import esbuild from 'esbuild';
+import { promises as fsPromises } from 'fs';
+import { parse } from '../parser/index.js';
+import { walk } from 'estree-walker';
+import babelParser from '@babel/parser';
+import path from 'path';
+import {rollup} from 'rollup';
+
+const { transformSync } = esbuild;
+const { readFile } = fsPromises;
+
+type DynamicImportMap = Map<'vue' | 'react' | 'react-dom' | 'preact', string>;
+
+async function acquireDynamicComponentImports(plugins: Set<ValidExtensionPlugins>, resolve: (s: string) => Promise<string>): Promise<DynamicImportMap> {
+  const importMap: DynamicImportMap = new Map();
+  for (let plugin of plugins) {
+    switch (plugin) {
+      case 'vue': {
+        importMap.set('vue', await resolve('vue'));
+        break;
+      }
+      case 'react': {
+        importMap.set('react', await resolve('react'));
+        importMap.set('react-dom', await resolve('react-dom'));
+        break;
+      }
+      case 'preact': {
+        importMap.set('preact', await resolve('preact'));
+        break;
+      }
+    }
+  }
+  return importMap;
+}
+
+function compileExpressionSafe(raw: string): string {
+  let { code } = transformSync(raw, {
+    loader: 'tsx',
+    jsxFactory: 'h',
+    jsxFragment: 'Fragment',
+    charset: 'utf8',
+  });
+  return code;
+}
+
+const defaultExtensions: Readonly<Record<string, ValidExtensionPlugins>> = {
+  '.jsx': 'react',
+  '.svelte': 'svelte',
+  '.vue': 'vue'
+};
+
+export async function collectDynamicImports(filename: URL, astroConfig: AstroConfig, resolve: (s: string) => Promise<string>) {
+  const imports = new Set<string>();
+
+  // No markdown for now
+  if(filename.pathname.endsWith('md')) {
+    return imports;
+  }
+
+  const extensions = astroConfig.extensions || defaultExtensions;
+  const source = await readFile(filename, 'utf-8');
+  const ast = parse(source, {
+    filename
+  });
+
+  if(!ast.module) {
+    return imports;
+  }
+
+  const componentImports: ImportDeclaration[] = [];
+  const components: Record<string, { plugin: ValidExtensionPlugins; type: string; specifier: string; }> = {};
+  const plugins = new Set<ValidExtensionPlugins>();
+
+  const program = babelParser.parse(ast.module.content, {
+    sourceType: 'module',
+    plugins: ['jsx', 'typescript', 'topLevelAwait'],
+  }).program;
+
+  const { body } = program;
+  let i = body.length;
+  while (--i >= 0) {
+    const node = body[i];
+    if (node.type === 'ImportDeclaration') {
+      componentImports.push(node);
+    }
+  }
+
+  for (const componentImport of componentImports) {
+    const importUrl = componentImport.source.value;
+    const componentType = path.posix.extname(importUrl);
+    const componentName = path.posix.basename(importUrl, componentType);
+    const plugin = extensions[componentType] || defaultExtensions[componentType];
+    plugins.add(plugin);
+    components[componentName] = {
+      plugin,
+      type: componentType,
+      specifier: importUrl,
+    };
+  }
+
+  const dynamic = await acquireDynamicComponentImports(plugins, resolve);
+
+  function appendImports(rawName: string, filename: URL, astroConfig: AstroConfig) {
+    const [componentName, componentType] = rawName.split(':');
+    if(!componentType) {
+      return;
+    }
+
+    if (!components[componentName]) {
+      throw new Error(`Unknown Component: ${componentName}`);
+    }
+
+    const defn = components[componentName];
+    const fileUrl = new URL(defn.specifier, filename);
+    let rel = path.posix.relative(astroConfig.astroRoot.pathname, fileUrl.pathname);
+    
+    switch(defn.plugin) {
+      case 'preact': {
+        imports.add(dynamic.get('preact')!);
+        rel = rel.replace(/\.[^.]+$/, '.js');
+        break;
+      }
+      case 'react': {
+        imports.add(dynamic.get('react')!);
+        imports.add(dynamic.get('react-dom')!);
+        rel = rel.replace(/\.[^.]+$/, '.js');
+        break;
+      }
+      case 'vue': {
+        imports.add(dynamic.get('vue')!);
+        rel = rel.replace(/\.[^.]+$/, '.vue.js');
+        break;
+      }
+    }
+
+    imports.add(`/_astro/${rel}`);
+  }
+
+  walk(ast.html, {
+    enter(node) {
+      switch (node.type) {
+        case 'MustacheTag': {
+          let code: string;
+          try {
+            code = compileExpressionSafe(node.content);
+          } catch {
+            return;
+          }
+
+          let matches: RegExpExecArray[] = [];
+          let match: RegExpExecArray | null | undefined;
+          const H_COMPONENT_SCANNER = /h\(['"]?([A-Z].*?)['"]?,/gs;
+          const regex = new RegExp(H_COMPONENT_SCANNER);
+          while ((match = regex.exec(code))) {
+            matches.push(match);
+          }
+          for (const match of matches.reverse()) {
+            const name = match[1];
+            appendImports(name, filename, astroConfig);
+          }
+          break;
+        }
+        case 'InlineComponent': {
+          if(/^[A-Z]/.test(node.name)) {
+            appendImports(node.name, filename, astroConfig);
+            return;
+          }
+
+          break;
+        }
+      }
+    }
+  });
+
+  return imports;
+}
+
+interface BundleOptions {
+  runtime: AstroRuntime;
+  dist: URL;
+  astroConfig: AstroConfig;
+}
+
+export async function bundle(imports: Set<string>, { runtime, dist }: BundleOptions) {
+  const ROOT = 'astro:root';
+  const root = `
+    ${[...imports].map(url => `import '${url}';`).join('\n')}
+  `;
+
+  const inputOptions: InputOptions = {
+    input: [...imports],
+    plugins: [
+      {
+        name: 'astro:build',
+        resolveId(source: string, imported?: string) {
+          if(source === ROOT) {
+            return source;
+          }
+          if(source.startsWith('/')) {
+            return source;
+          }
+
+          if(imported) {
+            const outUrl = new URL(source, 'http://example.com' + imported);
+            return outUrl.pathname;
+          }
+
+          return null;
+        },
+        async load(id: string) {
+          if(id === ROOT) {
+            return root;
+          }
+
+          const result = await runtime.load(id);
+
+          if(result.statusCode !== 200) {
+            return null;
+          }
+
+          return result.contents.toString('utf-8');
+        }
+      }
+    ]
+  }
+
+  const build = await rollup(inputOptions);
+
+  const outputOptions: OutputOptions = {
+    dir: dist.pathname,
+    format: 'esm',
+    exports: 'named',
+    entryFileNames(chunk) {
+      return chunk.facadeModuleId!.substr(1);
+    }
+  };
+
+  await build.write(outputOptions);
+}

src/build/static.ts

@@ -0,0 +1,26 @@
+import type {Element} from 'domhandler';
+import cheerio from 'cheerio';
+
+export function collectStatics(html: string) {
+  const statics = new Set<string>();
+
+  const $ = cheerio.load(html);
+
+  const append = (el: Element, attr: string) => {
+    const value: string = $(el).attr(attr)!;
+    if(value.startsWith('http')) {
+      return;
+    }
+    statics.add(value);
+  }
+
+  $('link[href]').each((i, el) => {
+    append(el, 'href');
+  });
+
+  $('img[src]').each((i, el) => {
+    append(el, 'src');
+  });
+
+  return statics;
+}

src/compiler/codegen.ts

@@ -9,9 +9,7 @@ import path from 'path';
 import { walk } from 'estree-walker';
 import babelParser from '@babel/parser';
 import _babelGenerator from '@babel/generator';
-import traverse from '@babel/traverse';
-import { ImportDeclaration, ExportNamedDeclaration, VariableDeclarator, Identifier, VariableDeclaration } from '@babel/types';
-import { type } from 'node:os';
+import { ImportDeclaration, ExportNamedDeclaration, VariableDeclarator, Identifier } from '@babel/types';
 
 const babelGenerator: typeof _babelGenerator =
   // @ts-ignore

src/config.ts

@@ -18,5 +18,8 @@ export async function loadConfig(rawRoot: string | undefined): Promise<AstroConf
   const astroConfig: AstroConfig = (await import(astroConfigPath)).default;
   astroConfig.projectRoot = new URL(astroConfig.projectRoot + '/', fileProtocolRoot);
   astroConfig.astroRoot = new URL(astroConfig.astroRoot + '/', fileProtocolRoot);
+
+  const publicFolder = astroConfig.public ? astroConfig.public + '/' : './public/';
+  astroConfig.public = new URL(publicFolder, fileProtocolRoot);
   return astroConfig;
 }

src/dev.ts

@@ -21,7 +21,7 @@ const logging: LogOptions = {
 export default async function (astroConfig: AstroConfig) {
   const { projectRoot } = astroConfig;
 
-  const runtime = await createRuntime(astroConfig, { logging, env: 'dev' });
+  const runtime = await createRuntime(astroConfig, { logging });
 
   const server = http.createServer(async (req, res) => {
     const result = await runtime.load(req.url);

src/runtime.ts

@@ -106,12 +106,17 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro
   }
 }
 
-interface RuntimeOptions {
-  logging: LogOptions;
-  env: 'dev' | 'build';
+export interface AstroRuntime {
+  runtimeConfig: RuntimeConfig;
+  load: (rawPathname: string | undefined) => Promise<LoadResult>;
+  shutdown: () => Promise<void>;
 }
 
-export async function createRuntime(astroConfig: AstroConfig, { env, logging }: RuntimeOptions) {
+interface RuntimeOptions {
+  logging: LogOptions;
+}
+
+export async function createRuntime(astroConfig: AstroConfig, { logging }: RuntimeOptions): Promise<AstroRuntime> {
   const { projectRoot, astroRoot, extensions } = astroConfig;
 
   const internalPath = new URL('./frontend/', import.meta.url);
@@ -122,16 +127,8 @@ export async function createRuntime(astroConfig: AstroConfig, { env, logging }:
     extensions?: Record<string, string>;
   } = {
     extensions,
-    resolve: env === 'dev' ? async (pkgName: string) => snowpack.getUrlForPackage(pkgName) : async (pkgName: string) => `/_snowpack/pkg/${pkgName}.js`,
+    resolve: async (pkgName: string) => snowpack.getUrlForPackage(pkgName)
   };
-  /*if (existsSync(new URL('./package-lock.json', projectRoot))) {
-    const pkgLockStr = await readFile(new URL('./package-lock.json', projectRoot), 'utf-8');
-    const pkgLock = JSON.parse(pkgLockStr);
-    astroPlugOptions.resolve = (pkgName: string) => {
-      const ver = pkgLock.dependencies[pkgName].version;
-      return `/_snowpack/pkg/${pkgName}.v${ver}.js`;
-    };
-  }*/
 
   const snowpackConfig = await loadConfiguration({
     root: projectRoot.pathname,