perf: improve perf with more delegation

.changeset/html-streaming-pages.md

@@ -0,0 +1,6 @@
+---
+'@doc-kit/generator-react': minor
+'@doc-kit/core': minor
+---
+
+perf: improve (yay!)

packages/core/src/utils/configuration/index.mjs

@@ -20,6 +20,9 @@ import { deepMerge } from '#utils/misc.mjs';
 
 const configExplorer = cosmiconfig('doc-kit');
 
+// The default `threads` ceiling; `--threads` raises it explicitly.
+const MAX_THREADS = 4;
+
 /**
  * The name of the project being documented, from the manifest in the working
  * directory. Generators use it for titles, logos, and templated text.
@@ -68,7 +71,12 @@ export const getDefaultConfig = (generators, config) =>
       // riscv64 with sv39. Running multiple generators that use wasm in
       // parallel could cause failures to allocate new wasm instance.
       // See also https://github.com/nodejs/node/pull/60591
-      threads: process.arch === 'riscv64' ? 1 : cpus().length,
+      //
+      // Elsewhere the count is capped: each worker that highlights code holds
+      // Shiki's grammars and regex engine (~300MB) on top of the pages it is
+      // building, so past a few threads memory, not CPU, is what runs out.
+      threads:
+        process.arch === 'riscv64' ? 1 : Math.min(cpus().length, MAX_THREADS),
       chunkSize: 10,
     })
   );

packages/core/src/utils/remark-shiki.mjs

@@ -0,0 +1,29 @@
+'use strict';
+
+import rehypeStringify from 'rehype-stringify';
+import remarkParse from 'remark-parse';
+import remarkRehype from 'remark-rehype';
+import { unified } from 'unified';
+
+import syntaxHighlighter from './highlighter.mjs';
+import { lazy } from './misc.mjs';
+import { rehypeOptions } from './remark.mjs';
+
+/**
+ * Retrieves an instance of Remark configured to output stringified HTML code
+ * including parsing Code Boxes with syntax highlighting.
+ *
+ * This lives apart from `./remark.mjs` because importing it loads Shiki (every
+ * grammar plus the regex engine, ~250MB per process). Only the generators that
+ * highlight code — `legacy-html` here — should pay for that.
+ */
+export const getRemarkRehypeWithShiki = lazy(() =>
+  unified()
+    .use(remarkParse)
+    // legacy-html gets the minimal (unhighlighted) type rendering
+    .use(remarkRehype, rehypeOptions)
+    // This is a custom ad-hoc within the Shiki Rehype plugin, used to highlight code
+    // and transform them into HAST nodes
+    .use(syntaxHighlighter)
+    .use(rehypeStringify, { allowDangerousHtml: true })
+);

packages/core/src/utils/remark.mjs

@@ -8,13 +8,13 @@ import remarkRehype from 'remark-rehype';
 import remarkStringify from 'remark-stringify';
 import { unified } from 'unified';
 
-import syntaxHighlighter from './highlighter.mjs';
 import { lazy } from './misc.mjs';
 import { typeAnnotationToHast } from './type-annotations/hast.mjs';
 import remarkTypeAnnotations from './type-annotations/remark.mjs';
 
-// Nodes the rehype pipelines pass through untouched.
-const passThrough = ['element'];
+// Nothing in this module loads Shiki: the `ast` and `metadata` stages (and
+// every worker that runs them) import it, and none of them highlight code.
+// The highlighting pipeline lives in `./remark-shiki.mjs`.
 
 /**
  * Renders an MDX JSX element as just its children, so the surrounding prose
@@ -30,15 +30,28 @@ const mdxElementToChildren = (state, node) => state.all(node);
  */
 const dropNode = () => undefined;
 
-// The HTML-string pipelines cannot render MDX nodes (rendering those is the
-// React generators' job): JSX elements degrade to their children so the
-// surrounding prose still renders, and expressions/ESM are dropped.
-const mdxToHastHandlers = {
-  mdxJsxTextElement: mdxElementToChildren,
-  mdxJsxFlowElement: mdxElementToChildren,
-  mdxFlowExpression: dropNode,
-  mdxTextExpression: dropNode,
-  mdxjsEsm: dropNode,
+/**
+ * The `remark-rehype` options shared by the HTML-string pipelines.
+ *
+ * Existing HTML nodes pass through untouched (they were created during the
+ * rehype process), and dangerous HTML is allowed since the Markdown sources
+ * are trusted. The MDX node types cannot be rendered to an HTML string (that
+ * is the React generators' job): JSX elements degrade to their children so the
+ * surrounding prose still renders, and expressions/ESM are dropped.
+ *
+ * @type {import('remark-rehype').Options}
+ */
+export const rehypeOptions = {
+  allowDangerousHtml: true,
+  passThrough: ['element'],
+  handlers: {
+    typeAnnotation: typeAnnotationToHast,
+    mdxJsxTextElement: mdxElementToChildren,
+    mdxJsxFlowElement: mdxElementToChildren,
+    mdxFlowExpression: dropNode,
+    mdxTextExpression: dropNode,
+    mdxjsEsm: dropNode,
+  },
 };
 
 /**
@@ -72,41 +85,6 @@ export const getRemarkMdx = lazy(() =>
 export const getRemarkRehype = lazy(() =>
   unified()
     .use(remarkParse)
-    // We make Rehype ignore existing HTML nodes (just the node itself, not its children)
-    // as these are nodes we manually created during the rehype process
-    // We also allow dangerous HTML to be passed through, since we have HTML within our Markdown
-    // and we trust the sources of the Markdown files
-    .use(remarkRehype, {
-      allowDangerousHtml: true,
-      passThrough,
-      handlers: { typeAnnotation: typeAnnotationToHast, ...mdxToHastHandlers },
-    })
-    // We allow dangerous HTML to be passed through, since we have HTML within our Markdown
-    // and we trust the sources of the Markdown files
-    .use(rehypeStringify, { allowDangerousHtml: true })
-);
-
-/**
- * Retrieves an instance of Remark configured to output stringified HTML code
- * including parsing Code Boxes with syntax highlighting
- */
-export const getRemarkRehypeWithShiki = lazy(() =>
-  unified()
-    .use(remarkParse)
-    // We make Rehype ignore existing HTML nodes (just the node itself, not its children)
-    // as these are nodes we manually created during the rehype process
-    // We also allow dangerous HTML to be passed through, since we have HTML within our Markdown
-    // and we trust the sources of the Markdown files
-    .use(remarkRehype, {
-      allowDangerousHtml: true,
-      passThrough,
-      // legacy-html gets the minimal (unhighlighted) type rendering
-      handlers: { typeAnnotation: typeAnnotationToHast, ...mdxToHastHandlers },
-    })
-    // This is a custom ad-hoc within the Shiki Rehype plugin, used to highlight code
-    // and transform them into HAST nodes
-    .use(syntaxHighlighter)
-    // We allow dangerous HTML to be passed through, since we have HTML within our Markdown
-    // and we trust the sources of the Markdown files
+    .use(remarkRehype, rehypeOptions)
     .use(rehypeStringify, { allowDangerousHtml: true })
 );

packages/core/src/utils/type-annotations/__tests__/hast.test.mjs

@@ -4,10 +4,8 @@ import { describe, it } from 'node:test';
 import { toHtml } from 'hast-util-to-html';
 import { toString } from 'hast-util-to-string';
 
-import {
-  typeAnnotationToHast,
-  typeAnnotationToHighlightedHast,
-} from '../hast.mjs';
+import { typeAnnotationToHast } from '../hast.mjs';
+import { typeAnnotationToHighlightedHast } from '../highlighted.mjs';
 
 // A minimal mdast-util-to-hast state — the handlers only use patch/applyData
 const state = { patch: () => {}, applyData: (_, result) => result };

packages/core/src/utils/type-annotations/hast.mjs

@@ -1,9 +1,5 @@
 'use strict';
 
-import { highlighter } from '#utils/highlighter.mjs';
-
-const [lightTheme, darkTheme] = highlighter.shiki.getLoadedThemes();
-
 /**
  * Slices a type's text by its resolved link ranges into hast children —
  * plain text segments interleaved with `<a class="type-link">` anchors.
@@ -59,57 +55,3 @@ export const typeAnnotationToHast = (state, node) => {
 
   return state.applyData(node, result);
 };
-
-/**
- * Syntax-highlighted mdast→hast handler for `typeAnnotation` nodes, used by
- * the web (JSX) pipeline. The whole type is highlighted as one inline
- * fragment, and each resolved identifier's exact character range is wrapped
- * in an `<a>` via Shiki decorations. Values that are not TypeScript (display
- * names such as `HTTP/2 Headers Object`) are highlighted as plain text, so
- * their prose is not coloured as operators and numeric literals.
- *
- * Falls back to the minimal handler when the type failed to parse or nothing
- * resolved (no point paying for highlighting then).
- *
- * @param {import('mdast-util-to-hast').State} state
- * @param {import('mdast').Node} node
- * @returns {import('hast').Element}
- */
-export const typeAnnotationToHighlightedHast = (state, node) => {
-  const links = node.data?.links ?? [];
-
-  if (node.data?.parseError || links.length === 0) {
-    return typeAnnotationToHast(state, node);
-  }
-
-  const root = highlighter.shiki.codeToHast(node.value, {
-    lang: node.data?.typescript ? 'typescript' : 'text',
-    themes: { light: lightTheme, dark: darkTheme },
-    decorations: links.map(({ start, end, href }) => ({
-      start,
-      end,
-      tagName: 'a',
-      properties: { href, class: 'type-link' },
-      alwaysWrap: true,
-    })),
-  });
-
-  // codeToHast wraps the highlighted line in <pre><code>; re-shape that into
-  // a single inline <code> element ("only the outermost type opens/closes
-  // the code fragment")
-  const [preElement] = root.children;
-  const [codeElement] = preElement.children;
-
-  const result = {
-    type: 'element',
-    tagName: 'code',
-    properties: {
-      class: `${preElement.properties.class} type`,
-    },
-    children: codeElement.children,
-  };
-
-  state.patch(node, result);
-
-  return state.applyData(node, result);
-};

packages/core/src/utils/type-annotations/highlighted.mjs

@@ -0,0 +1,64 @@
+'use strict';
+
+import { highlighter } from '#utils/highlighter.mjs';
+
+import { typeAnnotationToHast } from './hast.mjs';
+
+// Kept apart from `./hast.mjs` on purpose: importing this module loads Shiki
+// (every grammar plus the regex engine), which only the pipelines that
+// highlight should pay for. The `ast` and `metadata` stages never do.
+const [lightTheme, darkTheme] = highlighter.shiki.getLoadedThemes();
+
+/**
+ * Syntax-highlighted mdast→hast handler for `typeAnnotation` nodes, used by
+ * the web (JSX) pipeline. The whole type is highlighted as one inline
+ * fragment, and each resolved identifier's exact character range is wrapped
+ * in an `<a>` via Shiki decorations. Values that are not TypeScript (display
+ * names such as `HTTP/2 Headers Object`) are highlighted as plain text, so
+ * their prose is not coloured as operators and numeric literals.
+ *
+ * Falls back to the minimal handler when the type failed to parse or nothing
+ * resolved (no point paying for highlighting then).
+ *
+ * @param {import('mdast-util-to-hast').State} state
+ * @param {import('mdast').Node} node
+ * @returns {import('hast').Element}
+ */
+export const typeAnnotationToHighlightedHast = (state, node) => {
+  const links = node.data?.links ?? [];
+
+  if (node.data?.parseError || links.length === 0) {
+    return typeAnnotationToHast(state, node);
+  }
+
+  const root = highlighter.shiki.codeToHast(node.value, {
+    lang: node.data?.typescript ? 'typescript' : 'text',
+    themes: { light: lightTheme, dark: darkTheme },
+    decorations: links.map(({ start, end, href }) => ({
+      start,
+      end,
+      tagName: 'a',
+      properties: { href, class: 'type-link' },
+      alwaysWrap: true,
+    })),
+  });
+
+  // codeToHast wraps the highlighted line in <pre><code>; re-shape that into
+  // a single inline <code> element ("only the outermost type opens/closes
+  // the code fragment")
+  const [preElement] = root.children;
+  const [codeElement] = preElement.children;
+
+  const result = {
+    type: 'element',
+    tagName: 'code',
+    properties: {
+      class: `${preElement.properties.class} type`,
+    },
+    children: codeElement.children,
+  };
+
+  state.patch(node, result);
+
+  return state.applyData(node, result);
+};

packages/node-legacy/src/legacy-html/generate.mjs

@@ -7,7 +7,7 @@ import getConfig from '@doc-kit/core/utils/configuration/index.mjs';
 import { writeFile } from '@doc-kit/core/utils/file.mjs';
 import { groupNodesByModule } from '@doc-kit/core/utils/generators.mjs';
 import { minifyHTML } from '@doc-kit/core/utils/html-minifier.mjs';
-import { getRemarkRehypeWithShiki as remark } from '@doc-kit/core/utils/remark.mjs';
+import { getRemarkRehypeWithShiki as remark } from '@doc-kit/core/utils/remark-shiki.mjs';
 
 import buildContent from './utils/buildContent.mjs';
 import { replaceTemplateValues } from './utils/replaceTemplateValues.mjs';

packages/node-legacy/src/legacy-html/utils/buildContent.mjs

@@ -6,7 +6,7 @@ import {
   populate,
 } from '@doc-kit/core/utils/configuration/templates.mjs';
 import { UNIST } from '@doc-kit/core/utils/queries/index.mjs';
-import { getRemarkRehypeWithShiki as remark } from '@doc-kit/core/utils/remark.mjs';
+import { getRemarkRehypeWithShiki as remark } from '@doc-kit/core/utils/remark-shiki.mjs';
 import { h as createElement } from 'hastscript';
 import { u as createTree } from 'unist-builder';
 import { SKIP, visit } from 'unist-util-visit';