Reject incomplete sitemap generation and recover after source failures

scripts/prepare-sitemap-test.ts

@@ -0,0 +1,25 @@
+import { mkdtemp, mkdir, writeFile } from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import { libraries } from '../src/libraries/libraries'
+
+const root = await mkdtemp(path.join(os.tmpdir(), 'tanstack-sitemap-'))
+for (const library of libraries) {
+  if (
+    library.visible === false ||
+    !library.latestVersion ||
+    library.sitemap?.includeDocsPages !== true
+  )
+    continue
+  const directory = path.join(
+    root,
+    library.repo.split('/')[1],
+    library.docsRoot || 'docs',
+  )
+  await mkdir(directory, { recursive: true })
+  await writeFile(
+    path.join(directory, 'seo-completeness-probe.md'),
+    '---\ntitle: Sitemap verification\n---\nSitemap verification fixture.\n',
+  )
+}
+console.log(root)

sitemap-testing.md

@@ -0,0 +1,21 @@
+# Sitemap failure and recovery test
+
+Run `pnpm exec tsx scripts/prepare-sitemap-test.ts` to create isolated documentation repositories. It prints their temporary directory. The test only changes files containing its exact fixture text.
+
+Start a local preview with that directory as `TANSTACK_LOCAL_REPOS_DIR`:
+
+```sh
+DISABLE_REDACT=true TANSTACK_LOCAL_REPOS_DIR=/path/printed/by/script pnpm exec vite dev --host 127.0.0.1 --port 4314 --strictPort
+```
+
+In another terminal, run:
+
+```sh
+TANSTACK_SITEMAP_TEST_BASE_URL=http://127.0.0.1:4314 TANSTACK_SITEMAP_TEST_REPOS=/path/printed/by/script pnpm exec tsx --test tests/sitemap-completeness.integration.test.ts
+```
+
+The preview still needs access to the published Charts catalog or its existing GitHub artifact cache. This test isolates documentation reads, not the catalog service.
+
+The test verifies every sitemap-enabled documentation library, removes the Start fixture, restores it, corrupts its frontmatter, and restores it again. Each failure must return 503 with no-store headers and a Retry-After value. Each recovery must restore all expected document URLs. The server log identifies the failed library and records successful generation after recovery.
+
+Stop the preview and remove the generated temporary directory when finished. The GitHub directory fallback and deep local traversal tests run with the normal `pnpm test` suite without a preview.

src/routes/sitemap[.]xml.ts

@@ -6,7 +6,23 @@ export const Route = createFileRoute('/sitemap.xml')({
   server: {
     handlers: {
       GET: async () => {
-        const content = await generateSitemapXml(getSiteOrigin())
+        let content: string
+        try {
+          content = await generateSitemapXml(getSiteOrigin())
+        } catch (error) {
+          console.error('[sitemap] Generation failed', error)
+          return new Response('Sitemap temporarily unavailable', {
+            status: 503,
+            headers: {
+              'Content-Type': 'text/plain; charset=utf-8',
+              'Cache-Control': 'no-store',
+              'Cloudflare-CDN-Cache-Control': 'no-store',
+              'Retry-After': '60',
+            },
+          })
+        }
+
+        console.info('[sitemap] Generation succeeded')
 
         setResponseHeader('Content-Type', 'application/xml; charset=utf-8')
         setResponseHeader(

src/utils/docs.functions.ts

@@ -372,7 +372,8 @@ export const fetchDocsManifest = createServerFn({ method: 'GET' })
       gitRef: branch,
       docsRoot,
       artifactType: 'docs-manifest',
-      artifactKey: 'default',
+      // Older manifests may have been built from depth-limited directory trees.
+      artifactKey: 'complete-tree-v1',
       isValue: isDocsManifest,
       build: () => buildDocsManifest({ repo, branch, docsRoot }),
     })
@@ -397,7 +398,8 @@ export const fetchDocsPathManifest = createServerFn({ method: 'GET' })
       gitRef: branch,
       docsRoot,
       artifactType: 'docs-path-manifest',
-      artifactKey: 'default',
+      // Older manifests may have been built from depth-limited directory trees.
+      artifactKey: 'complete-tree-v1',
       isValue: isDocsManifest,
       build: () => buildDocsPathManifest({ repo, branch, docsRoot }),
     })

src/utils/documents.server.ts

@@ -11,7 +11,6 @@ import {
   getHostRuntimeEnv,
   isIsolateRuntime,
 } from '~/server/runtime/host.server'
-import { fetchCached } from '~/utils/cache.server'
 import {
   getCachedGitHubJsonContent,
   getCachedGitHubTextFile,
@@ -632,14 +631,8 @@ export async function fetchRepoRawFile(
     throw new InvalidCacheKeyError('path', filepath)
   }
 
-  const key = `raw:${repoPair}:${ref}:${filepath}`
-
   if (shouldUseLocalDocsFiles()) {
-    return fetchCached({
-      key,
-      ttl: 1,
-      fn: () => fetchRepoRawFileFromOrigin(repoPair, ref, filepath),
-    })
+    return fetchRepoRawFileFromOrigin(repoPair, ref, filepath)
   }
 
   try {
@@ -687,14 +680,8 @@ export async function fetchRepoFile(
   ref: string,
   filepath: string,
 ) {
-  const key = `${repoPair}:${ref}:${filepath}`
-
   if (shouldUseLocalDocsFiles()) {
-    return fetchCached({
-      key,
-      ttl: 1,
-      fn: () => fetchRepoFileFromOrigin(repoPair, ref, filepath),
-    })
+    return fetchRepoFileFromOrigin(repoPair, ref, filepath)
   }
 
   try {
@@ -1093,29 +1080,17 @@ interface GitHubRecursiveTreeResponse {
 }
 
 function isGitHubFileNode(value: unknown): value is GitHubFileNode {
-  const candidate = value as {
-    _links?: { self?: unknown }
-    children?: unknown
-    depth?: unknown
-    name?: unknown
-    path?: unknown
-    type?: unknown
-  } | null
-
+  if (!isRecord(value)) return false
   return (
-    typeof value === 'object' &&
-    value !== null &&
-    typeof candidate?.name === 'string' &&
-    typeof candidate.path === 'string' &&
-    typeof candidate.type === 'string' &&
-    typeof candidate.depth === 'number' &&
-    typeof candidate._links === 'object' &&
-    candidate._links !== null &&
-    typeof candidate._links.self === 'string' &&
-    (!('children' in value) ||
-      candidate.children === undefined ||
-      (Array.isArray(candidate.children) &&
-        candidate.children.every((child) => isGitHubFileNode(child))))
+    typeof value.name === 'string' &&
+    typeof value.path === 'string' &&
+    typeof value.type === 'string' &&
+    typeof value.depth === 'number' &&
+    isRecord(value._links) &&
+    typeof value._links.self === 'string' &&
+    ((value.type !== 'dir' && value.children === undefined) ||
+      (Array.isArray(value.children) &&
+        value.children.every((child) => isGitHubFileNode(child))))
   )
 }
 
@@ -1419,8 +1394,6 @@ function buildFileTreeFromRecursiveTree(
   return buildChildren(normalizedStart, 0, '')
 }
 
-const API_CONTENTS_MAX_DEPTH = 3
-
 function encodeGitHubContentsPath(path: string) {
   return removeLeadingSlash(path)
     .replace(/\/+$/g, '')
@@ -1436,11 +1409,7 @@ export function fetchApiContents(
   startingPath: string,
 ) {
   if (shouldUseLocalDocsFiles()) {
-    return fetchCached({
-      key: `${repoPair}:${branch}:${startingPath}`,
-      ttl: 1,
-      fn: () => fetchApiContentsFs(repoPair, startingPath),
-    })
+    return fetchApiContentsFs(repoPair, startingPath)
   }
 
   return getCachedGitHubJsonContent({
@@ -1506,30 +1475,15 @@ async function fetchApiContentsFs(
   async function getContentsForPath(
     filePath: string,
   ): Promise<Array<GitHubFile>> {
-    try {
-      const list = await fsp.readdir(filePath, { withFileTypes: true })
-      return list
-        .filter((item) => !dirsAndFilesToIgnore.includes(item.name))
-        .map((item) => {
-          return {
-            name: item.name,
-            path: path.join(filePath, item.name),
-            type: item.isDirectory() ? 'dir' : 'file',
-            _links: {
-              self: path.join(filePath, item.name),
-            },
-          }
-        })
-    } catch (error) {
-      if (
-        error instanceof Error &&
-        'code' in error &&
-        error.code === 'ENOENT'
-      ) {
-        return []
-      }
-      throw error
-    }
+    const list = await fsp.readdir(filePath, { withFileTypes: true })
+    return list
+      .filter((item) => !dirsAndFilesToIgnore.includes(item.name))
+      .map((item) => ({
+        name: item.name,
+        path: path.join(filePath, item.name),
+        type: item.isDirectory() ? 'dir' : 'file',
+        _links: { self: path.join(filePath, item.name) },
+      }))
   }
 
   const data = await getContentsForPath(fsStartPath)
@@ -1554,7 +1508,7 @@ async function fetchApiContentsFs(
         parentPath,
       }
 
-      if (file.type === 'dir' && depth <= API_CONTENTS_MAX_DEPTH) {
+      if (file.type === 'dir') {
         const directoryFiles = await getContentsForPath(file._links.self)
         file.children = await buildFileTree(
           directoryFiles,
@@ -1656,19 +1610,23 @@ async function fetchApiContentsRemoteFromContentsApi(
         parentPath,
       }
 
-      if (file.type === 'dir' && depth <= API_CONTENTS_MAX_DEPTH) {
+      if (file.type === 'dir') {
         const directoryFiles = await fetchGitHubDirectoryContents(
           repo,
           branch,
           file.path,
         )
-        file.children = directoryFiles
-          ? await buildFileTree(
-              directoryFiles,
-              depth + 1,
-              `${parentPath}${file.path}/`,
-            )
-          : []
+        if (directoryFiles === null) {
+          throw new GitHubContentError(
+            'invalid-response',
+            `Listed directory disappeared: ${repo}@${branch}:${file.path}`,
+          )
+        }
+        file.children = await buildFileTree(
+          directoryFiles,
+          depth + 1,
+          `${parentPath}${file.path}/`,
+        )
       }
 
       result.push(file)

src/utils/local-docs-tree.server.ts

@@ -46,7 +46,7 @@ export async function readLocalDocsTree(
         depth,
         parentPath: directory,
         _links: { self: filePath },
-        ...(entry.isDirectory() && depth <= 3
+        ...(entry.isDirectory()
           ? { children: await readLocalDocsTree(repoDir, filePath, depth + 1) }
           : {}),
       }

src/utils/sitemap.ts

@@ -89,7 +89,18 @@ async function getLibraryDocsEntries(
     repo: library.repo,
     branch,
     docsRoot,
-  }).catch(() => ({ paths: [], redirects: {}, lastModifiedByPath: undefined }))
+  }).catch((cause: unknown) => {
+    throw new Error(
+      `Sitemap docs unavailable: ${library.id} (${library.repo}@${branch}:${docsRoot})`,
+      { cause },
+    )
+  })
+
+  if (manifest.paths.length === 0) {
+    throw new Error(
+      `Sitemap docs empty: ${library.id} (${library.repo}@${branch}:${docsRoot})`,
+    )
+  }
 
   return manifest.paths
     .filter(Boolean)
@@ -146,8 +157,7 @@ async function getPublishedChartsCatalogEntries(): Promise<
     const publication = await getChartsCatalogIndexPublication()
     return getChartsCatalogSitemapEntries(publication.index)
   } catch (error) {
-    console.error('[sitemap] Charts catalog unavailable', error)
-    return [{ path: '/charts/catalog/' }]
+    throw new Error('Sitemap charts catalog unavailable', { cause: error })
   }
 }