diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 7fe39a4e2c95..cccf8d854a5b 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -983,6 +983,7 @@ jobs: test/e2e/app-dir/app/index.test.ts \ test/e2e/app-dir/app-edge/app-edge.test.ts \ test/e2e/app-dir/proxy-runtime-nodejs/proxy-runtime-nodejs.test.ts \ + test/e2e/app-dir/turbopack-additional-roots/turbopack-additional-roots.test.ts \ test/development/app-dir/segment-explorer/segment-explorer.test.ts stepName: 'test-dev-windows' runs_on_labels: '["windows-latest-8-core-oss"]' diff --git a/Cargo.lock b/Cargo.lock index ebee7cb6372a..516f278eded3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9312,6 +9312,7 @@ dependencies = [ "notify", "omnipath", "parking_lot", + "pathdiff", "rand 0.10.1", "regex", "rstest", diff --git a/Cargo.toml b/Cargo.toml index 2ded1edebe20..3cb6ca293a07 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -341,11 +341,9 @@ serde_qs = "1.1.1" serde_with = "3.18.0" sha2 = "0.10.2" smallvec = { version = "1.15.1", features = [ - "serde", "const_generics", "union", "const_new", - "impl_bincode", ] } shrink-to-fit = "0.2.10" strsim = "0.11.1" diff --git a/crates/next-api/Cargo.toml b/crates/next-api/Cargo.toml index c6e501e55ec2..c5b1d61f8cab 100644 --- a/crates/next-api/Cargo.toml +++ b/crates/next-api/Cargo.toml @@ -34,6 +34,7 @@ serde = { workspace = true } serde_json = { workspace = true } swc_core = { workspace = true } tracing = { workspace = true } +tokio = { workspace = true } turbo-bincode = { workspace = true } turbo-rcstr = { workspace = true } turbo-tasks = { workspace = true } diff --git a/crates/next-api/src/additional_roots.rs b/crates/next-api/src/additional_roots.rs new file mode 100644 index 000000000000..add58e690b31 --- /dev/null +++ b/crates/next-api/src/additional_roots.rs @@ -0,0 +1,465 @@ +use std::{ + collections::BTreeMap, + ops::Bound, + path::{Path, PathBuf}, +}; + +use anyhow::Result; +use async_trait::async_trait; +use bincode::{Decode, Encode}; +use serde::{Deserialize, Serialize}; +use turbo_rcstr::{RcStr, rcstr}; +use turbo_tasks::{ + FxIndexMap, NonLocalValue, OperationValue, OperationVc, ReadRef, ResolvedVc, Vc, + trace::TraceRawVcs, +}; +use turbo_tasks_fs::{ + DiskFileSystem, DiskFileSystemMap, DiskWatcherConfig, DiskWatcherRecursiveMode, FileSystemPath, + canonicalize_to_rcstr, +}; +use turbopack_core::issue::{Issue, IssueSeverity, IssueStage, PlainIssue, StyledString}; + +use crate::project::{ + ProjectContainer, additional_root_path_operation, disk_file_system_operation, +}; + +/// A named additional filesystem root. +#[derive( + Clone, + Debug, + PartialEq, + Eq, + Serialize, + Deserialize, + NonLocalValue, + OperationValue, + TraceRawVcs, + Encode, + Decode, +)] +pub struct AdditionalRootConfig { + pub key: RcStr, + pub path: RcStr, + pub ignore_if_missing: bool, +} + +#[turbo_tasks::task_input] +#[derive( + Clone, + Debug, + PartialEq, + Eq, + Hash, + OperationValue, + TraceRawVcs, + Serialize, + Deserialize, + Encode, + Decode, +)] +enum AdditionalRootInvalidName { + Empty, + TooLong, + InvalidCharacter, + WindowsDeviceName, +} + +impl AdditionalRootInvalidName { + fn description(&self) -> StyledString { + match self { + Self::Empty => StyledString::Text(rcstr!("the name must not be empty")), + Self::TooLong => { + StyledString::Text(rcstr!("the name must be at most 40 ASCII characters")) + } + Self::InvalidCharacter => StyledString::Text(rcstr!( + "the name must contain only ASCII letters, digits, underscores, and hyphens" + )), + Self::WindowsDeviceName => { + StyledString::Text(rcstr!("the name must not be a Windows device name")) + } + } + } +} + +#[turbo_tasks::task_input] +#[derive( + Clone, + Debug, + PartialEq, + Eq, + Hash, + OperationValue, + TraceRawVcs, + Serialize, + Deserialize, + Encode, + Decode, +)] +enum AdditionalRootIssueReason { + // io errors are stringified because `io::Error` does not implement the required traits + Io(RcStr), + InvalidName(AdditionalRootInvalidName), + NameCollision { existing_key: RcStr }, + OverlappingRoot { key: Option, path: RcStr }, +} + +impl AdditionalRootIssueReason { + fn description(&self) -> StyledString { + match self { + Self::Io(error) => StyledString::Text(error.clone()), + Self::InvalidName(reason) => reason.description(), + Self::NameCollision { existing_key } => StyledString::Line(vec![ + StyledString::Text(rcstr!( + "the name collides case-insensitively with the earlier root " + )), + StyledString::Code(existing_key.clone()), + ]), + Self::OverlappingRoot { + key: Some(key), + path, + } => StyledString::Line(vec![ + StyledString::Text(rcstr!("the root overlaps additional root ")), + StyledString::Code(path.clone()), + StyledString::Text(rcstr!(" configured as ")), + StyledString::Code(key.clone()), + ]), + Self::OverlappingRoot { key: None, path } => StyledString::Line(vec![ + StyledString::Text(rcstr!("the additional root overlaps the project root ")), + StyledString::Code(path.clone()), + ]), + } + } +} + +#[derive( + Clone, Debug, PartialEq, Eq, NonLocalValue, OperationValue, TraceRawVcs, Encode, Decode, +)] +pub(crate) struct AdditionalDiskFileSystem { + pub canonical_path: RcStr, + pub file_system: OperationVc, +} + +/// Constructed file systems and issues for the configured additional roots. +pub(crate) struct AdditionalRootsInitialization { + pub roots_by_name: FxIndexMap, + pub issues: Vec>, +} + +pub(crate) async fn create_additional_root_file_systems( + container: ResolvedVc, + additional_roots: Vec, + project_root: &RcStr, + watcher_config: DiskWatcherConfig, + map: OperationVc, + issue_path: FileSystemPath, +) -> Result { + let mut overlapping_check = OverlappingRootCheck::new(project_root.clone()); + let mut configured_names: FxIndexMap = FxIndexMap::default(); + let mut roots_by_name = FxIndexMap::default(); + let mut issues: Vec> = Vec::new(); + for additional_root in additional_roots { + let mut push_issue = async |reason: AdditionalRootIssueReason| -> Result<()> { + if let Some(issue) = &*additional_root_issue_operation( + container, + issue_path.clone(), + additional_root.key.clone(), + additional_root.path.clone(), + reason, + ) + .read_strongly_consistent() + .await? + { + issues.push(issue.clone()); + } + Ok(()) + }; + + if let Err(reason) = validate_additional_root_name(&additional_root.key) { + push_issue(AdditionalRootIssueReason::InvalidName(reason)).await?; + continue; + } + + let folded_name = RcStr::from(additional_root.key.to_ascii_lowercase()); + if let Some(existing_key) = configured_names.get(&folded_name) { + push_issue(AdditionalRootIssueReason::NameCollision { + existing_key: existing_key.clone(), + }) + .await?; + continue; + } + configured_names.insert(folded_name, additional_root.key.clone()); + + let canonical = match canonicalize_to_rcstr(Path::new(&*additional_root.path)) { + Ok(canonical) => canonical, + Err(_) if additional_root.ignore_if_missing => continue, + Err(error) => { + push_issue(AdditionalRootIssueReason::Io(RcStr::from( + error.to_string(), + ))) + .await?; + continue; + } + }; + if let Err((overlapping_key, overlapping_path)) = + overlapping_check.insert(Some(additional_root.key.clone()), canonical.clone()) + { + push_issue(AdditionalRootIssueReason::OverlappingRoot { + key: overlapping_key, + path: overlapping_path, + }) + .await?; + continue; + } + // We're not inside a turbo-task function: Call an operation to create a cell for us. We + // pass the `ProjectContainer` and a key, which both have a stable identity, this reduces + // invalidations when additional roots are added or removed. + let canonical_root = additional_root_path_operation(container, additional_root.key.clone()); + let operation = disk_file_system_operation( + RcStr::from(format!("@{}", additional_root.key)), + canonical_root, + Vec::new(), + DiskWatcherConfig { + // we assume that most files in an additional root won't be read, so a recursive + // watcher may be more expensive than we'd like, always use a non-recursive watcher. + recursive_mode: Some(DiskWatcherRecursiveMode::NonRecursive), + ..watcher_config + }, + map, + ); + roots_by_name.insert( + additional_root.key, + AdditionalDiskFileSystem { + canonical_path: canonical, + file_system: operation, + }, + ); + } + + Ok(AdditionalRootsInitialization { + roots_by_name, + issues, + }) +} + +fn validate_additional_root_name(name: &str) -> Result<(), AdditionalRootInvalidName> { + if name.is_empty() { + return Err(AdditionalRootInvalidName::Empty); + } + if name.len() > 40 { + return Err(AdditionalRootInvalidName::TooLong); + } + if !name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')) + { + return Err(AdditionalRootInvalidName::InvalidCharacter); + } + + let uppercase_name = name.to_ascii_uppercase(); + let is_device_number = + |suffix: &str| suffix.len() == 1 && matches!(suffix.as_bytes()[0], b'1'..=b'9'); + let is_device_name = matches!(uppercase_name.as_str(), "CON" | "PRN" | "AUX" | "NUL") + || uppercase_name + .strip_prefix("COM") + .is_some_and(is_device_number) + || uppercase_name + .strip_prefix("LPT") + .is_some_and(is_device_number); + if is_device_name { + return Err(AdditionalRootInvalidName::WindowsDeviceName); + } + + Ok(()) +} + +#[turbo_tasks::function(operation, root)] +async fn additional_root_issue_operation( + container: ResolvedVc, + path: FileSystemPath, + key: RcStr, + configured_path: RcStr, + reason: AdditionalRootIssueReason, +) -> Result> { + let issue = AdditionalRootIssue { + path, + key, + configured_path, + reason, + }; + let filter = container.project().issue_filter().await?; + Ok(Vc::cell(if filter.matches_ref(&issue).await? { + Some(ReadRef::new_owned( + PlainIssue::from_issue_ref(&issue, None).await?, + )) + } else { + None + })) +} + +#[turbo_tasks::value(transparent, serialization = "skip")] +struct OptionalAdditionalRootIssue(Option>); + +struct OverlappingRootCheck { + accepted: BTreeMap, RcStr)>, +} + +impl OverlappingRootCheck { + fn new(project_root: RcStr) -> Self { + Self { + accepted: BTreeMap::from([(PathBuf::from(&*project_root), (None, project_root))]), + } + } + + fn insert(&mut self, key: Option, path: RcStr) -> Result<(), (Option, RcStr)> { + let canonical = Path::new(&*path); + if let Some((root, value)) = self + .accepted + .range::((Bound::Unbounded, Bound::Included(canonical))) + .next_back() + && canonical.starts_with(root) + { + return Err(value.clone()); + } + if let Some((_, value)) = self + .accepted + .range::((Bound::Included(canonical), Bound::Unbounded)) + .next() + .filter(|(root, _)| root.starts_with(canonical)) + { + return Err(value.clone()); + } + self.accepted.insert(canonical.to_path_buf(), (key, path)); + Ok(()) + } +} + +#[turbo_tasks::value(shared)] +struct AdditionalRootIssue { + path: FileSystemPath, + key: RcStr, + configured_path: RcStr, + reason: AdditionalRootIssueReason, +} + +#[async_trait] +#[turbo_tasks::value_impl] +impl Issue for AdditionalRootIssue { + fn stage(&self) -> IssueStage { + IssueStage::Config + } + + fn severity(&self) -> IssueSeverity { + IssueSeverity::Warning + } + + async fn file_path(&self) -> Result { + Ok(self.path.clone()) + } + + async fn title(&self) -> Result { + Ok(StyledString::Text(rcstr!( + "Invalid Turbopack additional root" + ))) + } + + async fn description(&self) -> Result> { + Ok(Some(StyledString::Line(vec![ + StyledString::Text(rcstr!("The additional root ")), + StyledString::Code(self.configured_path.clone()), + StyledString::Text(rcstr!(" configured as ")), + StyledString::Code(self.key.clone()), + StyledString::Text(rcstr!(" is invalid: ")), + self.reason.description(), + ]))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn identifies_an_overlapping_root() { + let mut roots = OverlappingRootCheck::new(rcstr!("/workspace/project")); + assert_eq!( + roots.insert( + Some(rcstr!("packages")), + rcstr!("/workspace/project/packages") + ), + Err((None, rcstr!("/workspace/project"))) + ); + roots + .insert(Some(rcstr!("vendor")), rcstr!("/workspace/vendor")) + .unwrap(); + assert_eq!( + roots.insert(Some(rcstr!("workspace")), rcstr!("/workspace")), + Err((None, rcstr!("/workspace/project"))) + ); + assert_eq!( + roots.insert(Some(rcstr!("package")), rcstr!("/workspace/vendor/package")), + Err((Some(rcstr!("vendor")), rcstr!("/workspace/vendor"))) + ); + assert_eq!( + roots.insert( + Some(rcstr!("project-other")), + rcstr!("/workspace/project-other") + ), + Ok(()) + ); + } + + #[test] + fn validates_additional_root_names() { + for valid in [ + "linkedPackages", + "packages-1", + "with_underscore", + "letters-AND_123", + "COM10", + ] { + assert_eq!(validate_additional_root_name(valid), Ok(()), "{valid}"); + } + + assert_eq!( + validate_additional_root_name(""), + Err(AdditionalRootInvalidName::Empty) + ); + assert_eq!( + validate_additional_root_name("this-name-is-more-than-forty-ascii-characters-long"), + Err(AdditionalRootInvalidName::TooLong) + ); + + for invalid_character in [ + ".", + "..", + "nön-ascii", + "control\u{1f}", + "delete\u{7f}", + "with/slash", + "with space", + "trailing ", + "trailing.", + ] { + assert_eq!( + validate_additional_root_name(invalid_character), + Err(AdditionalRootInvalidName::InvalidCharacter), + "{invalid_character}" + ); + } + + for device_name in ["CON", "prn", "Aux", "NUL", "cOm1", "LPT9"] { + assert_eq!( + validate_additional_root_name(device_name), + Err(AdditionalRootInvalidName::WindowsDeviceName), + "{device_name}" + ); + } + + assert_eq!( + AdditionalRootInvalidName::InvalidCharacter.description(), + StyledString::Text(rcstr!( + "the name must contain only ASCII letters, digits, underscores, and hyphens" + )) + ); + } +} diff --git a/crates/next-api/src/analyze.rs b/crates/next-api/src/analyze.rs index 2d374133ada7..90ba5d688698 100644 --- a/crates/next-api/src/analyze.rs +++ b/crates/next-api/src/analyze.rs @@ -450,7 +450,7 @@ pub async fn analyze_output_assets( let decoded_source = urlencoding::decode(&chunk_part.source)?; let source = if let Some(stripped) = decoded_source.strip_prefix(&prefix) { Cow::Borrowed(stripped) - } else if decoded_source.starts_with("[project]/") { + } else if decoded_source.starts_with('[') && decoded_source.contains("]/") { decoded_source } else { Cow::Owned(format!( diff --git a/crates/next-api/src/lib.rs b/crates/next-api/src/lib.rs index 2a86c4140344..dda3f7ed689b 100644 --- a/crates/next-api/src/lib.rs +++ b/crates/next-api/src/lib.rs @@ -2,6 +2,7 @@ #![feature(arbitrary_self_types_pointers)] #![feature(impl_trait_in_assoc_type)] +mod additional_roots; pub mod aggregate_hmr; pub mod analyze; mod app; @@ -18,6 +19,7 @@ mod module_graph; pub mod next_server_nft; mod nft; mod nft_json; +mod nft_json_builder; pub mod operation; mod pages; mod path_utils; diff --git a/crates/next-api/src/next_server_nft.rs b/crates/next-api/src/next_server_nft.rs index ab5fb3ea3907..54b8d92d8b7d 100644 --- a/crates/next-api/src/next_server_nft.rs +++ b/crates/next-api/src/next_server_nft.rs @@ -4,7 +4,6 @@ use anyhow::{Context, Result, bail}; use bincode::{Decode, Encode}; use either::Either; use next_core::{get_next_package, next_server::get_tracing_compile_time_info}; -use serde_json::json; use turbo_rcstr::{RcStr, rcstr}; use turbo_tasks::{ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc, trace::TraceRawVcs}; use turbo_tasks_fs::{ @@ -16,6 +15,7 @@ use turbopack::externals_tracing_module_context; use turbopack_core::{ asset::{Asset, AssetContent}, context::AssetContext, + file_source::FileSource, module::{Module, Modules}, module_graph::{GraphEntries, ModuleGraph, SingleModuleGraph}, output::{OutputAsset, OutputAssets, OutputAssetsReference}, @@ -24,7 +24,7 @@ use turbopack_core::{ }; use turbopack_resolve::ecmascript::cjs_resolve; -use crate::{nft::traced_modules_for_entries, project::Project}; +use crate::{nft::traced_modules_for_entries, nft_json_builder::NftJsonBuilder, project::Project}; /// The modules `next/dist/server/require-hook` resolves its aliased requests to at runtime /// (currently all of styled-jsx), so that the Pages Router renderer and user code share a single @@ -187,11 +187,8 @@ impl Asset for ServerNftJsonAsset { let this = self.await?; // Example: [project]/apps/my-website/.next/ - let base_dir = this - .project - .project_root_path() - .await? - .join(&this.project.node_root().await?.path)?; + let nft_path = self.path().owned().await?; + let mut nft_json = NftJsonBuilder::new(this.project, &nft_path).await?; let module_graph = ModuleGraph::from_graphs( vec![SingleModuleGraph::new_with_entries( @@ -205,7 +202,7 @@ impl Asset for ServerNftJsonAsset { let hash_salt = this.project.next_config().output_hash_salt(); - let mut server_output_assets = traced_modules_for_entries( + let server_output_assets = traced_modules_for_entries( module_graph, Modules::empty(), self.entries(), @@ -215,33 +212,36 @@ impl Asset for ServerNftJsonAsset { .await? .iter() .map(async |m| { + let path = m.ident().await?.path.clone(); + let source = m.source().await?.context("NFT module has no content")?; + let content = source.content(); Ok(( - base_dir - .get_relative_path_to(&m.ident().await?.path) - .context("failed to compute relative path for server NFT JSON")?, - m.source() - .await? - .context("NFT module has no content")? - .content() + path, + content .hash(hash_salt, HashAlgorithm::Xxh3Hash128Hex) + .owned() .await?, + content.await?, )) }) .try_join() .await?; + for (path, hash, content) in server_output_assets { + nft_json.add(path, hash, &content)?; + } + let next_dir = get_next_package(this.project.project_path().owned().await?).await?; for ty in ["app-page", "pages"] { let dir = next_dir.join(&format!("dist/server/route-modules/{ty}"))?; let module_path = dir.join("module.compiled.js")?; - server_output_assets.push(( - base_dir - .get_relative_path_to(&module_path) - .context("failed to compute relative path for server NFT JSON")?, - module_path - .hash_file(hash_salt, HashAlgorithm::Xxh3Hash128Hex) - .await?, - )); + let content = FileSource::new(module_path.clone()).content(); + let hash = content + .hash(hash_salt, HashAlgorithm::Xxh3Hash128Hex) + .owned() + .await?; + let content = content.await?; + nft_json.add(module_path, hash, &content)?; let contexts_dir = dir.join("vendored/contexts")?; let DirectoryContent::Entries(contexts_files) = &*contexts_dir.read_dir().await? else { @@ -255,32 +255,21 @@ impl Asset for ServerNftJsonAsset { continue; }; if file.extension() == Some("js") { - server_output_assets.push(( - base_dir - .get_relative_path_to(file) - .context("failed to compute relative path for server NFT JSON")?, - file.hash_file(hash_salt, HashAlgorithm::Xxh3Hash128Hex) - .await?, - )) + let content = FileSource::new(file.clone()).content(); + let hash = content + .hash(hash_salt, HashAlgorithm::Xxh3Hash128Hex) + .owned() + .await?; + let content = content.await?; + nft_json.add(file.clone(), hash, &content)?; } } } - server_output_assets.sort_unstable(); - // Dedupe as some entries may be duplicates: a file might be referenced multiple times, - // e.g. as a RawModule (from an FS operation) and as an EcmascriptModuleAsset because it - // was required. - server_output_assets.dedup(); - - let (files, file_hashes): (Vec<_>, Vec<_>) = server_output_assets.into_iter().unzip(); - let json = json!({ - "version": 1, - "files": files, - "fileHashes": file_hashes - }); + let json = serde_json::to_string(&nft_json.into_json(None))?; Ok(AssetContent::file( - FileContent::Content(File::from(json.to_string())).cell(), + FileContent::Content(File::from(json)).cell(), )) } } diff --git a/crates/next-api/src/nft_json.rs b/crates/next-api/src/nft_json.rs index 138c540c56aa..d17220900c72 100644 --- a/crates/next-api/src/nft_json.rs +++ b/crates/next-api/src/nft_json.rs @@ -1,23 +1,22 @@ -use anyhow::{Context, Result, bail}; -use either::Either; -use serde_json::json; +use anyhow::{Context, Result}; use tracing::{Instrument, Level, Span}; use turbo_rcstr::RcStr; use turbo_tasks::{ ReadRef, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, ValueToString, Vc, graph::{AdjacencyMap, GraphTraversal, Visit}, - turbofmt, }; use turbo_tasks_fs::{File, FileContent, FileSystem, FileSystemPath, glob::Glob}; use turbo_tasks_hash::HashAlgorithm; use turbopack_core::{ asset::{Asset, AssetContent}, + file_source::FileSource, module::Module, output::{OutputAsset, OutputAssets, OutputAssetsReference}, }; use crate::{ nft::{EndpointTraceResult, tracing_exclude_glob}, + nft_json_builder::NftJsonBuilder, project::Project, }; @@ -82,28 +81,6 @@ impl OutputAsset for NftJsonAsset { } } -fn get_output_specifier( - path_ref: &FileSystemPath, - ident_folder: &FileSystemPath, - ident_folder_in_project_fs: &FileSystemPath, - output_root: &FileSystemPath, - project_root: &FileSystemPath, -) -> Result { - // include assets in the outputs such as referenced chunks - if path_ref.is_inside_ref(output_root) { - return Ok(ident_folder.get_relative_path_to(path_ref).unwrap()); - } - - // include assets in the project root such as images and traced references (externals) - if path_ref.is_inside_ref(project_root) { - return Ok(ident_folder_in_project_fs - .get_relative_path_to(path_ref) - .unwrap()); - } - // This should effectively be unreachable - bail!("NftJsonAsset: cannot handle filepath '{path_ref}'"); -} - #[turbo_tasks::value_impl] impl Asset for NftJsonAsset { #[turbo_tasks::function] @@ -116,20 +93,14 @@ impl Asset for NftJsonAsset { async move { let project_path = this.project.project_path().owned().await?; - let output_root_ref = this.project.output_fs().root().await?; - let project_root_ref = this.project.project_fs().root().await?; let next_config = this.project.next_config(); let hash_salt = next_config.output_hash_salt(); let client_root = this.project.client_fs().root(); let client_root = client_root.owned().await?; - // [project]/ - let project_root_path = this.project.project_root_path().owned().await?; - // Example: [output]/apps/my-website/.next/server/app -- without the `page.js.nft.json` - let ident_folder = self.path().await?.parent(); - // Example: [project]/apps/my-website/.next/server/app -- without the `page.js.nft.json` - let ident_folder_in_project_fs = project_root_path.join(&ident_folder.path)?; + let nft_path = self.path().owned().await?; + let mut nft_json = NftJsonBuilder::new(this.project, &nft_path).await?; let chunk = this.chunk; let entries = this @@ -158,7 +129,7 @@ impl Asset for NftJsonAsset { let traced_files = this.traced_files.await?; let module_data = traced_files.module_data.await?; - let mut result: Vec<(RcStr, _)> = all_assets + let result = all_assets .iter() .filter(|a| **a != chunk) .copied() @@ -171,15 +142,18 @@ impl Asset for NftJsonAsset { .map(AssetOrModule::Module), ) .map(async |referenced| { - let (referenced_chunk_path, hash) = match referenced { - AssetOrModule::Asset(v) => ( - Either::Left(v.path().await?), - Either::Left( - v.content() + let (referenced_chunk_path, hash, content) = match referenced { + AssetOrModule::Asset(v) => { + let content = v.content().to_resolved().await?; + ( + v.path().owned().await?, + content .hash(hash_salt, HashAlgorithm::Xxh3Hash128Hex) + .owned() .await?, - ), - ), + content.await?, + ) + } AssetOrModule::Module(v) => { let ident = module_data .idents @@ -191,82 +165,37 @@ impl Asset for NftJsonAsset { .get(&v) .await? .context("missing hash for module")?; - (Either::Right(ident.path.clone()), Either::Right(hash)) + let source = v.source().await?.context("NFT module has no content")?; + ( + ident.path.clone(), + (**hash).clone(), + source.content().await?, + ) } }; - let referenced_chunk_path = match &referenced_chunk_path { - Either::Left(p) => &**p, - Either::Right(p) => p, - }; if referenced_chunk_path.has_extension(".map") { return Ok(None); } - let specifier = match get_output_specifier( - referenced_chunk_path, - &ident_folder, - &ident_folder_in_project_fs, - &output_root_ref, - &project_root_ref, - ) { - Ok(specifier) => specifier, - Err(err) => { - // ast-grep-ignore: no-context-turbofmt - return Err(err.context( - turbofmt!( - "NftJsonAsset: cannot handle filepath \ - '{referenced_chunk_path}', it is not under the output_root: \ - '{output_root_ref}' or the project_root: '{project_root_ref}'", - ) - .await?, - )); - } - }; - - Ok(Some((specifier, hash))) + Ok(Some((referenced_chunk_path, hash, content))) }) .try_flat_join() .await?; - result.extend( - traced_files - .includes - .iter() - .map(async |file_path| { - let relative_path = ident_folder_in_project_fs - .get_relative_path_to(file_path) - .unwrap(); - Ok(( - relative_path, - Either::Left( - file_path - .hash_file(hash_salt, HashAlgorithm::Xxh3Hash128Hex) - .await?, - ), - )) - }) - .try_join() - .await?, - ); - - // Some of the output assets may have been included multiple times (in multiple chunking - // contexts), or asset contexts. - result.sort_unstable(); - result.dedup(); + for (path, hash, content) in result { + nft_json.add(path, hash, &content)?; + } - let (files, file_hashes): (Vec<_>, Vec<_>) = result - .iter() - .map(|(name, hash)| { - ( - name, - match hash { - Either::Left(v) => &**v, - Either::Right(v) => &**v, - }, - ) - }) - .unzip(); + for file_path in &traced_files.includes { + let content = FileSource::new(file_path.clone()).content(); + let hash = content + .hash(hash_salt, HashAlgorithm::Xxh3Hash128Hex) + .owned() + .await?; + let content = content.await?; + nft_json.add(file_path.clone(), hash, &content)?; + } // We can't just add this into "files" because Next.js sometimes decides to delete // output files such as `.next/server/pages/index.js` if that page was prerendered and // is fully static. An alternative would be to postprocess the nft file so that @@ -276,16 +205,12 @@ impl Asset for NftJsonAsset { let entry_hash = chunk .content() .hash(hash_salt, HashAlgorithm::Xxh3Hash128Hex) + .owned() .await?; - let json = json!({ - "version": 1, - "files": files, - "fileHashes": file_hashes, - "entryHash": entry_hash, - }); + let json = serde_json::to_string(&nft_json.into_json(Some(entry_hash)))?; Ok(AssetContent::file( - FileContent::Content(File::from(json.to_string())).cell(), + FileContent::Content(File::from(json)).cell(), )) } .instrument(span) diff --git a/crates/next-api/src/nft_json_builder.rs b/crates/next-api/src/nft_json_builder.rs new file mode 100644 index 000000000000..eef4f8394b83 --- /dev/null +++ b/crates/next-api/src/nft_json_builder.rs @@ -0,0 +1,246 @@ +use anyhow::{Context, Result, bail}; +use rustc_hash::FxHashMap; +use serde::{Serialize, Serializer, ser::SerializeTuple}; +use turbo_rcstr::RcStr; +use turbo_tasks::ResolvedVc; +use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath}; +use turbo_unix_path::{get_relative_path_to, sys_to_unix}; +use turbopack_core::asset::AssetContent; + +use crate::project::Project; + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +enum AssetLocation { + Base { path: RcStr }, + AdditionalRoot { root_index: usize, path: RcStr }, +} + +impl AssetLocation { + fn parts(&self) -> (Option, &RcStr) { + match self { + Self::Base { path } => (None, path), + Self::AdditionalRoot { root_index, path } => (Some(*root_index), path), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct AssetReference { + location: AssetLocation, + hash: RcStr, + /// Present for symlinks. + symlink_target: Option, +} + +struct AdditionalRootConfig { + name: RcStr, + path: RcStr, +} + +struct RootConfig { + base: RcStr, + additional_root_index: Option, +} + +struct NftSymlink { + file_index: usize, + target: RcStr, + root: Option, +} + +impl Serialize for NftSymlink { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut tuple = serializer.serialize_tuple(if self.root.is_some() { 3 } else { 2 })?; + tuple.serialize_element(&self.file_index)?; + tuple.serialize_element(&self.target)?; + if let Some(root) = self.root { + tuple.serialize_element(&root)?; + } + tuple.end() + } +} + +#[derive(Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct NftFileList { + files: Vec, + file_hashes: Vec, + symlinks: Vec, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct NftAdditionalRoot { + #[serde(flatten)] + file_list: NftFileList, + name: RcStr, + path: RcStr, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct NftJson { + #[serde(flatten)] + file_list: NftFileList, + version: u8, + #[serde(skip_serializing_if = "Option::is_none")] + entry_hash: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + additional_roots: Vec, +} + +pub(crate) struct NftJsonBuilder { + root_configs: FxHashMap>, RootConfig>, + additional_roots: Vec, + /// These eventually get converted to `NftFileList` in `into_json`, but we store them in this + /// intermediate format because it's easier to sort and dedupe. + asset_refs: Vec, +} + +impl NftJsonBuilder { + pub async fn new(project: ResolvedVc, nft_path: &FileSystemPath) -> Result { + let project_ref = project.await?; + let mut root_configs = FxHashMap::default(); + + // Files not listed under `additionalRoots` have paths relative to the nft.json file, which + // lives in the output filesystem. The project and output filesystems share a root path, so + // their paths can be compared directly even when the output is outside the project. + let project_root = project.project_fs().root().owned().await?; + let output_base = nft_path.parent(); + let output_file_system = ResolvedVc::try_downcast_type::(output_base.fs) + .context("NFT path must use a disk filesystem")?; + let output_base_path = output_file_system.await?.to_sys_path_raw(&output_base); + let output_base_path = sys_to_unix( + output_base_path + .to_str() + .context("NFT path must be valid Unicode")?, + ); + root_configs.insert( + project_root.fs, + RootConfig { + base: output_base.path.clone(), + additional_root_index: None, + }, + ); + // The NFT file includes references to bundled JS, which exists in the output filesystem. We + // treat these the same as files in the project filesystem, but use an output-relative base + // path. + root_configs.insert( + output_base.fs, + RootConfig { + base: output_base.path, + additional_root_index: None, + }, + ); + + let mut additional_roots = Vec::with_capacity(project_ref.additional_roots.len()); + for (name, root) in &project_ref.additional_roots { + let file_system = root.file_system.connect().to_resolved().await?; + root_configs.insert( + ResolvedVc::upcast(file_system), + RootConfig { + base: file_system.root().owned().await?.path, + additional_root_index: Some(additional_roots.len()), + }, + ); + let root_path = sys_to_unix(&root.canonical_path); + additional_roots.push(AdditionalRootConfig { + name: name.clone(), + path: get_relative_path_to(&output_base_path, &root_path).into(), + }); + } + + Ok(Self { + root_configs, + additional_roots, + asset_refs: Vec::new(), + }) + } + + fn location_for_path(&self, path: &FileSystemPath) -> Result { + let Some(root) = self.root_configs.get(&path.fs) else { + bail!("NFT cannot handle filepath '{path}' because it is outside every accepted root") + }; + let relative_path = RcStr::from(get_relative_path_to(&root.base, &path.path)); + if let Some(root_index) = root.additional_root_index { + Ok(AssetLocation::AdditionalRoot { + root_index, + path: relative_path, + }) + } else { + Ok(AssetLocation::Base { + path: relative_path, + }) + } + } + + pub fn add(&mut self, path: FileSystemPath, hash: RcStr, content: &AssetContent) -> Result<()> { + let location = self.location_for_path(&path)?; + let symlink_target = match content { + AssetContent::File(_) => None, + AssetContent::Redirect(content) => Some(self.location_for_path(&content.target)?), + }; + self.asset_refs.push(AssetReference { + location, + hash, + symlink_target, + }); + Ok(()) + } + + pub fn into_json(mut self, entry_hash: Option) -> NftJson { + self.asset_refs + .sort_unstable_by(|a, b| a.location.cmp(&b.location)); + self.asset_refs.dedup_by(|a, b| a.location == b.location); + + let mut base = NftFileList::default(); + let mut roots = (0..self.additional_roots.len()) + .map(|_| NftFileList::default()) + .collect::>(); + + for asset in self.asset_refs { + let (source_root, path) = asset.location.parts(); + let list = match source_root { + None => &mut base, + Some(index) => &mut roots[index], + }; + let file_index = list.files.len(); + list.files.push(path.clone()); + list.file_hashes.push(asset.hash); + + if let Some(target) = asset.symlink_target { + let (target_root, target_path) = target.parts(); + let root = if source_root == target_root { + None + } else { + Some(target_root.map(|index| index as isize).unwrap_or(-1)) + }; + list.symlinks.push(NftSymlink { + file_index, + target: target_path.clone(), + root, + }); + } + } + + let additional_roots = self + .additional_roots + .into_iter() + .zip(roots) + .map(|(root, list)| NftAdditionalRoot { + file_list: list, + name: root.name, + path: root.path, + }) + .collect(); + NftJson { + file_list: base, + version: 1, + entry_hash, + additional_roots, + } + } +} diff --git a/crates/next-api/src/project.rs b/crates/next-api/src/project.rs index 9b8089a10c10..66edea07e861 100644 --- a/crates/next-api/src/project.rs +++ b/crates/next-api/src/project.rs @@ -1,4 +1,8 @@ -use std::{path::Path, time::Duration}; +use std::{ + iter, + path::{Path, PathBuf}, + time::Duration, +}; use anyhow::{Context, Result, bail}; use async_trait::async_trait; @@ -39,14 +43,14 @@ use serde::{Deserialize, Serialize}; use tracing::{Instrument, field::Empty}; use turbo_rcstr::{RcStr, rcstr}; use turbo_tasks::{ - Completion, Completions, FxIndexMap, NonLocalValue, OperationValue, OperationVc, ReadRef, - ResolvedVc, State, TransientInstance, TryFlatJoinIterExt, TryJoinIterExt, Vc, - debug::ValueDebugFormat, fxindexmap, trace::TraceRawVcs, + Completion, Completions, FxIndexMap, InvalidationReason, NonLocalValue, OperationValue, + OperationVc, ReadRef, ResolvedVc, State, TransientInstance, TryFlatJoinIterExt, TryJoinIterExt, + Vc, debug::ValueDebugFormat, fxindexmap, trace::TraceRawVcs, }; use turbo_tasks_env::{EnvMap, ProcessEnv}; use turbo_tasks_fs::{ - DiskFileSystem, DiskWatcherConfig, FileContent, FileSystem, FileSystemPath, VirtualFileSystem, - canonicalize_to_rcstr, invalidation, + DiskFileSystem, DiskFileSystemMap, DiskWatcherConfig, FileContent, FileSystem, FileSystemPath, + VirtualFileSystem, invalidation, }; use turbo_unix_path::join_path; use turbopack::{ @@ -66,7 +70,8 @@ use turbopack_core::{ file_source::FileSource, ident::Layer, issue::{ - CollectibleIssuesExt, Issue, IssueExt, IssueFilter, IssueSeverity, IssueStage, StyledString, + CollectibleIssuesExt, Issue, IssueExt, IssueFilter, IssueSeverity, IssueStage, PlainIssue, + StyledString, }, module::{Module, Modules}, module_graph::{ @@ -97,7 +102,9 @@ use turbopack_node::execution_context::ExecutionContext; use turbopack_node::worker_threads_backend; use turbopack_nodejs::{NodeJsChunkingContext, fs::NodeModulesPathMatcher}; +pub use crate::additional_roots::AdditionalRootConfig; use crate::{ + additional_roots::{AdditionalDiskFileSystem, create_additional_root_file_systems}, aggregate_hmr::ServerHmrChunkLists, app::{AppProject, OptionAppProject}, empty::EmptyEndpoint, @@ -301,6 +308,9 @@ pub struct ProjectOptions { /// - It gives us a root to configure the file system watcher with. /// - It ensures the cache is portable when the root path is moved, since every other path is /// relative to it. + /// + /// Symlinks outside of the `root_path` can still be resolved if the targets exist in + /// [`ProjectOptions::additional_roots`]. pub root_path: RcStr, /// A path which contains the app/pages directories, relative to [`Project::project_path`]. @@ -310,6 +320,9 @@ pub struct ProjectOptions { /// The contents of next.config.js, serialized to JSON. pub next_config: RcStr, + /// Additional filesystem roots. Canonicalized during initialization. + pub additional_roots: Vec, + /// A map of environment variables to use when compiling code. pub env: Vec<(RcStr, RcStr)>, @@ -367,18 +380,12 @@ pub struct ProjectOptions { /// Refer to [`ProjectOptions`] for documentation on this struct's fields. #[derive(Default)] pub struct PartialProjectOptions { - pub root_path: Option, - - pub project_path: Option, - pub next_config: Option, pub env: Option>, pub define_env: Option, - pub watch: Option, - pub dev: Option, pub encryption_key: Option, @@ -429,10 +436,23 @@ pub struct Instrumentation { pub edge: ResolvedVc>, } -#[turbo_tasks::value] +#[derive( + Clone, Debug, PartialEq, Eq, NonLocalValue, OperationValue, TraceRawVcs, Encode, Decode, +)] +struct ProjectFileSystemState { + project_file_system: OperationVc, + output_file_system: OperationVc, +} + +#[turbo_tasks::value(evict = "never", eq = "manual", cell = "new")] pub struct ProjectContainer { name: RcStr, options_state: State>, + file_systems_state: State>, + additional_roots_state: State>, + #[turbo_tasks(debug_ignore, trace_ignore)] + #[bincode(skip)] + fs_map_init_lock: tokio::sync::Mutex<()>, versioned_content_map: Option>, } @@ -450,14 +470,152 @@ impl ProjectContainer { None }, options_state: State::new(None), + file_systems_state: State::new(None), + additional_roots_state: State::new(Vec::new()), + fs_map_init_lock: tokio::sync::Mutex::new(()), } .cell()) } } -#[turbo_tasks::function(operation, root)] -fn project_operation(project: ResolvedVc) -> Vc { - project.project() +/// Constructs and activates the initial project container state, including its filesystem +/// watchers. Called by [`ProjectContainer::initialize`]. +async fn prepare_project_container_state( + container_vc: ResolvedVc, + options: ProjectOptions, +) -> Result>> { + let container = container_vc.await?; + // Operations created during initialization may begin running immediately. Keep operations that + // require the complete filesystem map blocked until both filesystem states are populated. + let fs_map_init_guard = container.fs_map_init_lock.lock().await; + + let map = disk_file_system_map_operation(container_vc); + let config_json: serde_json::Value = serde_json::from_str(&options.next_config)?; + + let dist_dir_root = config_json + .get("distDirRoot") + .and_then(|value| value.as_str()) + .unwrap_or(".next"); + + let watcher_config = DiskWatcherConfig { + poll_interval: options.watch.poll_interval, + report_invalidation_reason: true, + ..Default::default() + }; + + let denied_paths = vec![ + RcStr::from( + join_path(&options.project_path, dist_dir_root) + .context("distDirRoot must stay inside the project root")?, + ), + // CPU profiles are written to `.next-profiles/` at the project root (see `--cpu-prof`). + // Deny access to it so the bundler doesn't traverse into the profiling output directory. + RcStr::from(join_path(&options.project_path, DIST_PROFILES_DIR_NAME).unwrap()), + ]; + + let enable_watch = options.watch.enable; + let configured_additional_roots = options.additional_roots.clone(); + let project_root = options.root_path.clone(); + let project_path = options.project_path.clone(); + + // `project_root_path_operation` reads `options_state`, so publish it first. This operation + // cannot depend on `fs_map_init_lock` because we must eagerly resolve `project_fs_op` to + // create `config_path`. + container.options_state.set(Some(options)); + + // Wrap `options.root_path` in an `OperationVc` + // Note: It's important that the identity of this operation is stable, so that we don't end up + // changing the identity of every `FileSystemPath` that depends on its output cell. + let root_path_op = project_root_path_operation(container_vc); + + let project_fs_op = disk_file_system_operation( + PROJECT_FILESYSTEM_NAME, + root_path_op, + denied_paths, + watcher_config, + map, + ); + + let output_fs_op = disk_file_system_operation( + rcstr!("output"), + root_path_op, + Vec::new(), + DiskWatcherConfig::default(), + DiskFileSystemMap::empty(), + ); + + // The filesystem only stores (and does not resolve) the filesystem map. + container + .file_systems_state + .set(Some(ProjectFileSystemState { + project_file_system: project_fs_op, + output_file_system: output_fs_op, + })); + let project_fs_vc = project_fs_op.resolve().strongly_consistent().await?; + let project_fs = project_fs_op.read_strongly_consistent().await?; + + let config_file_name = config_json + .get("configFileName") + .and_then(|value| value.as_str()) + .unwrap_or("next.config.js"); + let config_path = FileSystemPath::new_normalized_unchecked( + ResolvedVc::upcast(project_fs_vc), + RcStr::default(), + ) + .join(&project_path)? + .join(config_file_name)?; + let additional_roots = create_additional_root_file_systems( + container_vc, + configured_additional_roots, + &project_root, + watcher_config, + map, + config_path, + ) + .await?; + + let additional_file_systems = additional_roots + .roots_by_name + .values() + .map(|root| root.file_system) + .collect::>(); + + // This state must be populated before the lazy additional filesystem operations or the + // filesystem map are first resolved. + container + .additional_roots_state + .set(additional_roots.roots_by_name.into_iter().collect()); + drop(fs_map_init_guard); + + // perform complete invalidations of all paths and watcher setup after finalizing the `map` + fn invalidation_reason(path: &Path) -> impl InvalidationReason + Clone + use<> { + invalidation::Initialize { + path: RcStr::from(path.to_string_lossy()), + } + } + if enable_watch { + project_fs.start_watching().await?; + for op in &additional_file_systems { + let fs = op.read_strongly_consistent().await?; + fs.start_watching().await?; + } + } else { + project_fs.invalidate_with_reason(invalidation_reason); + for op in &additional_file_systems { + op.read_strongly_consistent() + .await? + .invalidate_with_reason(invalidation_reason); + } + } + + // we never watch `output_file_system`, but we do invalidate it across restarts, in case some + // other process modified or deleted files. + output_fs_op + .read_strongly_consistent() + .await? + .invalidate_with_reason(invalidation_reason); + + Ok(additional_roots.issues) } /// Activates the lazy dynamic import that `chunk_path` names, returning whether it named one. The @@ -472,13 +630,85 @@ pub async fn activate_lazy_chunk_operation(chunk_path: RcStr) -> Result } #[turbo_tasks::function(operation, root)] -fn project_fs_operation(project: ResolvedVc) -> Vc { - project.project_fs() +pub(crate) fn disk_file_system_operation( + name: RcStr, + canonical_root: OperationVc, + denied_paths: Vec, + mut watcher_config: DiskWatcherConfig, + map: OperationVc, +) -> Vc { + watcher_config.extended_batch_delay_matcher = + Some(ResolvedVc::upcast(NodeModulesPathMatcher.resolved_cell())); + DiskFileSystem::new_with_options( + name, + canonical_root.connect(), + denied_paths, + watcher_config, + map, + ) +} + +#[turbo_tasks::function(operation, root)] +async fn project_root_path_operation(container: ResolvedVc) -> Result> { + let container = container.await?; + let root_path = container + .options_state + .get() + .as_ref() + .context("Unexpected: ProjectContainer is uninitialized")? + .root_path + .clone(); + Ok(Vc::cell(root_path)) +} + +#[turbo_tasks::function(operation, root)] +pub(crate) async fn additional_root_path_operation( + container: ResolvedVc, + key: RcStr, +) -> Result> { + let container = container.await?; + let _guard = container.fs_map_init_lock.lock().await; + let roots = container.additional_roots_state.get(); + let root = roots + .iter() + .find_map(|(name, root)| (name == &key).then_some(root)) + .with_context(|| format!("Unexpected: additional root {key} is missing"))?; + Ok(Vc::cell(root.canonical_path.clone())) } #[turbo_tasks::function(operation, root)] -fn output_fs_operation(project: ResolvedVc) -> Vc { - project.project_fs() +async fn disk_file_system_map_operation( + container: ResolvedVc, +) -> Result> { + let (project_file_system, additional_file_systems) = { + let container = container.await?; + let _guard = container.fs_map_init_lock.lock().await; + let file_systems = container.file_systems_state.get(); + let file_systems = file_systems + .as_ref() + .context("Unexpected: ProjectContainer is uninitialized")?; + ( + file_systems.project_file_system, + container + .additional_roots_state + .get() + .iter() + .map(|(_, root)| root.file_system) + .collect::>(), + ) + }; + let filesystems = iter::once(project_file_system) + .chain(additional_file_systems) + .map(async |operation| { + let fs = operation.connect().to_resolved().await?; + Ok((PathBuf::from(fs.await?.root()), fs)) + }) + .try_join() + .await?; + Ok(filesystems + .into_iter() + .collect::() + .cell()) } enum EnvDiffType { @@ -564,7 +794,10 @@ impl ProjectContainer { /// /// This is an associated function instead of a method because we don't currently implement /// [`std::ops::Receiver`] on [`OperationVc`]. - pub async fn initialize(this_op: OperationVc, options: ProjectOptions) -> Result<()> { + pub async fn initialize( + this_op: OperationVc, + options: ProjectOptions, + ) -> Result>> { let this = this_op.read_strongly_consistent().await?; let span = tracing::info_span!( "initialize project", @@ -582,44 +815,8 @@ impl ProjectContainer { ); let span_clone = span.clone(); async move { - let watch = options.watch; - - if let Some(old_options) = &*this.options_state.get_untracked() { - span.record( - "env_diff", - define_env_diff_report(&old_options.define_env, &options.define_env).as_str(), - ); - } - this.options_state.set(Some(options)); - - #[turbo_tasks::function(operation, root)] - fn project_from_container_operation( - container: OperationVc, - ) -> Vc { - container.connect().project() - } - let project = project_from_container_operation(this_op) - .resolve() - .strongly_consistent() - .await?; - let project_fs = project_fs_operation(project) - .read_strongly_consistent() - .await?; - if watch.enable { - project_fs.start_watching().await?; - } else { - project_fs.invalidate_with_reason(|path| invalidation::Initialize { - // this path is just used for display purposes - path: RcStr::from(path.to_string_lossy()), - }); - } - let output_fs = output_fs_operation(project) - .read_strongly_consistent() - .await?; - output_fs.invalidate_with_reason(|path| invalidation::Initialize { - path: RcStr::from(path.to_string_lossy()), - }); - Ok(()) + let container = this_op.resolve().strongly_consistent().await?; + prepare_project_container_state(container, options).await } .instrument(span_clone) .await @@ -648,12 +845,9 @@ impl ProjectContainer { .read_strongly_consistent() .await?; let PartialProjectOptions { - root_path, - project_path, next_config, env, define_env, - watch, dev, encryption_key, build_id, @@ -664,18 +858,15 @@ impl ProjectContainer { debug_build_paths, } = options; + // Filesystem roots and watcher options are initialization-only. Changing them requires + // restarting the process so their process-local watchers can be recreated safely. let mut new_options = this .options_state - .get() + .get_untracked() .clone() .context("ProjectContainer need to be initialized with initialize()")?; + let old_define_env = new_options.define_env.clone(); - if let Some(root_path) = root_path { - new_options.root_path = canonicalize_to_rcstr(Path::new(&*root_path))?; - } - if let Some(project_path) = project_path { - new_options.project_path = project_path; - } if let Some(next_config) = next_config { new_options.next_config = next_config; } @@ -685,9 +876,6 @@ impl ProjectContainer { if let Some(define_env) = define_env { new_options.define_env = define_env; } - if let Some(watch) = watch { - new_options.watch = watch; - } if let Some(dev) = dev { new_options.dev = dev; } @@ -713,55 +901,11 @@ impl ProjectContainer { new_options.debug_build_paths = Some(debug_build_paths); } - // TODO: Handle mode switch, should prevent mode being switched. - let watch = new_options.watch; - - let project = project_operation(self) - .resolve() - .strongly_consistent() - .await?; - let prev_project_fs = project_fs_operation(project) - .read_strongly_consistent() - .await?; - let prev_output_fs = output_fs_operation(project) - .read_strongly_consistent() - .await?; - - if let Some(old_options) = &*this.options_state.get_untracked() { - span.record( - "env_diff", - define_env_diff_report(&old_options.define_env, &new_options.define_env) - .as_str(), - ); - } + span.record( + "env_diff", + define_env_diff_report(&old_define_env, &new_options.define_env).as_str(), + ); this.options_state.set(Some(new_options)); - let project = project_operation(self) - .resolve() - .strongly_consistent() - .await?; - let project_fs = project_fs_operation(project) - .read_strongly_consistent() - .await?; - let output_fs = output_fs_operation(project) - .read_strongly_consistent() - .await?; - - if !ReadRef::ptr_eq(&prev_project_fs, &project_fs) { - if watch.enable { - // TODO stop watching: prev_project_fs.stop_watching()?; - project_fs.start_watching().await?; - } else { - project_fs.invalidate_with_reason(|path| invalidation::Initialize { - // this path is just used for display purposes - path: RcStr::from(path.to_string_lossy()), - }); - } - } - if !ReadRef::ptr_eq(&prev_output_fs, &output_fs) { - prev_output_fs.invalidate_with_reason(|path| invalidation::Initialize { - path: RcStr::from(path.to_string_lossy()), - }); - } Ok(()) } @@ -792,11 +936,18 @@ impl ProjectContainer { let deferred_entries; let is_persistent_caching_enabled; let server_hmr; + let project_file_system; + let output_file_system; + let additional_roots; { let options = self.options_state.get(); let options = options .as_ref() .context("ProjectContainer need to be initialized with initialize()")?; + let file_systems = self.file_systems_state.get(); + let file_systems = file_systems + .as_ref() + .context("ProjectContainer need to be initialized with initialize()")?; env_map = Vc::cell(options.env.iter().cloned().collect()); define_env = ProjectDefineEnv { client: ResolvedVc::cell(options.define_env.client.iter().cloned().collect()), @@ -820,6 +971,9 @@ impl ProjectContainer { deferred_entries = options.deferred_entries.clone().unwrap_or_default(); is_persistent_caching_enabled = options.is_persistent_caching_enabled; server_hmr = options.server_hmr; + project_file_system = file_systems.project_file_system; + output_file_system = file_systems.output_file_system; + additional_roots = self.additional_roots_state.get().iter().cloned().collect(); } let root_path = ResolvedVc::cell(root_path_str); @@ -851,6 +1005,9 @@ impl ProjectContainer { deferred_entries, is_persistent_caching_enabled, server_hmr, + project_file_system, + output_file_system, + additional_roots, } .cell()) } @@ -954,6 +1111,11 @@ pub struct Project { /// Whether server-side HMR is enabled (disabled with --no-server-fast-refresh). server_hmr: bool, + + project_file_system: OperationVc, + output_file_system: OperationVc, + #[bincode(with = "turbo_bincode::indexmap")] + pub(crate) additional_roots: FxIndexMap, } #[turbo_tasks::value] @@ -1041,37 +1203,7 @@ impl Project { #[turbo_tasks::function] pub fn project_fs(&self) -> Result> { - let denied_path = match join_path(&self.project_path, &self.dist_dir_root) { - Some(dist_dir_root) => dist_dir_root.into(), - None => { - bail!( - "Invalid distDirRoot: {:?}. distDirRoot should not navigate out of the \ - projectPath.", - self.dist_dir_root - ); - } - }; - - // CPU profiles are written to `.next-profiles/` at the project root (see `--cpu-prof`). - // Deny access to it so the bundler doesn't traverse into the profiling output directory. - let denied_profiles_path = join_path(&self.project_path, DIST_PROFILES_DIR_NAME) - .unwrap() - .into(); - - Ok(DiskFileSystem::new_with_options( - PROJECT_FILESYSTEM_NAME, - *self.root_path, - vec![denied_path, denied_profiles_path], - DiskWatcherConfig { - poll_interval: self.watch.poll_interval, - // the dev server reports these to the user - report_invalidation_reason: true, - extended_batch_delay_matcher: Some(ResolvedVc::upcast( - NodeModulesPathMatcher.resolved_cell(), - )), - ..Default::default() - }, - )) + Ok(self.project_file_system.connect()) } #[turbo_tasks::function] @@ -1082,7 +1214,7 @@ impl Project { #[turbo_tasks::function] pub fn output_fs(&self) -> Vc { - DiskFileSystem::new(rcstr!("output"), *self.root_path) + self.output_file_system.connect() } #[turbo_tasks::function] @@ -1431,8 +1563,8 @@ impl Project { let result = GraphEntries::concatenate( endpoint_entries .into_iter() - .chain(std::iter::once(self.client_main_modules().owned().await?)) - .chain(std::iter::once(GraphEntries::new( + .chain(iter::once(self.client_main_modules().owned().await?)) + .chain(iter::once(GraphEntries::new( vec![], // The superset of what any endpoint traces, so that these modules and their // references are part of the graph. Which endpoint actually traces them is diff --git a/crates/next-build-test/src/main.rs b/crates/next-build-test/src/main.rs index 98a2e052e7a9..32fba8ef948a 100644 --- a/crates/next-build-test/src/main.rs +++ b/crates/next-build-test/src/main.rs @@ -171,6 +171,7 @@ fn main() { encryption_key: rcstr!("deadbeef"), env: vec![], next_config: include_str!("../nextConfig.json").into(), + additional_roots: vec![], preview_props: next_api::project::DraftModeOptions { preview_mode_encryption_key: rcstr!("deadbeef"), preview_mode_id: rcstr!("test"), diff --git a/crates/next-core/src/next_import_map.rs b/crates/next-core/src/next_import_map.rs index be5af986e9db..4908c8174e6a 100644 --- a/crates/next-core/src/next_import_map.rs +++ b/crates/next-core/src/next_import_map.rs @@ -1406,7 +1406,7 @@ pub async fn try_get_next_package( context_directory.clone(), ReferenceType::CommonJs(CommonJsReferenceSubType::Undefined), Request::parse(Pattern::Constant(rcstr!("next/package.json"))), - node_cjs_resolve_options(root.clone()), + node_cjs_resolve_options(), ); if let Some(source) = result.await?.first_source() { Ok(Vc::cell(Some(source.ident().await?.path.parent()))) diff --git a/crates/next-core/src/next_server/resolve.rs b/crates/next-core/src/next_server/resolve.rs index b9cfcb576bf5..f1ff05819c86 100644 --- a/crates/next-core/src/next_server/resolve.rs +++ b/crates/next-core/src/next_server/resolve.rs @@ -212,9 +212,9 @@ impl AfterResolvePlugin for ExternalCjsModulesResolvePlugin { let mut request_str = request_str.to_string(); let node_resolve_options = if is_esm { - node_esm_resolve_options(lookup_path.root().owned().await?) + node_esm_resolve_options() } else { - node_cjs_resolve_options(lookup_path.root().owned().await?) + node_cjs_resolve_options() }; let result_from_original_location = loop { let node_resolved_from_original_location = resolve( @@ -274,8 +274,7 @@ impl AfterResolvePlugin for ExternalCjsModulesResolvePlugin { // It would be more efficient to use an CJS external instead of an ESM external, // but we need to verify if that would be correct (as in resolves to the same // file). - let node_resolve_options = - node_cjs_resolve_options(lookup_path.root().owned().await?); + let node_resolve_options = node_cjs_resolve_options(); let node_resolved = resolve( lookup_path.clone(), reference_type.clone(), diff --git a/crates/next-core/src/next_shared/webpack_rules/babel.rs b/crates/next-core/src/next_shared/webpack_rules/babel.rs index b040195a728b..85aab907e968 100644 --- a/crates/next-core/src/next_shared/webpack_rules/babel.rs +++ b/crates/next-core/src/next_shared/webpack_rules/babel.rs @@ -260,7 +260,7 @@ pub async fn detect_react_compiler_target( project_path.clone(), ReferenceType::CommonJs(CommonJsReferenceSubType::Undefined), Request::parse(Pattern::Constant(rcstr!("react/package.json"))), - node_cjs_resolve_options(project_path.root().owned().await?), + node_cjs_resolve_options(), ); let Some(source) = react_pkg_result.await?.first_source() else { @@ -336,7 +336,7 @@ pub async fn resolve_babel_plugin_react_compiler( next_package.clone(), ReferenceType::CommonJs(CommonJsReferenceSubType::Undefined), Request::parse(Pattern::Constant(BABEL_PLUGIN_REACT_COMPILER_PACKAGE_JSON)), - node_cjs_resolve_options(project_path.root().owned().await?), + node_cjs_resolve_options(), ); let Some(source) = babel_plugin_result.await?.first_source() else { BabelPluginReactCompilerResolutionIssue { diff --git a/crates/next-core/src/transform_options.rs b/crates/next-core/src/transform_options.rs index f86d4d1e07db..a71676787b3b 100644 --- a/crates/next-core/src/transform_options.rs +++ b/crates/next-core/src/transform_options.rs @@ -24,7 +24,7 @@ async fn get_typescript_options( let tsconfigs = read_tsconfigs( tsconfig_path.read(), ResolvedVc::upcast(FileSource::new(tsconfig_path.clone()).to_resolved().await?), - node_cjs_resolve_options(tsconfig_path.root().owned().await?), + node_cjs_resolve_options(), ) .await .ok(); @@ -36,7 +36,7 @@ async fn get_typescript_options( Some(FindContextFileResult::Found(path, _)) => read_tsconfigs( path.read(), ResolvedVc::upcast(FileSource::new(path.clone()).to_resolved().await?), - node_cjs_resolve_options(path.root().owned().await?), + node_cjs_resolve_options(), ) .await .ok(), diff --git a/crates/next-napi-bindings/src/next_api/project.rs b/crates/next-napi-bindings/src/next_api/project.rs index 794bfd6cc2c0..7ebbd12a2bb7 100644 --- a/crates/next-napi-bindings/src/next_api/project.rs +++ b/crates/next-napi-bindings/src/next_api/project.rs @@ -29,8 +29,8 @@ use next_api::{ RouteOperation, }, project::{ - DebugBuildPaths, DefineEnv, DraftModeOptions, PartialProjectOptions, Project, - ProjectContainer, ProjectOptions, WatchOptions, activate_lazy_chunk_operation, + AdditionalRootConfig, DebugBuildPaths, DefineEnv, DraftModeOptions, PartialProjectOptions, + Project, ProjectContainer, ProjectOptions, WatchOptions, activate_lazy_chunk_operation, }, project_asset_hashes_manifest::immutable_hashes_manifest_asset_if_enabled, route::{Endpoint, EndpointGroupKey, Route}, @@ -148,6 +148,13 @@ pub struct NapiWatchOptions { pub poll_interval_ms: Option, } +#[napi(object)] +pub struct NapiAdditionalRoot { + pub key: RcStr, + pub path: RcStr, + pub ignore_if_missing: Option, +} + #[napi(object)] pub struct NapiProjectOptions { /// An absolute root path (Unix or Windows path) from which all files must be nested under. @@ -170,6 +177,9 @@ pub struct NapiProjectOptions { /// The contents of next.config.js, serialized to JSON. pub next_config: RcStr, + /// Additional filesystem roots from next.config.js. + pub additional_roots: Vec, + /// A map of environment variables to use when compiling code. pub env: Vec, @@ -226,12 +236,6 @@ pub struct NapiProjectOptions { /// Refer to [`NapiProjectOptions`] for documentation on this struct's fields. #[napi(object)] pub struct NapiPartialProjectOptions { - pub root_path: Option, - - pub project_path: Option, - - pub watch: Option, - pub next_config: Option, pub env: Option>, @@ -289,8 +293,9 @@ impl From for WatchOptions { } } -impl From for ProjectOptions { - fn from(val: NapiProjectOptions) -> Self { +impl NapiProjectOptions { + fn into_project_options(self) -> ProjectOptions { + let val = self; let NapiProjectOptions { root_path, project_path, @@ -298,6 +303,7 @@ impl From for ProjectOptions { dist_dir: _, watch, next_config, + additional_roots, env, define_env, dev, @@ -319,6 +325,14 @@ impl From for ProjectOptions { project_path, watch: watch.into(), next_config, + additional_roots: additional_roots + .into_iter() + .map(|root| AdditionalRootConfig { + key: root.key, + path: root.path, + ignore_if_missing: root.ignore_if_missing.unwrap_or(false), + }) + .collect(), env: env.into_iter().map(|var| (var.name, var.value)).collect(), define_env: define_env.into(), dev, @@ -341,12 +355,10 @@ impl From for ProjectOptions { } } -impl From for PartialProjectOptions { - fn from(val: NapiPartialProjectOptions) -> Self { +impl NapiPartialProjectOptions { + fn into_partial_project_options(self) -> PartialProjectOptions { + let val = self; let NapiPartialProjectOptions { - root_path, - project_path, - watch, next_config, env, define_env, @@ -359,9 +371,6 @@ impl From for PartialProjectOptions { write_routes_hashes_manifest, } = val; PartialProjectOptions { - root_path, - project_path, - watch: watch.map(From::from), next_config, env: env.map(|env| env.into_iter().map(|var| (var.name, var.value)).collect()), define_env: define_env.map(|env| env.into()), @@ -408,13 +417,19 @@ pub struct ProjectInstance { _container_gc_root: GcRoot, } -#[napi(ts_return_type = "Promise<{ __napiType: \"Project\" }>")] +#[napi(object, object_from_js = false)] +pub struct NapiProject { + #[napi(ts_type = "{ __napiType: \"Project\" }")] + pub project: External, +} + +#[napi(ts_return_type = "Promise>")] pub fn project_new<'env>( env: &'env Env, mut options: NapiProjectOptions, turbo_engine_options: NapiTurboEngineOptions, napi_callbacks: NapiNextTurbopackCallbacksJsObject, -) -> napi::Result>> { +) -> napi::Result>> { let napi_callbacks = NapiNextTurbopackCallbacks::from_js(env, napi_callbacks)?; let (exit, exit_receiver) = ExitHandler::new_receiver(); @@ -597,16 +612,17 @@ pub fn project_new<'env>( }); } - let options = ProjectOptions::from(options); + let options = options.into_project_options(); let is_dev = options.dev; let root_path = options.root_path.clone(); - let (container, container_op) = turbo_tasks + let (container, container_op, initialization_issues) = turbo_tasks .run(async move { let container_op = ProjectContainer::new_operation(rcstr!("next.js"), is_dev); - ProjectContainer::initialize(container_op, options).await?; + let initialization_issues = + ProjectContainer::initialize(container_op, options).await?; let container = container_op.resolve().strongly_consistent().await?; // Return the operation itself so we can pin it below - Ok((container, container_op)) + Ok((container, container_op, initialization_issues)) }) .or_else(|e| turbopack_ctx.throw_turbopack_internal_result(&e.into())) .await?; @@ -647,12 +663,20 @@ pub fn project_new<'env>( }); } - Ok(External::new(ProjectInstance { - turbopack_ctx, - container, - exit_receiver: Mutex::new(Some(exit_receiver)), - _container_gc_root: container_gc_root, - })) + Ok(TurbopackResult { + result: NapiProject { + project: External::new(ProjectInstance { + turbopack_ctx, + container, + exit_receiver: Mutex::new(Some(exit_receiver)), + _container_gc_root: container_gc_root, + }), + }, + issues: initialization_issues + .iter() + .map(|issue| NapiIssue::from(&**issue)) + .collect(), + }) } .instrument(tracing::info_span!("create project")), ) @@ -741,7 +765,7 @@ pub async fn project_update( options: NapiPartialProjectOptions, ) -> napi::Result<()> { let ctx = &project.turbopack_ctx; - let options = options.into(); + let options = options.into_partial_project_options(); let container = project.container; ctx.turbo_tasks() .run(async move { container.update(options).await }) diff --git a/docs/01-app/03-api-reference/05-config/01-next-config-js/output.mdx b/docs/01-app/03-api-reference/05-config/01-next-config-js/output.mdx index 5e06e34a98af..95946d831c15 100644 --- a/docs/01-app/03-api-reference/05-config/01-next-config-js/output.mdx +++ b/docs/01-app/03-api-reference/05-config/01-next-config-js/output.mdx @@ -19,6 +19,17 @@ Next.js' production server is also traced for its needed files and output at `.n To leverage the `.nft.json` files emitted to the `.next` output directory, you can read the list of files in each trace that are relative to the `.nft.json` file and then copy them to your deployment location. +### Turbopack NFT extensions + +Turbopack `.nft.json` files include `fileHashes` and `symlinks` extensions: + +- `fileHashes` is a parallel array with the same length and ordering as `files`. For example, `fileHashes[0]` is the hash for `files[0]`. A consumer can use the hash as a content identifier for caching or deduplicating traced assets. +- `symlinks` maps indexes in the sibling `files` array to their immediate targets. When `symlinks` is present, files without a corresponding entry can be assumed to be normal files or directories. + +Traces that use [`experimental.turbopackAdditionalRoots`](/docs/app/api-reference/config/next-config-js/turbopack#additional-roots-experimental) also include the `additionalRoots` extension: + +- `additionalRoots` is an ordered list of named source roots. Each root has its own `files`, `fileHashes`, and `symlinks` lists. Its `path` is relative to the directory containing the `.nft.json` file and identifies the source location on the build machine. `additionalRoots` names match those provided in `next.config.js` and are guaranteed to be valid directory names on most operating systems and filesystems. + ## Automatically Copying Traced Files Next.js can automatically create a `standalone` folder that copies only the necessary files for a production deployment including select files in `node_modules`. @@ -33,6 +44,8 @@ module.exports = { This will create a folder at `.next/standalone` which can then be deployed on its own without installing `node_modules`. +On Windows, creating directory symlinks [can require elevated privileges or Developer Mode](https://blogs.windows.com/windowsdeveloper/2016/12/02/symlinks-windows-10/). If standalone generation must fall back to a junction point, Windows stores an absolute target and that particular output is not relocatable. + Additionally, a minimal `server.js` file is also output which can be used instead of `next start`. This minimal server does not copy the `public` or `.next/static` folders by default as these should ideally be handled by a CDN instead, although these folders can be copied to the `standalone/public` and `standalone/.next/static` folders manually, after which `server.js` file will serve these automatically. To copy these manually, you can use the `cp` command-line tool after you `next build`: diff --git a/docs/01-app/03-api-reference/05-config/01-next-config-js/turbopack.mdx b/docs/01-app/03-api-reference/05-config/01-next-config-js/turbopack.mdx index d7b51b26e127..5d736a84f5e3 100644 --- a/docs/01-app/03-api-reference/05-config/01-next-config-js/turbopack.mdx +++ b/docs/01-app/03-api-reference/05-config/01-next-config-js/turbopack.mdx @@ -91,7 +91,7 @@ If you have a loader that is critically dependent upon one of these features ple Turbopack uses the root directory to resolve modules. Files outside of the project root are not resolved. -The reason files are not resolved outside of the project root is to improve cache validation, reduce filesystem watching overhead, and reduce the number of resolving steps needed. +Files are not resolved outside of the project root is to improve cache validation, reduce filesystem watching overhead, and reduce the number of [resolving steps](https://nodejs.org/api/esm.html#resolution-algorithm) needed. Next.js automatically detects the root directory of your project. It does so by looking for one of these files: @@ -112,9 +112,44 @@ module.exports = { } ``` -To resolve files from linked dependencies outside the project root (via `npm link`, `yarn link`, `pnpm link`, etc.), you must configure the `turbopack.root` to the parent directory of both the project and the linked dependencies. +To resolve files from linked dependencies outside the project root (via `npm link`, `yarn link`, `pnpm link`, etc.), you must either configure the `turbopack.root` to the parent directory of both the project and the linked dependencies or configure an additional root. -While this expands the scope of filesystem watching, it's typically only necessary during development when actively working on linked packages. +Do not extend the root directory too broadly (e.g. to your home directory). Turbopack may watch all files inside of roots, and configuring too broad of a root directory can cause significant performance problems. + +### Additional roots (experimental) + +`experimental.turbopackAdditionalRoots` allows Turbopack to follow symbolic links whose targets are outside `turbopack.root`. It is useful for linked packages that live outside the project. + +Additional roots are read-only. Additional roots can only be crossed at symlink boundaries. Direct relative imports that escape a root are unsupported. + +```js filename="next.config.js" +const path = require('path') + +module.exports = { + experimental: { + turbopackAdditionalRoots: { + linkedPackages: { path: path.join(__dirname, '../packages') }, + }, + }, +} +``` + +Each key identifies one root. `path` can be absolute or relative to the directory where you run Next.js. + +If a root path is not found when `next build` or `next dev` starts, a warning will be issued, unless `ignoreIfMissing` is set to `true` (defaults to false). + +To ensure portability of your configuration across a variety of platforms, including Windows, a root name must meet all of these requirements: + +- It contains 1 to 40 characters. +- Each character must be an ASCII letter (`a-z` or `A-Z`), a digit (`0-9`), an underscore (`_`), or a hyphen (`-`). +- It is not a Windows device name. +- It does not duplicate an earlier name under ASCII case-insensitive comparison. + +To avoid ambiguous references to files, roots may not contain overlapping paths. For example, configuring both `foo/` and `foo/bar/` as roots would produce a warning, and the last-configured value will be ignored. + +Additional roots changes the [`trace format`](/docs/app/api-reference/config/next-config-js/output#turbopack-nft-extensions) and requires support from the [build adapter](/docs/app/api-reference/adapters). + +When using [`output: "standalone"`](/docs/app/api-reference/config/next-config-js/output#automatically-copying-traced-files), unbundled files stored in these additional roots will be copied to a subdirectory of the output directory containing the configured name of the root. ### Configuring webpack loaders @@ -395,6 +430,7 @@ The option automatically adds a polyfill for debug IDs to the JavaScript bundle | Version | Changes | | -------- | ---------------------------------------------------- | +| `16.4.0` | `experimental.turbopackAdditionalRoots` was added | | `16.2.0` | `turbopackLoader` import attributes were added. | | `16.2.0` | `turbopack.rules.*.type` was added. | | `16.2.0` | `turbopack.rules.*.condition.contentType` was added. | diff --git a/docs/01-app/03-api-reference/08-turbopack.mdx b/docs/01-app/03-api-reference/08-turbopack.mdx index 805d96f94812..26667fcf2f86 100644 --- a/docs/01-app/03-api-reference/08-turbopack.mdx +++ b/docs/01-app/03-api-reference/08-turbopack.mdx @@ -344,9 +344,9 @@ There are a number of non-trivial behavior differences between webpack and Turbo Turbopack uses the root directory to resolve modules. Files outside of the project root are not resolved. -For example, when linking dependencies outside the project root (via `npm link`, `yarn link`, `pnpm link`, etc.), those linked files will not be resolved by default. To resolve these files, you must configure the root option to the parent directory of both the project and the linked dependencies. +For example, linked dependencies outside the project root (via `npm link`, `yarn link`, `pnpm link`, etc.) are not resolved by default. Configure [`experimental.turbopackAdditionalRoots`](/docs/app/api-reference/config/next-config-js/turbopack#additional-roots-experimental) for symlinked dependencies. -You can configure the filesystem root using [turbopack.root](/docs/app/api-reference/config/next-config-js/turbopack#root-directory) option in `next.config.js`. +Use [`turbopack.root`](/docs/app/api-reference/config/next-config-js/turbopack#root-directory) when your application directly imports files outside the detected root rather than reaching them through symlinks. ### CSS Module Ordering diff --git a/lerna.json b/lerna.json index 074823e7e8ea..8bc0786cf0ce 100644 --- a/lerna.json +++ b/lerna.json @@ -15,5 +15,5 @@ "registry": "https://registry.npmjs.org/" } }, - "version": "16.4.0-canary.35" + "version": "16.4.0-canary.36" } \ No newline at end of file diff --git a/packages/create-next-app/package.json b/packages/create-next-app/package.json index 20d15b0d051c..7bfa453ba619 100644 --- a/packages/create-next-app/package.json +++ b/packages/create-next-app/package.json @@ -1,6 +1,6 @@ { "name": "create-next-app", - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "keywords": [ "react", "next", diff --git a/packages/devlow-bench/package.json b/packages/devlow-bench/package.json index 78194c3a024a..9887678b22f6 100644 --- a/packages/devlow-bench/package.json +++ b/packages/devlow-bench/package.json @@ -1,6 +1,6 @@ { "name": "@vercel/devlow-bench", - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "description": "Benchmarking tool for the developer workflow", "repository": { "type": "git", diff --git a/packages/eslint-config-next/package.json b/packages/eslint-config-next/package.json index de48b6a8b79c..d26a92da718e 100644 --- a/packages/eslint-config-next/package.json +++ b/packages/eslint-config-next/package.json @@ -1,6 +1,6 @@ { "name": "eslint-config-next", - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "description": "ESLint configuration used by Next.js.", "license": "MIT", "repository": { @@ -12,7 +12,7 @@ "dist" ], "dependencies": { - "@next/eslint-plugin-next": "16.4.0-canary.35", + "@next/eslint-plugin-next": "16.4.0-canary.36", "eslint-import-resolver-node": "^0.3.6", "eslint-import-resolver-typescript": "^3.5.2", "eslint-plugin-import": "^2.32.0", diff --git a/packages/eslint-plugin-internal/package.json b/packages/eslint-plugin-internal/package.json index 3609a7a590bc..2ff0035e509e 100644 --- a/packages/eslint-plugin-internal/package.json +++ b/packages/eslint-plugin-internal/package.json @@ -1,7 +1,7 @@ { "name": "@next/eslint-plugin-internal", "private": true, - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "description": "ESLint plugin for working on Next.js.", "exports": { ".": "./src/eslint-plugin-internal.js" diff --git a/packages/eslint-plugin-next/package.json b/packages/eslint-plugin-next/package.json index fa29447d1b62..07cb6d70deb2 100644 --- a/packages/eslint-plugin-next/package.json +++ b/packages/eslint-plugin-next/package.json @@ -1,6 +1,6 @@ { "name": "@next/eslint-plugin-next", - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "description": "ESLint plugin for Next.js.", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/font/package.json b/packages/font/package.json index 4bd5b423d939..cdcc2a61fd77 100644 --- a/packages/font/package.json +++ b/packages/font/package.json @@ -1,7 +1,7 @@ { "name": "@next/font", "private": true, - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "repository": { "url": "vercel/next.js", "directory": "packages/font" diff --git a/packages/next-bundle-analyzer/package.json b/packages/next-bundle-analyzer/package.json index 086eb303778f..7b1425212657 100644 --- a/packages/next-bundle-analyzer/package.json +++ b/packages/next-bundle-analyzer/package.json @@ -1,6 +1,6 @@ { "name": "@next/bundle-analyzer", - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "main": "index.js", "types": "index.d.ts", "license": "MIT", diff --git a/packages/next-codemod/package.json b/packages/next-codemod/package.json index a988b04b62d0..19b326c945ca 100644 --- a/packages/next-codemod/package.json +++ b/packages/next-codemod/package.json @@ -1,6 +1,6 @@ { "name": "@next/codemod", - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "license": "MIT", "repository": { "type": "git", diff --git a/packages/next-env/package.json b/packages/next-env/package.json index d9a9030eb8af..58172a0441fb 100644 --- a/packages/next-env/package.json +++ b/packages/next-env/package.json @@ -1,6 +1,6 @@ { "name": "@next/env", - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "keywords": [ "react", "next", diff --git a/packages/next-mdx/package.json b/packages/next-mdx/package.json index 81eeb1f19d23..d23f69c271bc 100644 --- a/packages/next-mdx/package.json +++ b/packages/next-mdx/package.json @@ -1,6 +1,6 @@ { "name": "@next/mdx", - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "main": "index.js", "license": "MIT", "repository": { diff --git a/packages/next-playwright/package.json b/packages/next-playwright/package.json index 0816c82eead2..80872e4ebe68 100644 --- a/packages/next-playwright/package.json +++ b/packages/next-playwright/package.json @@ -1,6 +1,6 @@ { "name": "@next/playwright", - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "repository": { "url": "vercel/next.js", "directory": "packages/next-playwright" diff --git a/packages/next-plugin-storybook/package.json b/packages/next-plugin-storybook/package.json index bb11fc73dc7e..bc954d4118fc 100644 --- a/packages/next-plugin-storybook/package.json +++ b/packages/next-plugin-storybook/package.json @@ -1,6 +1,6 @@ { "name": "@next/plugin-storybook", - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "repository": { "url": "vercel/next.js", "directory": "packages/next-plugin-storybook" diff --git a/packages/next-polyfill-module/package.json b/packages/next-polyfill-module/package.json index 2fbff6b6d26e..a3323bb3bb0a 100644 --- a/packages/next-polyfill-module/package.json +++ b/packages/next-polyfill-module/package.json @@ -1,6 +1,6 @@ { "name": "@next/polyfill-module", - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "description": "A standard library polyfill for ES Modules supporting browsers (Edge 16+, Firefox 60+, Chrome 61+, Safari 10.1+)", "main": "dist/polyfill-module.js", "license": "MIT", diff --git a/packages/next-polyfill-nomodule/package.json b/packages/next-polyfill-nomodule/package.json index 27f9d2805009..2211265dea85 100644 --- a/packages/next-polyfill-nomodule/package.json +++ b/packages/next-polyfill-nomodule/package.json @@ -1,6 +1,6 @@ { "name": "@next/polyfill-nomodule", - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "description": "A polyfill for non-dead, nomodule browsers.", "main": "dist/polyfill-nomodule.js", "license": "MIT", diff --git a/packages/next-routing/package.json b/packages/next-routing/package.json index 6d3fea061c05..6b6d7e8474d3 100644 --- a/packages/next-routing/package.json +++ b/packages/next-routing/package.json @@ -1,6 +1,6 @@ { "name": "@next/routing", - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "keywords": [ "react", "next", diff --git a/packages/next-rspack/package.json b/packages/next-rspack/package.json index adfd11a80020..270e95d6a8e9 100644 --- a/packages/next-rspack/package.json +++ b/packages/next-rspack/package.json @@ -1,6 +1,6 @@ { "name": "next-rspack", - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "repository": { "url": "vercel/next.js", "directory": "packages/next-rspack" diff --git a/packages/next-swc/package.json b/packages/next-swc/package.json index 4ae347fd2afa..87abeee992d4 100644 --- a/packages/next-swc/package.json +++ b/packages/next-swc/package.json @@ -1,6 +1,6 @@ { "name": "@next/swc", - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "private": true, "files": [ "native/" diff --git a/packages/next/package.json b/packages/next/package.json index 441b223ca6c7..875820ef9f83 100644 --- a/packages/next/package.json +++ b/packages/next/package.json @@ -1,6 +1,6 @@ { "name": "next", - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "description": "The React Framework", "main": "./dist/server/next.js", "license": "MIT", @@ -100,7 +100,7 @@ ] }, "dependencies": { - "@next/env": "16.4.0-canary.35", + "@next/env": "16.4.0-canary.36", "@swc/helpers": "0.5.23", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", @@ -164,11 +164,11 @@ "@modelcontextprotocol/sdk": "1.18.1", "@mswjs/interceptors": "0.42.5", "@napi-rs/triples": "1.2.0", - "@next/font": "16.4.0-canary.35", - "@next/polyfill-module": "16.4.0-canary.35", - "@next/polyfill-nomodule": "16.4.0-canary.35", - "@next/react-refresh-utils": "16.4.0-canary.35", - "@next/swc": "16.4.0-canary.35", + "@next/font": "16.4.0-canary.36", + "@next/polyfill-module": "16.4.0-canary.36", + "@next/polyfill-nomodule": "16.4.0-canary.36", + "@next/react-refresh-utils": "16.4.0-canary.36", + "@next/swc": "16.4.0-canary.36", "@opentelemetry/api": "1.6.0", "@playwright/test": "1.61.0", "@rspack/core": "1.6.7", diff --git a/packages/next/src/build/adapter/build-complete.ts b/packages/next/src/build/adapter/build-complete.ts index ec5af1cdd40a..8be2b3b04e47 100644 --- a/packages/next/src/build/adapter/build-complete.ts +++ b/packages/next/src/build/adapter/build-complete.ts @@ -62,6 +62,7 @@ import { Bundler } from '../../lib/bundler' import { resolveCacheHandlerPathToFilesystem } from '../../lib/format-dynamic-import-path' import { InvariantError } from '../../shared/lib/invariant-error' import type { __ApiPreviewProps } from '../../server/api-utils' +import { mapNftFileEntries, type NftJson } from '../nft' interface SharedRouteFields { /** @@ -2765,26 +2766,17 @@ async function loadNFT( repoRoot: string, traceFilePath: string ): Promise<{ entryHash?: string }> { - const { files, fileHashes, entryHash } = (await JSON.parse( - await fs.readFile(traceFilePath, 'utf8') - )) as { - files: string[] - fileHashes?: string[] - entryHash?: string - } - - const traceFileDir = path.dirname(traceFilePath) - for (let i = 0; i < files.length; i++) { - const relativeFile = files[i] - const contentHash = fileHashes?.[i] - const tracedFilePath = path.join(traceFileDir, relativeFile) - const fileOutputPath = path.relative(repoRoot, tracedFilePath) - assets[fileOutputPath] = tracedFilePath - if (contentHash) { - assetsHashes[fileOutputPath] = contentHash + const nft = JSON.parse(await fs.readFile(traceFilePath, 'utf8')) as NftJson + + // This call site only records source locations and hashes, so it does not need + // the mapped symlink targets. + for (const entry of mapNftFileEntries(nft, traceFilePath, repoRoot)) { + assets[entry.destination] = entry.source + if (entry.hash) { + assetsHashes[entry.destination] = entry.hash } } - return { entryHash } + return { entryHash: nft.entryHash } } async function hashFile(salt: string, filePath: string): Promise { diff --git a/packages/next/src/build/nft.ts b/packages/next/src/build/nft.ts new file mode 100644 index 000000000000..ebdb862985bd --- /dev/null +++ b/packages/next/src/build/nft.ts @@ -0,0 +1,274 @@ +import path from 'path' + +/** Serialized file-list fields shared by an NFT and each additional root. */ +export interface NftFileList { + /** + * File paths relative to the directory containing the `.nft.json` file, or + * the current root when inside `NftAdditionalRoot`. + * + * When using webpack, these paths may exist outside the tracing root. The + * [`@vercel/next` package][@vercel/next] ignores these paths. + * + * When using Turbopack, these paths are guaranteed to exist within the + * `turbopack.root` specified or inferred from `next.config.js`. + * + * [@vercel/next]: https://github.com/vercel/vercel/blob/%40vercel/next%404.20.5/packages/next/src/server-build.ts#L1022-L1026 + */ + files: string[] + /** + * Turbopack extension: A parallel array to `files` with the same indices and + * length containing file content hashes. For symlinks, this stores the hash + * of the target path. + */ + fileHashes?: string[] + /** + * Turbopack extension: Explicit symlink mapping information. + * + * When present, files not listed in `symlinks` are not symlinks. An empty + * array means there are no symlinks. When omitted, the NFT consumer must call + * `readlink` on every file to identify symlinks and their targets. Entries + * are sorted by file index. + * + * This field is always included when `NftJson` includes `additionalRoots`, + * and on every `NftAdditionalRoot`. + */ + symlinks?: NftSymlink[] +} + +/** Serialized contents of a `.nft.json` trace file. */ +export interface NftJson extends NftFileList { + version: 1 + /** + * Turbopack extension: A hash of the entrypoint that refers to these traced + * files. This hash depends only on the entrypoint's contents, not on its + * traced dependencies. + */ + entryHash?: string + /** + * Turbopack extension: Paths stored with different base paths, typically + * outside the tracing root. + */ + additionalRoots?: NftAdditionalRoot[] +} + +/** Turbopack extension: Paths stored with a different base path. */ +export interface NftAdditionalRoot extends NftFileList { + /** + * Stable unique identifier provided in `next.config.js`. This can be used to + * generate the output path where these files are copied, such as + * `next_additional_roots/${name}`. + * + * This uses a character set that is valid on most filesystems, and identifiers + * are guaranteed not to overlap on case-insensitive filesystems. + */ + name: string + /** + * Source root that the paths in `files` are relative to, represented relative + * to the directory containing the `.nft.json` file. + */ + path: string + /** Always specified on `NftAdditionalRoot`. */ + symlinks: NftSymlink[] +} + +/** + * Turbopack extension: Information about a symlink, including which additional + * root it maps to. Symlinks that do not cross root boundaries (the common case) + * omit the index into `additionalRoots`. + * + * Transforming raw symlink targets into root-relative paths can be complicated; + * including this information ensures that the NFT consumer gets the same result + * expected by Turbopack's tracing system. + * + * Because the link target type is unspecified, on Windows the consumer must call + * `stat` to determine whether a link target is a directory or file. + */ +export type NftSymlink = + | [ + /** Index in `files` that refers to a symlink. */ + number, + /** + * Link target path. In `NftJson`, this is relative to the directory + * containing the `.nft.json` file. In `NftAdditionalRoot`, this is + * relative to the current root. + */ + string, + ] + | [ + /** Index in `files` that refers to a symlink. */ + number, + /** Link target path relative to the specified root. */ + string, + /** + * Index into `additionalRoots`, or `-1` when the target path is relative + * to the `.nft.json` file's directory. + * + * This element is omitted when the target is relative to the same root as + * the symlink itself. + */ + number, + ] + +export interface MappedNftFileEntry { + source: string + destination: string + hash?: string + /** + * If the `NftJson` has a top-level `symlink` field, it can be assumed that an + * `undefined` value here means that the file is not a symlink. + */ + symlinkTarget?: string +} + +function invalid(message: string): never { + throw new Error(`Invalid NFT metadata: ${message}`) +} + +function isRelativePathInside(relative: string): boolean { + return ( + relative === '' || + (!path.isAbsolute(relative) && + relative !== '..' && + !relative.startsWith(`..${path.sep}`)) + ) +} + +function relativePathIfInside( + root: string, + candidate: string +): string | undefined { + const relative = path.relative(root, candidate) + return isRelativePathInside(relative) ? relative : undefined +} + +function mapBasePath( + traceFileDirectory: string, + baseRoot: string, + relativePath: string +): { source: string; destination: string } { + const source = path.resolve(traceFileDirectory, relativePath) + return { source, destination: path.relative(baseRoot, source) } +} + +function mapBasePathInsideRoot( + traceFileDirectory: string, + baseRoot: string, + relativePath: string +): { source: string; destination: string } { + const mapped = mapBasePath(traceFileDirectory, baseRoot, relativePath) + if (!isRelativePathInside(mapped.destination)) { + invalid(`path ${JSON.stringify(relativePath)} escapes the base root`) + } + return mapped +} + +function mapAdditionalRootPath( + traceFileDirectory: string, + root: NftAdditionalRoot, + relativePath: string +): { source: string; destination: string } { + const rootPath = path.resolve(traceFileDirectory, root.path) + const source = path.resolve(rootPath, relativePath) + if (relativePathIfInside(rootPath, source) === undefined) { + invalid( + `path ${JSON.stringify(relativePath)} escapes additional root ${root.name}` + ) + } + return { + source, + destination: path.join('next_additional_roots', root.name, relativePath), + } +} + +export function mapNftFileEntries( + nft: NftJson, + traceFilePath: string, + baseRoot: string, + options?: { + skipBaseRootEscapes?: boolean + onBaseRootEscape?: (source: string) => void + } +): MappedNftFileEntry[] { + const traceFileDirectory = path.dirname(traceFilePath) + const roots = nft.additionalRoots ?? [] + const result: MappedNftFileEntry[] = [] + + const mapList = (list: NftFileList, currentRootIndex: number) => { + // The list of symlinks is always in sorted order (by file index) + const { files, fileHashes } = list + const symlinks = list.symlinks ?? [] + let symlinkCursor = 0 + let nextSymlink = symlinks[symlinkCursor] + + for (let fileIndex = 0; fileIndex < files.length; fileIndex++) { + const file = files[fileIndex] + + let symlink: NftSymlink | undefined + if (nextSymlink?.[0] === fileIndex) { + symlink = nextSymlink + nextSymlink = symlinks[++symlinkCursor] + } + + // a currentRootIndex of -1 denotes a path relative to the *.nft.json file + // (i.e. not an additional root) + const mapped = + currentRootIndex === -1 + ? mapBasePath(traceFileDirectory, baseRoot, file) + : mapAdditionalRootPath( + traceFileDirectory, + roots[currentRootIndex], + file + ) + if ( + currentRootIndex === -1 && + options?.skipBaseRootEscapes && + !isRelativePathInside(mapped.destination) + ) { + options.onBaseRootEscape?.(mapped.source) + continue + } + + let symlinkTarget: string | undefined + if (symlink !== undefined) { + const [, target, rootIndex] = symlink + const targetRootIndex = rootIndex ?? currentRootIndex + symlinkTarget = + targetRootIndex === -1 + ? (options?.skipBaseRootEscapes + ? mapBasePathInsideRoot(traceFileDirectory, baseRoot, target) + : mapBasePath(traceFileDirectory, baseRoot, target) + ).destination + : mapAdditionalRootPath( + traceFileDirectory, + roots[targetRootIndex], + target + ).destination + } + + result.push({ + ...mapped, + hash: fileHashes?.[fileIndex], + symlinkTarget, + }) + } + } + + mapList(nft, -1) + for (let rootIndex = 0; rootIndex < roots.length; rootIndex++) { + mapList(roots[rootIndex], rootIndex) + } + return result +} + +export function resolveNftOutputPath( + outputRoot: string, + destination: string +): string { + const outputPath = path.resolve(outputRoot, destination) + if (relativePathIfInside(outputRoot, outputPath) === undefined) { + invalid( + `output path ${JSON.stringify(destination)} escapes the deployment root` + ) + } + return outputPath +} diff --git a/packages/next/src/build/print-build-errors.ts b/packages/next/src/build/print-build-errors.ts index 104f9dcd1e77..cf29eca3ab7e 100644 --- a/packages/next/src/build/print-build-errors.ts +++ b/packages/next/src/build/print-build-errors.ts @@ -6,9 +6,9 @@ export function formatWarningsHeader(count: number): string { } /** - * Processes and reports build issues from Turbopack entrypoints. + * Processes and reports build issues from Turbopack's N-API functions. * - * @param entrypoints - The result object containing build issues to process. + * @param result - The result object containing build issues to process. * @param isDev - A flag indicating if the build is running in development mode. * @param opts.deferWarnings - When true, warnings are returned instead of * printed so the caller can print them later. @@ -17,7 +17,7 @@ export function formatWarningsHeader(count: number): string { * 'fatal' and 'bug' issues. In production mode, we also throw on 'error' issues. */ export function printBuildErrors( - entrypoints: TurbopackResult, + result: TurbopackResult, isDev: boolean, opts?: { deferWarnings?: boolean } ): { warnings: string[] } { @@ -33,7 +33,7 @@ export function printBuildErrors( const seenErrors = new Set() const seenWarnings = new Set() - for (const issue of entrypoints.issues) { + for (const issue of result.issues) { // We only want to completely shut down the server if (issue.severity === 'fatal' || issue.severity === 'bug') { const formatted = formatIssue(issue) diff --git a/packages/next/src/build/swc/generated-native.d.ts b/packages/next/src/build/swc/generated-native.d.ts index bfe06b106fb4..559eef46dceb 100644 --- a/packages/next/src/build/swc/generated-native.d.ts +++ b/packages/next/src/build/swc/generated-native.d.ts @@ -203,6 +203,12 @@ export interface NapiAdditionalIssueSource { codeFrame?: string } +export interface NapiAdditionalRoot { + key: RcStr + path: RcStr + ignoreIfMissing?: boolean +} + export interface NapiAssetPath { path: RcStr contentHash: RcStr @@ -361,9 +367,6 @@ export interface NapiOptionEnvVar { * Refer to [`NapiProjectOptions`] for documentation on this struct's fields. */ export interface NapiPartialProjectOptions { - rootPath?: RcStr - projectPath?: RcStr - watch?: NapiWatchOptions nextConfig?: RcStr env?: Array defineEnv?: NapiDefineEnv @@ -376,6 +379,10 @@ export interface NapiPartialProjectOptions { noMangling?: boolean } +export interface NapiProject { + project: { __napiType: 'Project' } +} + export interface NapiProjectOptions { /** * An absolute root path (Unix or Windows path) from which all files must be nested under. @@ -398,6 +405,8 @@ export interface NapiProjectOptions { watch: NapiWatchOptions /** The contents of next.config.js, serialized to JSON. */ nextConfig: RcStr + /** Additional filesystem roots from next.config.js. */ + additionalRoots: Array /** A map of environment variables to use when compiling code. */ env: Array /** @@ -509,10 +518,6 @@ export interface NapiTurboEngineOptions { gc?: NapiTurbopackGcOptions } -/** - * Tuning for Turbopack's reference-counting GC, mirroring the - * `experimental.turbopackGc` config option. - */ export interface NapiTurbopackGcOptions { /** How long a GC pass runs before it will honour an interrupt, in milliseconds. */ minProgressMs?: number @@ -664,7 +669,7 @@ export declare function projectNew( options: NapiProjectOptions, turboEngineOptions: NapiTurboEngineOptions, napiCallbacks: NapiNextTurbopackCallbacksJsObject -): Promise<{ __napiType: 'Project' }> +): Promise> /** * Runs exit handlers for the project registered using the [`ExitHandler`] API. diff --git a/packages/next/src/build/swc/index.ts b/packages/next/src/build/swc/index.ts index bcd510cefaad..a8a3884f73f5 100644 --- a/packages/next/src/build/swc/index.ts +++ b/packages/next/src/build/swc/index.ts @@ -653,8 +653,12 @@ function bindingToApi( async function rustifyProjectOptions( options: ProjectOptions ): Promise { + const additionalRoots = Object.entries( + options.nextConfig.experimental.turbopackAdditionalRoots ?? {} + ).map(([key, root]) => ({ key, ...root })) return { ...options, + additionalRoots, nextConfig: await serializeNextConfig( options.nextConfig, path.join(options.rootPath, options.projectPath) @@ -664,16 +668,14 @@ function bindingToApi( } async function rustifyPartialProjectOptions( - options: PartialProjectOptions + options: PartialProjectOptions, + projectPath: string ): Promise { return { ...options, nextConfig: options.nextConfig && - (await serializeNextConfig( - options.nextConfig, - path.join(options.rootPath, options.projectPath) - )), + (await serializeNextConfig(options.nextConfig, projectPath)), env: options.env && rustifyEnv(options.env), } } @@ -681,7 +683,10 @@ function bindingToApi( class ProjectImpl implements Project { private readonly _nativeProject: { __napiType: 'Project' } - constructor(nativeProject: { __napiType: 'Project' }) { + constructor( + nativeProject: { __napiType: 'Project' }, + private readonly projectPath: string + ) { this._nativeProject = nativeProject if (typeof binding.registerWorkerScheduler === 'function') { @@ -692,7 +697,7 @@ function bindingToApi( async update(options: PartialProjectOptions) { await binding.projectUpdate( this._nativeProject, - await rustifyPartialProjectOptions(options) + await rustifyPartialProjectOptions(options, this.projectPath) ) } @@ -1276,18 +1281,23 @@ function bindingToApi( turboEngineOptions, callbacks?: import('./types').TurbopackProjectCallbacks ) { - return new ProjectImpl( - await binding.projectNew( - await rustifyProjectOptions(options), - turboEngineOptions, - { - throwTurbopackInternalError: ( - require('../../shared/lib/turbopack/internal-error') as typeof import('../../shared/lib/turbopack/internal-error') - ).throwTurbopackInternalError, - onBeforeDeferredEntries: callbacks?.onBeforeDeferredEntries, - } - ) + const { value, issues } = await binding.projectNew( + await rustifyProjectOptions(options), + turboEngineOptions, + { + throwTurbopackInternalError: ( + require('../../shared/lib/turbopack/internal-error') as typeof import('../../shared/lib/turbopack/internal-error') + ).throwTurbopackInternalError, + onBeforeDeferredEntries: callbacks?.onBeforeDeferredEntries, + } ) + return { + value: new ProjectImpl( + value.project, + path.join(options.rootPath, options.projectPath) + ), + issues, + } } } @@ -1419,7 +1429,7 @@ async function loadWasm(importPath = '') { _options: ProjectOptions, _turboEngineOptions: TurboEngineOptions, _callbacks?: import('./types').TurbopackProjectCallbacks | undefined - ): Promise { + ): Promise> { throw new Error( `Turbopack is not supported on this platform (${PlatformName}/${ArchName}) because native bindings are not available. ` + `Only WebAssembly (WASM) bindings were loaded, and Turbopack requires native bindings. ` + diff --git a/packages/next/src/build/swc/types.ts b/packages/next/src/build/swc/types.ts index a583ec723d00..5495a2a240e8 100644 --- a/packages/next/src/build/swc/types.ts +++ b/packages/next/src/build/swc/types.ts @@ -44,7 +44,7 @@ export interface Binding { options: ProjectOptions, turboEngineOptions: NapiTurboEngineOptions, callbacks?: TurbopackProjectCallbacks - ): Promise + ): Promise> startTurbopackTraceServerHandle( traceFilePath: string, port: number | undefined @@ -499,7 +499,7 @@ export type WrittenEndpoint = } export interface ProjectOptions - extends Omit { + extends Omit { /** * The next.config.js contents. */ @@ -513,8 +513,6 @@ export interface ProjectOptions export interface PartialProjectOptions extends Omit { - rootPath: NapiProjectOptions['rootPath'] - projectPath: NapiProjectOptions['projectPath'] /** * The next.config.js contents. */ diff --git a/packages/next/src/build/turbopack-analyze/index.ts b/packages/next/src/build/turbopack-analyze/index.ts index 55620b46ce38..4efd07301dee 100644 --- a/packages/next/src/build/turbopack-analyze/index.ts +++ b/packages/next/src/build/turbopack-analyze/index.ts @@ -10,6 +10,7 @@ import { getSupportedBrowsers } from '../get-supported-browsers' import { trace } from '../../trace' import { normalizePath } from '../../lib/normalize-path' import { PHASE_PRODUCTION_BUILD } from '../../shared/lib/constants' +import { printBuildErrors } from '../print-build-errors' export type AnalyzeContext = { config: NextConfigComplete @@ -51,7 +52,7 @@ export async function turbopackAnalyze( const persistentCaching = config.experimental?.turbopackFileSystemCacheForBuild || false const rootPath = config.turbopack?.root || config.outputFileTracingRoot || dir - const project = await bindings.turbo.createProject( + const projectResult = await bindings.turbo.createProject( { rootPath: config.turbopack?.root || config.outputFileTracingRoot || dir, projectPath: normalizePath(path.relative(rootPath, dir) || '.'), @@ -100,8 +101,10 @@ export async function turbopackAnalyze( isShortSession: true, } ) - + const project = projectResult.value try { + printBuildErrors(projectResult, dev) + const analyzeEventsSpan = trace('turbopack-analyze-events') // Stop immediately: this span is only used as a parent for // manualTraceChild calls which carry their own timestamps. diff --git a/packages/next/src/build/turbopack-build/impl.ts b/packages/next/src/build/turbopack-build/impl.ts index 9747fa3b0e3f..38e0d2c50556 100644 --- a/packages/next/src/build/turbopack-build/impl.ts +++ b/packages/next/src/build/turbopack-build/impl.ts @@ -122,7 +122,7 @@ export async function turbopackBuild(telemetry: Telemetry): Promise<{ const sriEnabled = Boolean(config.experimental.sri?.algorithm) - const project = await bindings.turbo.createProject( + const projectResult = await bindings.turbo.createProject( { ...sharedProjectOptions, debugBuildPaths: NextBuildContext.debugBuildPaths, @@ -143,6 +143,7 @@ export async function turbopackBuild(telemetry: Telemetry): Promise<{ } : undefined ) + const project = projectResult.value const shutdownController = new AbortController() const compilationEvents = backgroundLogCompilationEvents(project, { // Compilation events carry their own timestamps, so they hang directly off @@ -162,6 +163,8 @@ export async function turbopackBuild(telemetry: Telemetry): Promise<{ } try { + printBuildErrors(projectResult, dev) + // Write an empty file in a known location to signal this was built with Turbopack await fs.writeFile(path.join(distDir, 'turbopack'), '') diff --git a/packages/next/src/build/utils.ts b/packages/next/src/build/utils.ts index 1fcd611ef20e..d53667df9b3a 100644 --- a/packages/next/src/build/utils.ts +++ b/packages/next/src/build/utils.ts @@ -85,6 +85,7 @@ import { parseNormalizedAppRoute } from '../shared/lib/router/routes/app' import { getStaticMetadataPrerenderPathname } from '../lib/metadata/get-metadata-route' import { isStaticMetadataFile } from '../lib/metadata/is-metadata-route' import { normalizeAppPath } from '../shared/lib/router/utils/app-paths' +import { mapNftFileEntries, type NftJson, resolveNftOutputPath } from './nft' /** * Get the display path for build output. For static metadata files under @@ -1269,59 +1270,102 @@ export async function copyTracedFiles( await fs.writeFile(packageJsonOutputPath, packageJsonContent) } catch {} const copiedFiles = new Set() + const skippedTraceFiles = new Set() + + async function createTracedSymlink( + target: string, + linkPath: string, + sourcePath: string + ) { + let isDirectory = false + if (process.platform === 'win32') { + // Windows requires the target type when creating a symlink. Files are + // copied in an arbitrary order, so the target might not exist in the + // output yet. Inspect the original target through the source symlink + // instead. + try { + isDirectory = (await fs.stat(sourcePath)).isDirectory() + } catch (err: any) { + if (err.code !== 'ENOENT' && err.code !== 'ELOOP') { + throw err + } + } + } + + try { + // the target type argument is ignored on non-windows platforms + await fs.symlink(target, linkPath, isDirectory ? 'dir' : 'file') + } catch (err: any) { + // Windows doesn't support creating symlinks without elevated privileges, + // unless "Developer Mode" is turned on. If we failed to create a symlink + // due to EPERM, try creating a junction point instead. + // + // Ideally we'd just preserve the input file type (junction point or + // symlink), but there's no API in node.js to differentiate between a + // junction point and a symlink, so we just try making a symlink first. + // Symlinks are preferred because they support relative paths and + // non-directory (file) targets. + // + // Note: Junction targets are stored as absolute paths, so this fallback + // is not relocatable even when the preferred symlink above is relative, + // but it's the best we can do. + if (process.platform === 'win32' && err.code === 'EPERM' && isDirectory) { + try { + await fs.symlink( + path.resolve(path.dirname(linkPath), target), + linkPath, + 'junction' + ) + } catch (junctionErr: any) { + if (junctionErr.code !== 'EEXIST') { + throw junctionErr + } + } + } else if (err.code !== 'EEXIST') { + throw err + } + } + } async function handleTraceFiles(traceFilePath: string) { const traceData = JSON.parse( await fs.readFile(/* turbopackIgnore: true */ traceFilePath, 'utf8') - ) as { - files: string[] - } - const copySema = new Sema(10, { capacity: traceData.files.length }) - const traceFileDir = path.dirname(traceFilePath) + ) as NftJson + const entries = mapNftFileEntries(traceData, traceFilePath, tracingRoot, { + skipBaseRootEscapes: true, + onBaseRootEscape: (source) => skippedTraceFiles.add(source), + }) + const copySema = new Sema(10, { capacity: entries.length }) await Promise.all( - traceData.files.map(async (relativeFile) => { + entries.map(async (entry) => { await copySema.acquire() - const tracedFilePath = path.join(traceFileDir, relativeFile) - const fileOutputPath = path.join( + const tracedFilePath = entry.source + const fileOutputPath = resolveNftOutputPath( outputPath, - path.relative(tracingRoot, tracedFilePath) + entry.destination ) if (!copiedFiles.has(fileOutputPath)) { copiedFiles.add(fileOutputPath) await fs.mkdir(path.dirname(fileOutputPath), { recursive: true }) - const symlink = await fs.readlink(tracedFilePath).catch(() => null) - - if (symlink) { - try { - await fs.symlink(symlink, fileOutputPath) - } catch (err: any) { - // Windows doesn't support creating symlinks without elevated privileges, unless - // "Developer Mode" is turned on. If we failed to create a symlink due to EPERM, try - // creating a junction point instead. - // - // Ideally we'd just preserve the input file type (junction point or symlink), but - // there's no API in node.js to differentiate between a junction point and a symlink, - // so we just try making a symlink first. Symlinks are preferred because they support - // relative paths and non-directory (file) targets. - if ( - process.platform === 'win32' && - err.code === 'EPERM' && - path.isAbsolute(symlink) - ) { - try { - await fs.symlink(symlink, fileOutputPath, 'junction') - } catch (junctionErr: any) { - if (junctionErr.code !== 'EEXIST') { - throw junctionErr - } - } - } else if (err.code !== 'EEXIST') { - throw err - } + if (entry.symlinkTarget !== undefined) { + const targetOutputPath = resolveNftOutputPath( + outputPath, + entry.symlinkTarget + ) + const target = + path.relative(path.dirname(fileOutputPath), targetOutputPath) || + '.' + await createTracedSymlink(target, fileOutputPath, tracedFilePath) + } else if (traceData.symlinks === undefined) { + const target = await fs.readlink(tracedFilePath).catch(() => null) + if (target) { + await createTracedSymlink(target, fileOutputPath, tracedFilePath) + } else { + await fs.copyFile(tracedFilePath, fileOutputPath) } } else { await fs.copyFile(tracedFilePath, fileOutputPath) @@ -1473,6 +1517,23 @@ startServer({ process.exit(1); });` ) + + if (skippedTraceFiles.size > 0) { + const count = skippedTraceFiles.size + const skippedFilesOutput = [...skippedTraceFiles] + .slice(0, 100) + .map((file) => ` - ${path.relative(tracingRoot, file)}`) + .join('\n') + const warning = [ + `${count} traced files were not included in the standalone output`, + 'because their paths are outside of `outputFileTracingRoot`.', + 'First 100 skipped files:', + skippedFilesOutput, + 'Set `outputFileTracingRoot` to a common parent directory', + 'to include these files.', + ].join('\n') + Log.warn(warning) + } } export function isReservedPage(page: string) { diff --git a/packages/next/src/cli/internal/static-routes-info.ts b/packages/next/src/cli/internal/static-routes-info.ts index fe3d0a7a4cf5..38f98187a9f6 100644 --- a/packages/next/src/cli/internal/static-routes-info.ts +++ b/packages/next/src/cli/internal/static-routes-info.ts @@ -19,6 +19,7 @@ import fs from 'fs' import path from 'path' import loadConfig from '../../server/config' import { PHASE_PRODUCTION_BUILD } from '../../shared/lib/constants' +import { mapNftFileEntries, type NftJson } from '../../build/nft' export interface StaticRoutesInfoOptions { json?: boolean @@ -347,32 +348,30 @@ function collectServerEntryFiles( sets: FileSets ): void { const entryRel = path.join('server', serverEntry) // e.g. server/app/page.js - const entryDirRel = path.dirname(entryRel) // e.g. server/app - const entryDirAbs = path.join(distDir, entryDirRel) - // The entry .js is always part of the bundle, even if no nft.json exists. sets.serverBundled.add(entryRel) - const nft = readJsonFile<{ files: string[] }>( - path.join(distDir, entryRel + '.nft.json') - ) - if (!nft?.files) return - - for (const relPath of nft.files) { - // Resolve relative to the entry's dir. If the normalized result stays - // inside distDir it's a server chunk; if it leaves distDir it's an - // unbundled trace dep (e.g. ../../../node_modules/...). - const inDistDirPath = path.normalize(path.join(entryDirRel, relPath)) - const outsideDistDir = inDistDirPath.startsWith('..') + const nftPath = path.join(distDir, entryRel + '.nft.json') + const nft = readJsonFile(nftPath) + if (!nft) return + const entries = mapNftFileEntries(nft, nftPath, path.parse(nftPath).root) + + for (const entry of entries) { + // NFT paths are relative to the entry's directory. If the mapped source + // stays inside distDir it's a server chunk; if it leaves distDir it's an + // unbundled trace dependency (e.g. ../../../node_modules/...). + const inDistDirPath = path.relative(distDir, entry.source) + const outsideDistDir = + path.isAbsolute(inDistDirPath) || + inDistDirPath === '..' || + inDistDirPath.startsWith(`..${path.sep}`) const isMap = inDistDirPath.endsWith('.map') if (isMap) { // Source maps go into the maps category whether they're in or outside // distDir, so they don't double-count under serverUnbundled. - sets.serverMaps.add( - outsideDistDir ? path.resolve(entryDirAbs, relPath) : inDistDirPath - ) + sets.serverMaps.add(outsideDistDir ? entry.source : inDistDirPath) } else if (outsideDistDir) { - sets.serverUnbundled.add(path.resolve(entryDirAbs, relPath)) + sets.serverUnbundled.add(entry.source) } else if ( inDistDirPath.endsWith('.js') && !inDistDirPath.endsWith('_client-reference-manifest.js') diff --git a/packages/next/src/server/config-schema.ts b/packages/next/src/server/config-schema.ts index 15e91c9bcc81..029053e58863 100644 --- a/packages/next/src/server/config-schema.ts +++ b/packages/next/src/server/config-schema.ts @@ -196,6 +196,15 @@ export const experimentalSchema = { .union([z.enum(['security', 'latest', 'future']), z.literal(false)]) .optional(), agentFeedback: z.boolean().optional(), + turbopackAdditionalRoots: z + .record( + z.string(), + z.strictObject({ + path: z.string(), + ignoreIfMissing: z.boolean().optional(), + }) + ) + .optional(), outputHashSalt: z.string().optional(), useSkewCookie: z.boolean().optional(), after: z.boolean().optional(), diff --git a/packages/next/src/server/config-shared.ts b/packages/next/src/server/config-shared.ts index 6301840157b7..74a96c6aec52 100644 --- a/packages/next/src/server/config-shared.ts +++ b/packages/next/src/server/config-shared.ts @@ -496,6 +496,20 @@ export interface ExperimentalConfig { * agents prepare anonymized Next.js feedback for user review. */ agentFeedback?: boolean + /** + * Additional filesystem roots that symlinked dependencies may resolve into. + * Relative paths are resolved from the current working directory. + * + * Root names must contain 1-40 characters, using only ASCII letters, digits, + * underscores, or hyphens. They must not be Windows device names and must be + * unique under ASCII case-insensitive comparison. Invalid roots produce a + * warning and are ignored. + */ + turbopackAdditionalRoots?: Record< + string, + { path: string; ignoreIfMissing?: boolean } + > + /** * @deprecated Use the top-level `outputHashSalt` option instead. */ diff --git a/packages/next/src/server/dev/hot-reloader-turbopack.ts b/packages/next/src/server/dev/hot-reloader-turbopack.ts index 58562c467afd..11c30586b974 100644 --- a/packages/next/src/server/dev/hot-reloader-turbopack.ts +++ b/packages/next/src/server/dev/hot-reloader-turbopack.ts @@ -494,7 +494,7 @@ export async function createHotReloaderTurbopack( }) } - const project = await bindings.turbo.createProject( + const projectResult = await bindings.turbo.createProject( { rootPath, projectPath: normalizePath(relative(rootPath, projectPath) || '.'), @@ -539,6 +539,10 @@ export async function createHotReloaderTurbopack( isShortSession: false, } ) + for (const issue of projectResult.issues) { + printNonFatalIssue(issue) + } + const project = projectResult.value backgroundLogCompilationEvents(project, { eventTypes: [ 'StartupCacheInvalidationEvent', diff --git a/packages/next/src/server/lib/router-utils/setup-dev-bundler.ts b/packages/next/src/server/lib/router-utils/setup-dev-bundler.ts index 2db6020a8429..63ac4b070dbc 100644 --- a/packages/next/src/server/lib/router-utils/setup-dev-bundler.ts +++ b/packages/next/src/server/lib/router-utils/setup-dev-bundler.ts @@ -88,7 +88,6 @@ import { } from '../../../shared/lib/turbopack/utils' import { getDefineEnv } from '../../../build/define-env' import { TurbopackInternalError } from '../../../shared/lib/turbopack/internal-error' -import { normalizePath } from '../../../lib/normalize-path' import { recursiveReadDir } from '../../../lib/recursive-readdir' import { JSON_CONTENT_TYPE_HEADER, @@ -942,10 +941,6 @@ async function startWatcher( opts.fsChecker.rewrites.beforeFiles.length > 0 || opts.fsChecker.rewrites.fallback.length > 0 - const rootPath = - opts.nextConfig.turbopack?.root || - opts.nextConfig.outputFileTracingRoot || - opts.dir await hotReloader.turbopackProject.update({ defineEnv: createDefineEnv({ isTurbopack: true, @@ -961,8 +956,6 @@ async function startWatcher( projectPath: opts.dir, rewrites: opts.fsChecker.rewrites, }), - rootPath, - projectPath: normalizePath(path.relative(rootPath, dir)), }) } else { let tsconfigResult: diff --git a/packages/react-refresh-utils/package.json b/packages/react-refresh-utils/package.json index c73a3fcfa9c4..83b2eb39ba96 100644 --- a/packages/react-refresh-utils/package.json +++ b/packages/react-refresh-utils/package.json @@ -1,6 +1,6 @@ { "name": "@next/react-refresh-utils", - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "description": "An experimental package providing utilities for React Refresh.", "repository": { "url": "vercel/next.js", diff --git a/packages/third-parties/package.json b/packages/third-parties/package.json index 88ebaa7784f1..ae9d9adb0ef7 100644 --- a/packages/third-parties/package.json +++ b/packages/third-parties/package.json @@ -1,6 +1,6 @@ { "name": "@next/third-parties", - "version": "16.4.0-canary.35", + "version": "16.4.0-canary.36", "repository": { "url": "vercel/next.js", "directory": "packages/third-parties" @@ -26,7 +26,7 @@ "third-party-capital": "1.0.20" }, "devDependencies": { - "next": "16.4.0-canary.35", + "next": "16.4.0-canary.36", "outdent": "0.8.0", "prettier": "2.5.1", "typescript": "6.0.2" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b5faacb7eded..dac6d72c46b9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1021,7 +1021,7 @@ importers: packages/eslint-config-next: dependencies: '@next/eslint-plugin-next': - specifier: 16.4.0-canary.35 + specifier: 16.4.0-canary.36 version: link:../eslint-plugin-next eslint: specifier: '>=9.0.0' @@ -1104,7 +1104,7 @@ importers: packages/next: dependencies: '@next/env': - specifier: 16.4.0-canary.35 + specifier: 16.4.0-canary.36 version: link:../next-env '@swc/helpers': specifier: 0.5.23 @@ -1225,19 +1225,19 @@ importers: specifier: 1.2.0 version: 1.2.0 '@next/font': - specifier: 16.4.0-canary.35 + specifier: 16.4.0-canary.36 version: link:../font '@next/polyfill-module': - specifier: 16.4.0-canary.35 + specifier: 16.4.0-canary.36 version: link:../next-polyfill-module '@next/polyfill-nomodule': - specifier: 16.4.0-canary.35 + specifier: 16.4.0-canary.36 version: link:../next-polyfill-nomodule '@next/react-refresh-utils': - specifier: 16.4.0-canary.35 + specifier: 16.4.0-canary.36 version: link:../react-refresh-utils '@next/swc': - specifier: 16.4.0-canary.35 + specifier: 16.4.0-canary.36 version: link:../next-swc '@opentelemetry/api': specifier: 1.6.0 @@ -1980,7 +1980,7 @@ importers: version: 1.0.20 devDependencies: next: - specifier: 16.4.0-canary.35 + specifier: 16.4.0-canary.36 version: link:../next outdent: specifier: 0.8.0 diff --git a/rspack/Cargo.lock b/rspack/Cargo.lock index 348c0609f6be..2886792472c7 100644 --- a/rspack/Cargo.lock +++ b/rspack/Cargo.lock @@ -6758,6 +6758,9 @@ dependencies = [ [[package]] name = "turbo-unix-path" version = "0.0.1" +dependencies = [ + "smallvec", +] [[package]] name = "typenum" diff --git a/test/development/basic/next-rs-api.test.ts b/test/development/basic/next-rs-api.test.ts index 9ed57aa57995..24628256a595 100644 --- a/test/development/basic/next-rs-api.test.ts +++ b/test/development/basic/next-rs-api.test.ts @@ -162,7 +162,7 @@ async function main() { const bindings = await loadBindings(); const rootPath = __dirname; const distDir = '.next'; - const project = await bindings.turbo.createProject({ + const projectResult = await bindings.turbo.createProject({ env: {}, nextConfig: nextConfig, rootPath, @@ -204,6 +204,12 @@ async function main() { }, { turbopackMemoryEviction: 'off', }); + if (projectResult.issues.length > 0) { + throw new Error( + \`Project initialization failed: \${JSON.stringify(projectResult.issues)}\` + ); + } + const project = projectResult.value; const entrypointsSubscription = project.entrypointsSubscribe(); const entrypoints = (await entrypointsSubscription.next()).value.value; @@ -316,7 +322,7 @@ describe('next.rs api', () => { ? path.resolve(__dirname, '../../..') : next.testDir const distDir = '.next' - project = await bindings.turbo.createProject( + const projectResult = await bindings.turbo.createProject( { env: {}, nextConfig: nextConfig, @@ -361,6 +367,13 @@ describe('next.rs api', () => { turbopackMemoryEviction: 'off' as MemoryEvictionMode, } ) + const initializationIssues = normalizeIssues(projectResult.issues) + if (initializationIssues.length > 0) { + throw new Error( + `Project initialization failed:\n${JSON.stringify(initializationIssues, null, 2)}` + ) + } + project = projectResult.value projectUpdateSubscription = filterMapAsyncIterator( project.updateInfoSubscribe(1000), (update) => (update.updateType === 'end' ? update.value : undefined) diff --git a/test/e2e/app-dir/app-invalid-revalidate/app-invalid-revalidate.test.ts b/test/e2e/app-dir/app-invalid-revalidate/app-invalid-revalidate.test.ts index a35c4840df10..334b37043cfa 100644 --- a/test/e2e/app-dir/app-invalid-revalidate/app-invalid-revalidate.test.ts +++ b/test/e2e/app-dir/app-invalid-revalidate/app-invalid-revalidate.test.ts @@ -1,100 +1,38 @@ import { nextTestSetup } from 'e2e-utils' -import { check } from 'next-test-utils' - -// TODO(deploy-test-completion): Re-enable this suite in deploy mode. -// It likely asserts local CLI or runtime output that deploy tests do not expose. -// @force-gate !deploy -describe('app-invalid-revalidate', () => { - const { next, isNextDev } = nextTestSetup({ - files: __dirname, - skipStart: true, - }) - - it('should error properly for invalid revalidate at layout', async () => { - await next.stop().catch(() => {}) - const origText = await next.readFile('app/layout.tsx') - - try { - await next.patchFile( - 'app/layout.tsx', - origText.replace('// export', 'export') - ) - await next.start().catch(() => {}) - - await check(async () => { - if (isNextDev) { - await next.fetch('/') - } - return next.cliOutput - }, /Invalid revalidate value "1" on "\/", must be a non-negative number or false/) - } finally { - await next.patchFile('app/layout.tsx', origText) - } - }) - - it('should error properly for invalid revalidate at page', async () => { - await next.stop().catch(() => {}) - const origText = await next.readFile('app/page.tsx') - - try { - await next.patchFile( - 'app/page.tsx', - origText.replace('// export', 'export') - ) - await next.start().catch(() => {}) - - await check(async () => { - if (isNextDev) { - await next.fetch('/') - } - return next.cliOutput - }, /Invalid revalidate value "1" on "\/", must be a non-negative number or false/) - } finally { - await next.patchFile('app/page.tsx', origText) - } - }) - - it('should error properly for invalid revalidate on fetch', async () => { - await next.stop().catch(() => {}) - const origText = await next.readFile('app/page.tsx') - - try { - await next.patchFile( - 'app/page.tsx', - origText.replace('// await', 'await') - ) - await next.start().catch(() => {}) - - await check(async () => { - if (isNextDev) { - await next.fetch('/') - } - return next.cliOutput - }, /Invalid revalidate value "1" on "\/", must be a non-negative number or false/) - } finally { - await next.patchFile('app/page.tsx', origText) - } - }) - - it('should error properly for invalid revalidate on unstable_cache', async () => { - await next.stop().catch(() => {}) - const origText = await next.readFile('app/page.tsx') - - try { - await next.patchFile( - 'app/page.tsx', - origText.replace('// await unstable', 'await unstable') - ) - await next.start().catch(() => {}) - - await check(async () => { - if (isNextDev) { - await next.fetch('/') - } - return next.cliOutput - }, /Invalid revalidate value "1" on "unstable_cache/) - } finally { - await next.patchFile('app/page.tsx', origText) - } - }) -}) +import { retry } from 'next-test-utils' +import path from 'path' + +describe.each(['layout', 'page', 'fetch', 'unstable-cache'])( + 'app-invalid-revalidate (%s)', + (fixture) => { + const { next, isNextDev } = nextTestSetup({ + files: path.join(__dirname, 'fixtures', fixture), + skipStart: true, + nextConfig: { + typescript: { + ignoreBuildErrors: true, + }, + experimental: { + prerenderEarlyExit: false, + }, + }, + }) + + it('reports the invalid revalidate value', async () => { + if (isNextDev) { + await next.start() + await next.fetch('/') + } else { + await expect(next.start()).rejects.toThrow() + } + + await retry(() => { + expect(next.cliOutput).toMatch( + fixture === 'unstable-cache' + ? /Invalid revalidate value "1" on "unstable_cache/ + : /Invalid revalidate value "1" on "\/", must be a non-negative number or false/ + ) + }) + }, 240_000) // This test includes the build/deployment, not just runtime assertions. + } +) diff --git a/test/e2e/app-dir/app-invalid-revalidate/app/page.tsx b/test/e2e/app-dir/app-invalid-revalidate/app/page.tsx deleted file mode 100644 index 02b0c157e111..000000000000 --- a/test/e2e/app-dir/app-invalid-revalidate/app/page.tsx +++ /dev/null @@ -1,9 +0,0 @@ -// eslint-disable-next-line @typescript-eslint/no-unused-vars -import { unstable_cache } from 'next/cache' -// export const revalidate = '1' - -export default async function Page() { - // await fetch('https://example.vercel.sh', { next: { revalidate: '1' } }) - // await unstable_cache(async () => Date.now(), [], { revalidate: '1' })() - return

hello world

-} diff --git a/test/e2e/app-dir/app-invalid-revalidate/fixtures/fetch/app/layout.tsx b/test/e2e/app-dir/app-invalid-revalidate/fixtures/fetch/app/layout.tsx new file mode 100644 index 000000000000..e7077399c03c --- /dev/null +++ b/test/e2e/app-dir/app-invalid-revalidate/fixtures/fetch/app/layout.tsx @@ -0,0 +1,7 @@ +export default function Root({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ) +} diff --git a/test/e2e/app-dir/app-invalid-revalidate/fixtures/fetch/app/page.tsx b/test/e2e/app-dir/app-invalid-revalidate/fixtures/fetch/app/page.tsx new file mode 100644 index 000000000000..17515ba9cbc9 --- /dev/null +++ b/test/e2e/app-dir/app-invalid-revalidate/fixtures/fetch/app/page.tsx @@ -0,0 +1,4 @@ +export default async function Page() { + await fetch('https://example.vercel.sh', { next: { revalidate: '1' } }) + return

hello world

+} diff --git a/test/e2e/app-dir/app-invalid-revalidate/app/layout.tsx b/test/e2e/app-dir/app-invalid-revalidate/fixtures/layout/app/layout.tsx similarity index 81% rename from test/e2e/app-dir/app-invalid-revalidate/app/layout.tsx rename to test/e2e/app-dir/app-invalid-revalidate/fixtures/layout/app/layout.tsx index b159af341fa4..d19301e1115a 100644 --- a/test/e2e/app-dir/app-invalid-revalidate/app/layout.tsx +++ b/test/e2e/app-dir/app-invalid-revalidate/fixtures/layout/app/layout.tsx @@ -1,4 +1,4 @@ -// export const revalidate = '1' +export const revalidate = '1' export default function Root({ children }: { children: React.ReactNode }) { return ( diff --git a/test/e2e/app-dir/app-invalid-revalidate/fixtures/layout/app/page.tsx b/test/e2e/app-dir/app-invalid-revalidate/fixtures/layout/app/page.tsx new file mode 100644 index 000000000000..ff7159d9149f --- /dev/null +++ b/test/e2e/app-dir/app-invalid-revalidate/fixtures/layout/app/page.tsx @@ -0,0 +1,3 @@ +export default function Page() { + return

hello world

+} diff --git a/test/e2e/app-dir/app-invalid-revalidate/fixtures/page/app/layout.tsx b/test/e2e/app-dir/app-invalid-revalidate/fixtures/page/app/layout.tsx new file mode 100644 index 000000000000..e7077399c03c --- /dev/null +++ b/test/e2e/app-dir/app-invalid-revalidate/fixtures/page/app/layout.tsx @@ -0,0 +1,7 @@ +export default function Root({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ) +} diff --git a/test/e2e/app-dir/app-invalid-revalidate/fixtures/page/app/page.tsx b/test/e2e/app-dir/app-invalid-revalidate/fixtures/page/app/page.tsx new file mode 100644 index 000000000000..420b1790a64b --- /dev/null +++ b/test/e2e/app-dir/app-invalid-revalidate/fixtures/page/app/page.tsx @@ -0,0 +1,5 @@ +export const revalidate = '1' + +export default function Page() { + return

hello world

+} diff --git a/test/e2e/app-dir/app-invalid-revalidate/fixtures/unstable-cache/app/layout.tsx b/test/e2e/app-dir/app-invalid-revalidate/fixtures/unstable-cache/app/layout.tsx new file mode 100644 index 000000000000..e7077399c03c --- /dev/null +++ b/test/e2e/app-dir/app-invalid-revalidate/fixtures/unstable-cache/app/layout.tsx @@ -0,0 +1,7 @@ +export default function Root({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ) +} diff --git a/test/e2e/app-dir/app-invalid-revalidate/fixtures/unstable-cache/app/page.tsx b/test/e2e/app-dir/app-invalid-revalidate/fixtures/unstable-cache/app/page.tsx new file mode 100644 index 000000000000..267a12f853d6 --- /dev/null +++ b/test/e2e/app-dir/app-invalid-revalidate/fixtures/unstable-cache/app/page.tsx @@ -0,0 +1,6 @@ +import { unstable_cache } from 'next/cache' + +export default async function Page() { + await unstable_cache(async () => Date.now(), [], { revalidate: '1' })() + return

hello world

+} diff --git a/test/e2e/app-dir/app-invalid-revalidate/next.config.js b/test/e2e/app-dir/app-invalid-revalidate/next.config.js deleted file mode 100644 index 974ac073a5f9..000000000000 --- a/test/e2e/app-dir/app-invalid-revalidate/next.config.js +++ /dev/null @@ -1,13 +0,0 @@ -/** - * @type {import('next').NextConfig} - */ -const nextConfig = { - typescript: { - ignoreBuildErrors: true, - }, - experimental: { - prerenderEarlyExit: false, - }, -} - -module.exports = nextConfig diff --git a/test/e2e/app-dir/turbopack-additional-roots/app/layout.tsx b/test/e2e/app-dir/turbopack-additional-roots/app/layout.tsx new file mode 100644 index 000000000000..888614deda3b --- /dev/null +++ b/test/e2e/app-dir/turbopack-additional-roots/app/layout.tsx @@ -0,0 +1,8 @@ +import { ReactNode } from 'react' +export default function Root({ children }: { children: ReactNode }) { + return ( + + {children} + + ) +} diff --git a/test/e2e/app-dir/turbopack-additional-roots/app/page.tsx b/test/e2e/app-dir/turbopack-additional-roots/app/page.tsx new file mode 100644 index 000000000000..1645f059dd14 --- /dev/null +++ b/test/e2e/app-dir/turbopack-additional-roots/app/page.tsx @@ -0,0 +1,5 @@ +import linked from '../linked' + +export default function Page() { + return

{linked.value}

+} diff --git a/test/e2e/app-dir/turbopack-additional-roots/fixtures/additional-root/node_modules/sibling/index.js b/test/e2e/app-dir/turbopack-additional-roots/fixtures/additional-root/node_modules/sibling/index.js new file mode 100644 index 000000000000..801ca83c17cd --- /dev/null +++ b/test/e2e/app-dir/turbopack-additional-roots/fixtures/additional-root/node_modules/sibling/index.js @@ -0,0 +1 @@ +module.exports = { value: 'initial' } diff --git a/test/e2e/app-dir/turbopack-additional-roots/fixtures/additional-root/node_modules/sibling/package.json b/test/e2e/app-dir/turbopack-additional-roots/fixtures/additional-root/node_modules/sibling/package.json new file mode 100644 index 000000000000..98d410d23524 --- /dev/null +++ b/test/e2e/app-dir/turbopack-additional-roots/fixtures/additional-root/node_modules/sibling/package.json @@ -0,0 +1,5 @@ +{ + "name": "sibling", + "version": "1.0.0", + "main": "index.js" +} diff --git a/test/e2e/app-dir/turbopack-additional-roots/fixtures/additional-root/packages/linked/index.js b/test/e2e/app-dir/turbopack-additional-roots/fixtures/additional-root/packages/linked/index.js new file mode 100644 index 000000000000..9b38013f0e6a --- /dev/null +++ b/test/e2e/app-dir/turbopack-additional-roots/fixtures/additional-root/packages/linked/index.js @@ -0,0 +1,6 @@ +const sibling = require('sibling') +const { formatUrl } = require('next/dist/shared/lib/router/utils/format-url') + +module.exports = { + value: `linked-${sibling.value}-${formatUrl({ pathname: '/next-plugin' })}`, +} diff --git a/test/e2e/app-dir/turbopack-additional-roots/fixtures/additional-root/packages/linked/package.json b/test/e2e/app-dir/turbopack-additional-roots/fixtures/additional-root/packages/linked/package.json new file mode 100644 index 000000000000..54ad46412bdf --- /dev/null +++ b/test/e2e/app-dir/turbopack-additional-roots/fixtures/additional-root/packages/linked/package.json @@ -0,0 +1,5 @@ +{ + "name": "linked", + "version": "1.0.0", + "main": "index.js" +} diff --git a/test/e2e/app-dir/turbopack-additional-roots/turbopack-additional-roots.test.ts b/test/e2e/app-dir/turbopack-additional-roots/turbopack-additional-roots.test.ts new file mode 100644 index 000000000000..8839238f9704 --- /dev/null +++ b/test/e2e/app-dir/turbopack-additional-roots/turbopack-additional-roots.test.ts @@ -0,0 +1,204 @@ +import { isNextStart, nextTestSetup } from 'e2e-utils' +import fs from 'fs-extra' +import os from 'os' +import path from 'path' +import { + fetchViaHTTP, + findPort, + initNextServerScript, + killApp, + retry, +} from 'next-test-utils' + +// Deploy only uploads `project`, but this suite intentionally uses a sibling filesystem root. +// +// @force-gate turbopack && !deploy +describe('turbopack additional roots', () => { + const { next, isNextDev } = nextTestSetup({ + files: __dirname, + subDir: 'project', + nextConfig: { + output: 'standalone', + serverExternalPackages: ['sibling'], + experimental: { + turbopackAdditionalRoots: { + linkedPackages: { path: '../additional-root' }, + missingOptional: { + path: './missing-optional-root', + ignoreIfMissing: false, + }, + }, + }, + }, + skipStart: true, + }) + + let externalRoot: string + let linkedPackage: string + + beforeAll(async () => { + externalRoot = path.resolve(next.testDir, '../additional-root') + linkedPackage = path.join(externalRoot, 'packages/linked') + + await fs.copy( + path.join(__dirname, 'fixtures/additional-root'), + externalRoot + ) + + await fs.symlink( + linkedPackage, + path.join(next.testDir, 'linked'), + 'junction' // use a junction point on windows (this argument is ignored everywhere else) + ) + + await next.start() + }) + + afterAll(async () => { + await next.stop() + await fs.remove(externalRoot) + }) + + it('resolves a linked package, sibling dependency, and next/dist', async () => { + const browser = await next.browser('/') + + expect(await browser.elementByCss('#value').text()).toBe( + 'linked-initial-/next-plugin' + ) + }) + + it('reports initialization warnings when startup succeeds', () => { + expect(next.cliOutput).toContain('Invalid Turbopack additional root') + }) + + if (isNextDev) { + it('tracks updates in an additional root', async () => { + const browser = await next.browser('/') + + await next.patchFile( + '../additional-root/packages/linked/index.js', + (content) => content.replace('linked-', 'updated-'), + async () => { + await retry(async () => { + expect(await browser.elementByCss('#value').text()).toBe( + 'updated-initial-/next-plugin' + ) + }) + } + ) + }) + } + + if (isNextStart) { + it('emits additional-root files and cross-root symlinks in the NFT', async () => { + const nftPath = path.join( + next.testDir, + '.next/server/app/page.js.nft.json' + ) + const nft = await fs.readJson(nftPath) + const crossRootSymlinks = nft.symlinks + .filter((symlink: [number, string, number?]) => symlink.length === 3) + .map( + ([fileIndex, target, additionalRootIndex]: [ + number, + string, + number, + ]) => ({ + file: nft.files[fileIndex], + target, + additionalRoot: nft.additionalRoots[additionalRootIndex].name, + }) + ) + const additionalRoots = nft.additionalRoots.map((root: any) => { + const copy = { ...root } + delete copy.fileHashes + return copy + }) + + expect(crossRootSymlinks).toMatchInlineSnapshot(` + [ + { + "additionalRoot": "linkedPackages", + "file": "../../node_modules/sibling-639f6b1f4617eee0", + "target": "node_modules/sibling", + }, + ] + `) + expect(additionalRoots).toMatchInlineSnapshot(` + [ + { + "files": [ + "node_modules/sibling/index.js", + "node_modules/sibling/package.json", + ], + "name": "linkedPackages", + "path": "../../../../additional-root", + "symlinks": [], + }, + ] + `) + }) + + it('runs after relocating standalone output away from the source root', async () => { + await next.stop() + const temporaryDirectory = await fs.mkdtemp( + path.join(os.tmpdir(), 'next-additional-roots-') + ) + const standaloneDirectory = path.join(temporaryDirectory, 'standalone') + let server: any + + try { + await fs.move( + path.join(next.testDir, '.next/standalone'), + standaloneDirectory + ) + await fs.remove(externalRoot) + + const stagedRoot = path.join( + standaloneDirectory, + 'next_additional_roots', + 'linkedPackages' + ) + expect( + await fs.pathExists( + path.join(stagedRoot, 'node_modules/sibling/index.js') + ) + ).toBe(true) + + const nodeModulesDirectory = path.join( + standaloneDirectory, + '.next/node_modules' + ) + const linkName = (await fs.readdir(nodeModulesDirectory)).find((name) => + name.startsWith('sibling-') + ) + expect(linkName).toBeDefined() + const linkPath = path.join(nodeModulesDirectory, linkName!) + const linkTarget = await fs.readlink(linkPath) + expect(path.isAbsolute(linkTarget)).toBe(false) + expect(path.resolve(nodeModulesDirectory, linkTarget)).toBe( + path.join(stagedRoot, 'node_modules/sibling') + ) + + const appPort = await findPort() + server = await initNextServerScript( + path.join(standaloneDirectory, 'server.js'), + /- Local:/, + { + ...process.env, + ...next.env, + PORT: appPort.toString(), + }, + undefined, + { cwd: standaloneDirectory } + ) + const response = await fetchViaHTTP(appPort, '/') + expect(response.status).toBe(200) + expect(await response.text()).toContain('linked-initial-/next-plugin') + } finally { + if (server) await killApp(server) + await fs.remove(temporaryDirectory) + } + }) + } +}) diff --git a/test/lib/next-modes/next-deploy.ts b/test/lib/next-modes/next-deploy.ts index 7fae1d3c3482..98a73d2b6764 100644 --- a/test/lib/next-modes/next-deploy.ts +++ b/test/lib/next-modes/next-deploy.ts @@ -20,6 +20,7 @@ export class NextDeployInstance extends NextInstance { private _supportsImmutableAssets: boolean = false private _writtenHostsLine: string | null = null private _restoreDnsLookup: (() => void) | null = null + private _startPromise: Promise | undefined constructor(opts: NextInstanceOpts) { super(opts) @@ -78,16 +79,19 @@ export class NextDeployInstance extends NextInstance { ...this.env, } - const deployRes = await execa(deployScriptPath, [], { + const deployment = execa(deployScriptPath, [], { cwd: this.testDir, env: scriptEnv, reject: false, - stderr: 'inherit', }) + deployment.stderr?.pipe(process.stderr) + const deployRes = await deployment if (deployRes.exitCode !== 0) { - throw new Error( - `Custom deploy script failed: ${deployRes.stdout} ${deployRes.stderr} (${deployRes.exitCode})` + await this.throwDeploymentError( + deployRes, + () => this.fetchBuildLogsUsingCustomScript(), + 'Custom deploy script failed' ) } @@ -111,6 +115,31 @@ export class NextDeployInstance extends NextInstance { return { url } } + private async throwDeploymentError( + result: { exitCode: number; stdout: string; stderr?: string }, + fetchBuildLogs: () => Promise, + message: string + ): Promise { + this._cliOutput = result.stdout + (result.stderr || '') + const error = new Error( + `${message}: ${this._cliOutput} (${result.exitCode})` + ) + + try { + // Upload/authentication failures may not have a deployment URL. Retain + // their CLI output even when there are no remote build logs to fetch. + this._parsedUrl = new URL(result.stdout.trim()) + this._url = this._parsedUrl.href + this._cliOutput += await fetchBuildLogs() + } catch (cause) { + error.cause = cause + } + + // Preserve the instance and its logs for callers that catch start(). + // Failed builds do not produce the successful-build ID markers. + throw error + } + private async fetchBuildLogsUsingCustomScript(): Promise { const logsScriptPath = process.env.NEXT_TEST_DEPLOY_LOGS_SCRIPT_PATH! @@ -253,11 +282,21 @@ export class NextDeployInstance extends NextInstance { } public async setup(parentSpan: Span) { - super.setup(parentSpan) + await super.setup(parentSpan) await super.createTestDir({ parentSpan, skipInstall: true }) await this.writeMirrorNpmrcIfNecessary() + if ( + !process.env.NEXT_TEST_DEPLOY_URL?.trim() && + !process.env.NEXT_TEST_DEPLOY_SCRIPT_PATH?.trim() && + !process.env.NEXT_TEST_VERSION + ) { + await this.prepareLocalPackages(parentSpan) + } + } + + private async deploy() { const existingDeployUrl = process.env.NEXT_TEST_DEPLOY_URL?.trim() const customDeployScriptPath = process.env.NEXT_TEST_DEPLOY_SCRIPT_PATH?.trim() @@ -294,12 +333,12 @@ export class NextDeployInstance extends NextInstance { reject: false, } ) + this._cliOutput = buildLogs.stdout + buildLogs.stderr if (buildLogs.exitCode !== 0) { throw new Error( `Failed to get build output logs: ${buildLogs.stderr}` ) } - this._cliOutput = buildLogs.stdout + buildLogs.stderr } this.parseIdsFromCliOutput() @@ -330,10 +369,6 @@ export class NextDeployInstance extends NextInstance { return } - if (!process.env.NEXT_TEST_VERSION) { - await this.prepareLocalPackages(parentSpan) - } - // Original Vercel CLI deployment logic // ensure Vercel CLI is installed try { @@ -478,7 +513,7 @@ export class NextDeployInstance extends NextInstance { additionalEnv.push(`NEXT_ENABLE_ADAPTER=0`) } - const deployRes = await execa( + const deployment = execa( 'vercel', [ 'deploy', @@ -501,16 +536,28 @@ export class NextDeployInstance extends NextInstance { cwd: this.testDir, env: vercelEnv, reject: false, - // This will print deployment information earlier to the console so we - // don't have to wait until the deployment is complete to get the - // inspect URL. - stderr: 'inherit', } ) + // Keep showing deployment progress while also retaining failure output. + deployment.stderr?.pipe(process.stderr) + const deployRes = await deployment if (deployRes.exitCode !== 0) { - throw new Error( - `Failed to deploy project ${deployRes.stdout} ${deployRes.stderr} (${deployRes.exitCode})` + await this.throwDeploymentError( + deployRes, + async () => { + const logs = await execa( + 'vercel', + ['inspect', '--logs', this._url, ...vercelFlags], + { env: vercelEnv, reject: false } + ) + // inspect exits 1 for a failed deployment even when logs are returned. + if (logs.exitCode !== 0 && logs.exitCode !== 1) { + throw new Error(`Failed to get build output logs: ${logs.stderr}`) + } + return logs.stdout + logs.stderr + }, + 'Failed to deploy project' ) } @@ -929,7 +976,7 @@ export class NextDeployInstance extends NextInstance { // Run custom cleanup script if provided const customCleanupScriptPath = process.env.NEXT_TEST_CLEANUP_SCRIPT_PATH?.trim() - if (customCleanupScriptPath) { + if (customCleanupScriptPath && this._url) { await this.cleanupUsingCustomScript().catch((err) => { require('console').error( 'Error running custom cleanup script, continuing with destroy:', @@ -978,7 +1025,18 @@ export class NextDeployInstance extends NextInstance { } public async start() { - // no-op as the deployment is created during setup() + this.throwIfUnavailable() + if (!this._startPromise) { + this._cliOutput = '' + this._url = '' + // Reuse a ready deployment on subsequent calls, as start() did before + // deployment moved out of setup(). A failed attempt can be retried. + this._startPromise = this.deploy().catch((error) => { + this._startPromise = undefined + throw error + }) + } + await this._startPromise } public async patchFile( diff --git a/test/production/build-trace-extra-entries-monorepo/build-trace-extra-entries-monorepo.test.ts b/test/production/build-trace-extra-entries-monorepo/build-trace-extra-entries-monorepo.test.ts index fedf9eaa9f2e..299138007044 100644 --- a/test/production/build-trace-extra-entries-monorepo/build-trace-extra-entries-monorepo.test.ts +++ b/test/production/build-trace-extra-entries-monorepo/build-trace-extra-entries-monorepo.test.ts @@ -1,5 +1,5 @@ import path from 'path' -import { nextTestSetup } from 'e2e-utils' +import { FileRef, nextTestSetup } from 'e2e-utils' describe('build trace with extra entries in monorepo', () => { describe('production mode', () => { @@ -26,4 +26,36 @@ describe('build trace with extra entries in monorepo', () => { ) }) }) + + // @force-gate webpack + describe('standalone output outside outputFileTracingRoot', () => { + const { next, skipped } = nextTestSetup({ + files: { + app: new FileRef(path.join(__dirname, 'app/app')), + '../other': new FileRef(path.join(__dirname, 'other')), + }, + subDir: 'app', + nextConfig: { + output: 'standalone', + outputFileTracingRoot: '.', + outputFileTracingIncludes: { + '/route1': ['../other/included.txt'], + }, + }, + skipStart: true, + skipDeployment: true, + }) + if (skipped) return + + it('warns and completes the build', async () => { + const { exitCode, cliOutput } = await next.runCommand(['build']) + + expect(exitCode).toBe(0) + expect(cliOutput).toMatch( + /\d+ traced files were not included in the standalone output/ + ) + expect(cliOutput).toContain('First 100 skipped files:') + expect(cliOutput).toContain('outputFileTracingRoot') + }) + }) }) diff --git a/test/production/next-server-nft/next-server-nft.test.ts b/test/production/next-server-nft/next-server-nft.test.ts index 8cb727f47488..12dfcc1fd108 100644 --- a/test/production/next-server-nft/next-server-nft.test.ts +++ b/test/production/next-server-nft/next-server-nft.test.ts @@ -145,6 +145,7 @@ async function readNormalizedNFT(next, name) { "/node_modules/next/dist/build/get-supported-browsers.js", "/node_modules/next/dist/build/next-config-ts/require-hook.js", "/node_modules/next/dist/build/next-config-ts/transpile-config.js", + "/node_modules/next/dist/build/nft.js", "/node_modules/next/dist/build/output/format.js", "/node_modules/next/dist/build/output/log.js", "/node_modules/next/dist/build/segment-config/app/app-segment-config.js", diff --git a/test/unit/next-deploy-failure/next-deploy-failure.test.ts b/test/unit/next-deploy-failure/next-deploy-failure.test.ts new file mode 100644 index 000000000000..4348b8cc8fe0 --- /dev/null +++ b/test/unit/next-deploy-failure/next-deploy-failure.test.ts @@ -0,0 +1,313 @@ +import execa from 'execa' +import { trace } from 'next/dist/trace' + +jest.mock('execa', () => jest.fn()) + +// Initialize the real harness in deploy mode, without running a deployment. +const originalMode = process.env.NEXT_TEST_MODE +process.env.NEXT_TEST_MODE = 'deploy' +const { nextTestSetup } = + require('../../lib/e2e-utils') as typeof import('../../lib/e2e-utils') +if (originalMode === undefined) delete process.env.NEXT_TEST_MODE +else process.env.NEXT_TEST_MODE = originalMode + +const { NextDeployInstance } = + require('../../lib/next-modes/next-deploy') as typeof import('../../lib/next-modes/next-deploy') +const { NextInstance } = + require('../../lib/next-modes/base') as typeof import('../../lib/next-modes/base') + +const deploymentUrl = 'https://fixture.vercel.app' +const diagnostic = + 'Invalid revalidate value "1" on "/", must be a non-negative number or false' +const ids = + 'BUILD_ID: build-id\nDEPLOYMENT_ID: deployment-id\nNEXT_SUPPORTS_IMMUTABLE_ASSETS: 1' +type Result = { exitCode: number; stdout: string; stderr: string } + +describe('deployment lifecycle', () => { + let deployResult: Result + let logs: Result + let customLogs: Result + + beforeEach(() => { + jest.replaceProperty(process, 'env', { + ...process.env, + NEXT_TEST_MODE: 'deploy', + NEXT_TEST_VERSION: 'test-unit', + NEXT_TEST_DEPLOY_URL: '', + NEXT_TEST_DEPLOY_SCRIPT_PATH: '', + NEXT_TEST_DEPLOY_LOGS_SCRIPT_PATH: '', + NEXT_TEST_CLEANUP_SCRIPT_PATH: '', + NEXT_TEST_PROXY_ADDRESS: '', + NEXT_TEST_JOB: '', + VERCEL_FORCE_BUILD_IN_HIVE: '', + VERCEL_BUILD_CONTAINER_VERSION: '', + }) + jest.spyOn(require('console'), 'log').mockImplementation(() => {}) + jest.spyOn(require('console'), 'error').mockImplementation(() => {}) + jest.spyOn(NextInstance.prototype, 'setup').mockResolvedValue() + jest + .spyOn(NextInstance.prototype, 'destroy') + .mockImplementation(async function (this: any) { + this.emit('destroy', []) + }) + jest + .spyOn(NextInstance.prototype as any, 'createTestDir') + .mockResolvedValue(undefined) + jest + .spyOn(NextDeployInstance.prototype as any, 'writeMirrorNpmrcIfNecessary') + .mockResolvedValue(undefined) + jest + .spyOn(NextDeployInstance.prototype as any, 'configureProxyAddress') + .mockResolvedValue(undefined) + + deployResult = { exitCode: 1, stdout: deploymentUrl, stderr: '' } + logs = { exitCode: 1, stdout: '', stderr: diagnostic } + customLogs = { ...logs, exitCode: 0 } + + jest + .mocked(execa) + .mockReset() + .mockImplementation((command, args) => { + let result: Result + if (command === 'mock-deploy') { + result = deployResult + } else if (command === 'mock-logs') { + result = customLogs + } else if (command === 'mock-cleanup') { + result = { exitCode: 0, stdout: '', stderr: '' } + } else if (command === 'vercel' && Array.isArray(args)) { + switch (args[0]) { + case '--version': + case 'link': + result = { exitCode: 0, stdout: '', stderr: '' } + break + case 'deploy': + result = deployResult + break + case 'inspect': + result = logs + break + default: + throw new Error('Unexpected Vercel command') + } + } else { + throw new Error('Unexpected subprocess') + } + return Promise.resolve(result) as unknown as ReturnType + }) + }) + + afterEach(() => { + jest.restoreAllMocks() + }) + + async function instance() { + const next = new NextDeployInstance({ files: __dirname }) + await next.setup(trace('test')) + return next + } + + function setupHarness(skipStart: boolean) { + let setup!: () => Promise + let teardown!: () => Promise + jest.spyOn(global, 'beforeAll').mockImplementation((hook) => { + setup = hook as () => Promise + }) + jest.spyOn(global, 'afterAll').mockImplementation((hook) => { + teardown = hook as () => Promise + }) + const { next } = nextTestSetup({ files: __dirname, skipStart }) + return { next, setup, teardown } + } + + function successfulDeployment() { + deployResult.exitCode = 0 + logs = { exitCode: 0, stdout: '', stderr: ids } + customLogs = logs + } + + it('skipStart leaves deployment to the test body, where failures can be asserted', async () => { + const { next, setup, teardown } = setupHarness(true) + try { + await setup() + expect(execa).not.toHaveBeenCalled() + expect(next.cliOutput).toBe('') + + await expect(next.start()).rejects.toThrow('Failed to deploy project') + expect(next.cliOutput).toContain(diagnostic) + } finally { + await teardown() + } + }) + + it('nextTestSetup still deploys automatically by default', async () => { + successfulDeployment() + const { next, setup, teardown } = setupHarness(false) + try { + await setup() + expect(next.url).toBe(deploymentUrl) + expect(next.buildId).toBe('build-id') + } finally { + await teardown() + } + }) + + it('an uncaught deployment failure still rejects the default setup hook', async () => { + const { setup, teardown } = setupHarness(false) + try { + await expect(setup()).rejects.toThrow('Failed to deploy project') + expect(NextInstance.prototype.destroy).toHaveBeenCalled() + } finally { + await teardown() + } + }) + + it('collects failed build logs before start rejects, without requiring build IDs', async () => { + const next = await instance() + deployResult.stderr = 'Build command failed\n' + await expect(next.start()).rejects.toThrow('Failed to deploy project') + expect(next.cliOutput).toContain(deployResult.stderr) + expect(next.cliOutput).toContain(diagnostic) + expect(next.buildId).toBeUndefined() + expect(next.url).toBe(deploymentUrl + '/') + expect(execa).toHaveBeenCalledWith( + 'vercel', + expect.arrayContaining(['inspect', '--logs', deploymentUrl + '/']), + expect.anything() + ) + }) + + it('retains CLI diagnostics when failure happens before a deployment URL is returned', async () => { + const next = await instance() + deployResult.stdout = '' + deployResult.stderr = 'Unauthorized' + await expect(next.start()).rejects.toThrow('Failed to deploy project') + expect(next.cliOutput).toBe('Unauthorized') + expect(execa).not.toHaveBeenCalledWith( + 'vercel', + expect.arrayContaining(['inspect']), + expect.anything() + ) + }) + + it('does not treat a canceled deployment as a successful start', async () => { + const next = await instance() + logs.stderr = 'Deployment canceled' + await expect(next.start()).rejects.toThrow('Failed to deploy project') + expect(next.cliOutput).toContain('Deployment canceled') + }) + + it('retains the deployment error if fetching its logs fails', async () => { + const next = await instance() + logs = { exitCode: 2, stdout: '', stderr: 'Invalid arguments' } + await expect(next.start()).rejects.toMatchObject({ + message: expect.stringContaining('Failed to deploy project'), + cause: expect.objectContaining({ + message: 'Failed to get build output logs: Invalid arguments', + }), + }) + }) + + it('loads successful deployment IDs during start', async () => { + successfulDeployment() + const next = await instance() + expect(execa).not.toHaveBeenCalled() + await next.start() + expect(next.buildId).toBe('build-id') + expect(next.deploymentId).toBe('deployment-id') + }) + + it('reuses a deployment for concurrent and subsequent start calls', async () => { + successfulDeployment() + const next = await instance() + await Promise.all([next.start(), next.start()]) + await next.start() + expect( + jest.mocked(execa).mock.calls.filter(([, args]) => args?.[0] === 'deploy') + ).toHaveLength(1) + }) + + it('can retry a failed start without retaining stale failure logs', async () => { + const next = await instance() + await expect(next.start()).rejects.toThrow('Failed to deploy project') + successfulDeployment() + await next.start() + expect(next.cliOutput).not.toContain(diagnostic) + expect(next.buildId).toBe('build-id') + expect( + jest.mocked(execa).mock.calls.filter(([, args]) => args?.[0] === 'deploy') + ).toHaveLength(2) + }) + + it('defers custom deployment scripts and exposes their failed build logs', async () => { + process.env.NEXT_TEST_DEPLOY_SCRIPT_PATH = 'mock-deploy' + process.env.NEXT_TEST_DEPLOY_LOGS_SCRIPT_PATH = 'mock-logs' + const next = await instance() + expect(execa).not.toHaveBeenCalled() + await expect(next.start()).rejects.toThrow('Custom deploy script failed') + expect(next.cliOutput).toContain(diagnostic) + }) + + it('continues loading IDs from successful custom deployments', async () => { + process.env.NEXT_TEST_DEPLOY_SCRIPT_PATH = 'mock-deploy' + process.env.NEXT_TEST_DEPLOY_LOGS_SCRIPT_PATH = 'mock-logs' + successfulDeployment() + const next = await instance() + await next.start() + expect(next.buildId).toBe('build-id') + expect(next.deploymentId).toBe('deployment-id') + }) + + it('does not replace a custom deployment failure when its logs are unavailable', async () => { + process.env.NEXT_TEST_DEPLOY_SCRIPT_PATH = 'mock-deploy' + process.env.NEXT_TEST_DEPLOY_LOGS_SCRIPT_PATH = 'mock-logs' + customLogs = { exitCode: 1, stdout: '', stderr: 'Logs unavailable' } + const next = await instance() + await expect(next.start()).rejects.toMatchObject({ + message: expect.stringContaining('Custom deploy script failed'), + cause: expect.objectContaining({ + message: expect.stringContaining('Custom deploy logs script failed'), + }), + }) + }) + + it('attaches to an existing deployment only on start', async () => { + process.env.NEXT_TEST_DEPLOY_URL = deploymentUrl + successfulDeployment() + const next = await instance() + expect(execa).not.toHaveBeenCalled() + await next.start() + expect(next.url).toBe(deploymentUrl + '/') + expect(next.buildId).toBe('build-id') + expect(execa).toHaveBeenCalledTimes(1) + expect(execa).toHaveBeenCalledWith( + 'vercel', + ['inspect', '--logs', deploymentUrl + '/'], + expect.anything() + ) + }) + + it('does not run deployment cleanup when start was never called', async () => { + process.env.NEXT_TEST_CLEANUP_SCRIPT_PATH = 'mock-cleanup' + const next = await instance() + await next.destroy() + expect(execa).not.toHaveBeenCalled() + }) + + it('retains logs when attaching to an existing failed deployment', async () => { + process.env.NEXT_TEST_DEPLOY_URL = deploymentUrl + const next = await instance() + await expect(next.start()).rejects.toThrow( + 'Failed to get build output logs' + ) + expect(next.cliOutput).toContain(diagnostic) + }) + + it('still cleans up a deployment whose start failed', async () => { + process.env.NEXT_TEST_CLEANUP_SCRIPT_PATH = 'mock-cleanup' + const next = await instance() + await expect(next.start()).rejects.toThrow('Failed to deploy project') + await next.destroy() + expect(execa).toHaveBeenCalledWith('mock-cleanup', [], expect.anything()) + }) +}) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index 6ea99dcfddba..ca04e1c9304f 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -11,6 +11,7 @@ use crate::{ }, invalidate::make_task_dirty_internal, }, + storage::SpecificTaskDataCategory, storage_schema::TaskStorageAccessors, }, data::{InProgressState, InProgressStateInner}, @@ -34,15 +35,21 @@ pub(super) fn resurrect_deleted<'e, C: ExecuteContext<'e>>( drop(guard); let mut task = ctx.task(task_id, TaskDataCategory::All); - // Double-check under the re-acquired guard: a concurrent connect may have already done this + // Double-check under the re-acquired guard: a concurrent connect may have already done this. if task.deleted() { task.set_deleted(false); - // Mark dirty so it is rescheduled, GC has already dropped its edges and data, so we need to - // re-execute them to bring it back - // NOTE: recovering from disk is technically sometimes possible but doesn't work for new - // tasks, and the snapshot may have already persisted a tombstone. So it would at - // best be an optimistic way to recover data that is in the process of being deleted. It - // shouldn't matter for resolving this rare race condition. + + // The GC snapshot may already have persisted this task's tombstone before the reconnect + // acquired its guard. Treat the resurrected task as new so the next snapshot restores the + // task-type index deleted by that tombstone, and persist Data so the type used to verify + // the index entry is restored too. A resident deleted task always has its type: GC + // restores All before marking it deleted, and eviction removes deleted tasks as + // whole entries. + task.set_new_task(true); + let _ = task.track_modification(SpecificTaskDataCategory::Data, "gc_resurrected"); + + // Mark dirty so it is rescheduled. GC has already dropped its edges and data, so it needs + // to re-execute to bring them back. make_task_dirty_internal( &mut task, /* make_stale */ true, diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index abdb40dea6cd..f53fb7159438 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -12,7 +12,7 @@ use std::{ sync::Arc, }; -use anyhow::{Context, Result, bail}; +use anyhow::{Context, Result}; use bincode::{Decode, Encode}; use parking_lot::RwLockReadGuard; use tracing::info_span; @@ -344,6 +344,13 @@ impl<'e> ExecuteContextImpl<'e> { } if do_data || do_meta || data_restoring || meta_restoring { + let waiting_for_restore = data_restoring || meta_restoring; + if waiting_for_restore { + // The caller holds the task id outside the graph while waiting, so pin it + // against GC until the restored guard reaches the use boundary. Eviction is + // still allowed; the wait loop restores the category again if needed. + task.update_and_get_transient_ref_count(1); + } // Drop lock while doing I/O (our I/O can overlap with the other thread). drop(task); @@ -369,6 +376,11 @@ impl<'e> ExecuteContextImpl<'e> { } else { self.backend.storage.access_mut(task_id) }; + if waiting_for_restore { + // This caller owns the pin and releases it only after acquiring the task + // guard it is about to use. + task.update_and_get_transient_ref_count(-1); + } // Apply results and clear restoring bits. if let Some(result) = storage_data @@ -473,53 +485,92 @@ impl<'e> ExecuteContextImpl<'e> { /// Waits for another thread's in-progress restore of a task to complete. /// - /// Precondition: the caller must have observed `is_restoring()` == true for - /// `task_id`+`category` and must have dropped the task lock before calling this. + /// Precondition: the caller must have observed `is_restoring()` == true, taken one restore + /// transient ref for `task_id`, and dropped the task lock before calling this. /// - /// Returns the `StorageWriteGuard` acquired at the end of the wait when successful, - /// or `Err` if the restoring thread failed (restoring was cleared without setting restored). + /// Returns the `StorageWriteGuard` acquired at the end of the wait with the caller's transient + /// ref still held. The caller releases that ref at its actual use boundary; this keeps pin + /// ownership consistent for single, paired, and batched task access. fn wait_for_restoring_task( &self, task_id: TaskId, category: TaskDataCategory, ) -> Result> { - // Fast path: acquire the write guard and check flags directly. - // By the time this is called, some I/O has elapsed and the other thread has - // likely already finished restoring. + // Fast path: the restoring thread usually finishes its I/O before this waiter gets here. + // Avoid registering a listener when the requested category is already available. { let task = self.backend.storage.access_mut(task_id); - let is_restoring = task.flags.is_restoring(category); - let is_restored = task.flags.is_restored(category); - if is_restored { + if task.flags.is_restored(category) { return Ok(task); } - if !is_restoring { - bail!("restoring failed"); - } - // Still restoring — drop the write guard before waiting. - drop(task); } - // Slow path: register a listener and wait until the other thread signals completion. loop { - // Register a listener BEFORE re-acquiring the lock (avoids a lost-wakeup race). + // Register before taking the task lock to avoid a lost wakeup when another restorer is + // still active. It is harmless when this thread becomes the replacement restorer. let listener = self.backend.storage.restored.listen(); + let mut task = self.backend.storage.access_mut(task_id); - let task = self.backend.storage.access_mut(task_id); - let is_restoring = task.flags.is_restoring(category); - let is_restored = task.flags.is_restored(category); - - if is_restored { - // The restoring thread finished successfully; return the write guard directly. + if task.flags.is_restored(category) { return Ok(task); } - if !is_restoring { - // The restoring bit was cleared without setting the restored bit. - // This means the restoring thread encountered an error. - bail!("restoring failed"); + + // No thread owns a missing category after a prior restore attempt failed and cleared + // its bit, or after eviction cleared the completed restore before this waiter acquired + // the guard. GC cannot collect it while our transient ref is held. Keep that ref while + // claiming the category and retrying the restore. + let restore_data = category.includes_data() + && !task.flags.data_restored() + && !task.flags.data_restoring(); + let restore_meta = category.includes_meta() + && !task.flags.meta_restored() + && !task.flags.meta_restoring(); + + if restore_data || restore_meta { + if restore_data { + task.flags.set_data_restoring(true); + } + if restore_meta { + task.flags.set_meta_restoring(true); + } + drop(task); + + let storage_data = restore_data + .then(|| self.restore_task_data(task_id, SpecificTaskDataCategory::Data)); + let storage_meta = restore_meta + .then(|| self.restore_task_data(task_id, SpecificTaskDataCategory::Meta)); + + let mut task = self.backend.storage.access_mut(task_id); + let mut restore_error = None; + if let Some(result) = storage_data + && let Err(error) = + apply_restore_result(&mut task, result, SpecificTaskDataCategory::Data) + { + restore_error = Some(error); + } + if let Some(result) = storage_meta + && let Err(error) = + apply_restore_result(&mut task, result, SpecificTaskDataCategory::Meta) + && restore_error.is_none() + { + restore_error = Some(error); + } + + // Keep the restored guard through notification. The caller's transient ref remains + // held until the guard reaches its actual use boundary. + self.backend.storage.restored.notify(usize::MAX); + if let Some(error) = restore_error { + task.update_and_get_transient_ref_count(-1); + return Err(error); + } + if task.flags.is_restored(category) { + return Ok(task); + } + drop(task); + continue; } - // Still restoring; drop the lock and block until notified, then loop to re-check. + // Every missing category is still owned by another restorer. drop(task); let _span = info_span!("blocking").entered(); listener.wait(); @@ -537,7 +588,7 @@ impl<'e> ExecuteContextImpl<'e> { match self.wait_for_restoring_task(task_id, category) { Ok(guard) => guard, Err(e) => { - panic!("Restore of {category:?} for task {task_id} failed in another thread: {e:?}") + panic!("Restore of {category:?} for task {task_id} failed while waiting: {e:?}") } } } @@ -614,6 +665,7 @@ impl<'e> ExecuteContextImpl<'e> { wait_meta: false, task_type: None, self_restored: false, + transient_ref_pinned: false, }) .collect::>(); data_count += all_count; @@ -662,6 +714,14 @@ impl<'e> ExecuteContextImpl<'e> { } } + if !ready { + // The callback's task id is held outside the graph until it acquires a guard, so + // keep it alive with a transient ref. Eviction may still clear the category; the + // callback path restores it again before use. + task.update_and_get_transient_ref_count(1); + entry.transient_ref_pinned = true; + } + self.task_lock_counter.release(); if ready { prepared_task_callback(self, task_id, category, task); @@ -792,6 +852,16 @@ impl<'e> ExecuteContextImpl<'e> { } if !restore_errors.is_empty() { + // No callback will consume these entries, so release every transient ref before the + // aggregated restore error tears the operation down. + for entry in &tasks { + if entry.transient_ref_pinned { + self.backend + .storage + .access_mut(entry.task_id) + .update_and_get_transient_ref_count(-1); + } + } let msgs: Vec = restore_errors .iter() .map(|(id, cat, e)| format!("Failed to restore {cat} for task {id}: {e:?}")) @@ -816,7 +886,10 @@ impl<'e> ExecuteContextImpl<'e> { // Only call the callback if no category is still being restored by another thread. // If so, Phase 3 calls the callback after all categories are fully restored. if !entry.wait_data && !entry.wait_meta { - let task = self.backend.storage.access_mut(entry.task_id); + // The classification-time transient ref prevents GC until this callback acquires + // the task. The helper re-restores the category if eviction won the handoff. + let mut task = self.wait_for_restore_or_panic(entry.task_id, entry.category); + task.update_and_get_transient_ref_count(-1); prepared_task_callback(self, entry.task_id, entry.category, task); } } @@ -830,7 +903,8 @@ impl<'e> ExecuteContextImpl<'e> { // Blocks (using shared read locks) until this task is fully restored. // Returns the write guard so we call the callback without re-acquiring. self.task_lock_counter.acquire(); - let task = self.wait_for_restore_or_panic(entry.task_id, cat); + let mut task = self.wait_for_restore_or_panic(entry.task_id, cat); + task.update_and_get_transient_ref_count(-1); self.task_lock_counter.release(); prepared_task_callback(self, entry.task_id, entry.category, task); } @@ -857,6 +931,8 @@ struct TaskRestoreEntry { task_type: Option, /// This thread performed the restore for at least one category (set in Phase 1c). self_restored: bool, + /// A transient GC ref for the restore-to-callback interval was taken during classification. + transient_ref_pinned: bool, } /// Whether a restore we performed proves the task exists on disk: we ran the I/O (outer `Some`), it @@ -1056,6 +1132,14 @@ impl<'e> ExecuteContext<'e> for ExecuteContextImpl<'e> { || data2_restoring || meta2_restoring { + let waiting1 = data1_restoring || meta1_restoring; + let waiting2 = data2_restoring || meta2_restoring; + if waiting1 { + task1.update_and_get_transient_ref_count(1); + } + if waiting2 { + task2.update_and_get_transient_ref_count(1); + } // Drop both locks while doing I/O or waiting. drop(task1); drop(task2); @@ -1093,6 +1177,12 @@ impl<'e> ExecuteContext<'e> for ExecuteContextImpl<'e> { let (t1, t2) = self.backend.storage.access_pair_mut(task_id1, task_id2); task1 = t1; task2 = t2; + if waiting1 { + task1.update_and_get_transient_ref_count(-1); + } + if waiting2 { + task2.update_and_get_transient_ref_count(-1); + } // Apply results and clear restoring bits. // On error: drop both locks, notify waiters, then panic. diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs index d77e1461d238..a08e1e2d025e 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage_schema.rs @@ -418,15 +418,6 @@ impl TaskFlags { } } - /// Check if the category's restoration is currently in progress by another thread - pub fn is_restoring(&self, category: TaskDataCategory) -> bool { - match category { - TaskDataCategory::Meta => self.meta_restoring(), - TaskDataCategory::Data => self.data_restoring(), - TaskDataCategory::All => self.meta_restoring() || self.data_restoring(), - } - } - /// Set or clear the restoring bits for the given category pub fn set_restoring(&mut self, category: TaskDataCategory, value: bool) { match category { @@ -864,6 +855,18 @@ impl TaskStorage { self.set_transient_ref_count(1); } + /// Adjust the transient in-session reference count and return the new value. + /// + /// Panics on underflow or overflow. + pub fn update_and_get_transient_ref_count(&mut self, delta: i32) -> u32 { + let current = self.gc_transient_ref_count(); + let new_value = current + .checked_add_signed(delta) + .expect("transient_ref_count underflow"); + self.set_transient_ref_count(new_value); + new_value + } + /// Whether a GC pass may collect this task: nothing references it, via parents, transient /// pins, aggregation edges, or dependency edges. /// diff --git a/turbopack/crates/turbo-tasks-fs/Cargo.toml b/turbopack/crates/turbo-tasks-fs/Cargo.toml index 3e3448f5dec2..17627e2c1883 100644 --- a/turbopack/crates/turbo-tasks-fs/Cargo.toml +++ b/turbopack/crates/turbo-tasks-fs/Cargo.toml @@ -39,6 +39,7 @@ jsonc-parser = { version = "0.26.3", features = ["serde"] } mime = { workspace = true } notify = { workspace = true } parking_lot = { workspace = true } +pathdiff = { workspace = true } regex = { workspace = true } rustc-hash = { workspace = true } serde = { workspace = true, features = ["rc"] } diff --git a/turbopack/crates/turbo-tasks-fs/src/content.rs b/turbopack/crates/turbo-tasks-fs/src/content.rs index a89fddcc9353..53d14b80e71f 100644 --- a/turbopack/crates/turbo-tasks-fs/src/content.rs +++ b/turbopack/crates/turbo-tasks-fs/src/content.rs @@ -8,19 +8,19 @@ use std::{ path::Path, }; -use anyhow::{Result, bail}; +use anyhow::{Context, Result, bail}; use bincode::{Decode, Encode}; use jsonc_parser::{ParseOptions, parse_to_serde_value}; use mime::Mime; use serde_json::Value; use turbo_rcstr::{RcStr, rcstr}; -use turbo_tasks::{NonLocalValue, ReadRef, ValueToString, Vc, trace::TraceRawVcs}; +use turbo_tasks::{NonLocalValue, ReadRef, ResolvedVc, ValueToString, Vc, trace::TraceRawVcs}; use turbo_tasks_hash::{ DeterministicHash, DeterministicHasher, HashAlgorithm, deterministic_hash, hash_xxh3_hash64, }; use crate::{ - FileSystemEntryType, FileSystemPath, RealPathErrorType, + DiskFileSystem, FileSystemEntryType, FileSystemPath, RealPathErrorType, json::UnparsableJson, retry::retry_blocking, rope::{Rope, RopeReader}, @@ -171,7 +171,10 @@ pub(crate) enum FileComparison { #[derive(Clone, Debug, Hash, PartialEq, Eq, TraceRawVcs, NonLocalValue, Encode, Decode)] pub enum LinkTarget { /// The link is an absolute path on disk. - Absolute { resolved: FileSystemPath }, + Absolute { + raw: RcStr, + resolved: FileSystemPath, + }, Relative { /// The value read from the link. The path is lexically converted to a [unix-style /// path][turbo_unix_path::sys_to_unix], but it may contain `..` relative to the *directory @@ -186,7 +189,9 @@ impl LinkTarget { /// The path this link points at. pub fn file_system_path(&self) -> &FileSystemPath { match self { - LinkTarget::Absolute { resolved } | LinkTarget::Relative { resolved, .. } => resolved, + LinkTarget::Absolute { resolved, .. } | LinkTarget::Relative { resolved, .. } => { + resolved + } } } @@ -243,52 +248,6 @@ pub enum LinkContent { Invalid { reason: RcStr }, } -#[turbo_tasks::value_impl] -impl LinkContent { - /// Hashes the link itself (its target and type), not the content of whatever the link points - /// at. This mirrors [`FileContent::hash`] and is the right content hash for consumers that - /// re-create a symlink as a symlink instead of copying the resolved file. - #[turbo_tasks::function] - pub async fn hash(&self, salt: Vc, algorithm: HashAlgorithm) -> Result> { - #[derive(DeterministicHash)] - enum SimplifiedLinkContent<'a> { - Absolute(&'a RcStr), - Relative(&'a RcStr), - NotFound, - Invalid, // the actual error message doesn't matter for this API - } - let simplified = match self { - LinkContent::Link { target } => match target { - LinkTarget::Absolute { resolved } => { - SimplifiedLinkContent::Absolute(&resolved.path) - } - LinkTarget::Relative { raw, resolved: _ } => SimplifiedLinkContent::Relative(raw), - }, - LinkContent::NotFound => SimplifiedLinkContent::NotFound, - LinkContent::Invalid { reason: _ } => SimplifiedLinkContent::Invalid, - }; - Ok(Vc::cell(RcStr::from(deterministic_hash( - &salt.await?, - simplified, - algorithm, - )))) - } -} - -/// The target of a symbolic link to create, used by [`WriteLinkContent`]. -/// -/// Unlike [`LinkTarget`] this carries only the raw path: the write side never needs the target -/// resolved, and the link being created may not even exist yet. -#[derive( - Clone, Debug, Hash, PartialEq, Eq, TraceRawVcs, NonLocalValue, DeterministicHash, Encode, Decode, -)] -pub enum WriteLinkTarget { - /// Normalized and relative to the *filesystem root*. - Absolute(RcStr), - /// Written verbatim, relative to the *directory containing the link*. - Relative(RcStr), -} - /// The file type of the target of a newly written link. This value is only used on Windows. #[derive( Clone, Debug, Hash, PartialEq, Eq, TraceRawVcs, NonLocalValue, DeterministicHash, Encode, Decode, @@ -308,12 +267,32 @@ pub enum WriteLinkTargetType { /// directories, because symlink creation may fail if "developer mode" is not enabled and we're /// running in an unprivileged environment. #[turbo_tasks::value(shared)] -#[derive(Clone, Debug, DeterministicHash)] +#[derive(Clone, Debug)] pub struct WriteLinkContent { - pub target: WriteLinkTarget, + pub target: FileSystemPath, pub target_type: WriteLinkTargetType, } +impl WriteLinkContent { + /// Hashes the link target and target type, not the target's contents. Returns a hash that is + /// stable across builds. + pub async fn hash(&self, salt: &RcStr, algorithm: HashAlgorithm) -> Result { + // convert the fs Vc (which is not stable across cold builds therefore cannot implement + // DeterministicHash) to the configured name, which should be globally unique and stable + // across cold builds. + let target_fs = ResolvedVc::try_downcast_type::(self.target.fs) + .context("link target must use a disk filesystem")? + .await?; + let target_fs_name = target_fs.name(); + + Ok(RcStr::from(deterministic_hash( + salt, + (target_fs_name, &self.target.path, &self.target_type), + algorithm, + ))) + } +} + #[turbo_tasks::value(shared)] #[derive(Clone, DeterministicHash, PartialOrd, Ord)] pub struct File { diff --git a/turbopack/crates/turbo-tasks-fs/src/disk.rs b/turbopack/crates/turbo-tasks-fs/src/disk.rs index 8dd768145de5..3d7d7f8810f9 100644 --- a/turbopack/crates/turbo-tasks-fs/src/disk.rs +++ b/turbopack/crates/turbo-tasks-fs/src/disk.rs @@ -7,6 +7,7 @@ use std::{ future::Future, io::{self, ErrorKind, Write as _}, mem::take, + ops::ControlFlow, path::{Component, MAIN_SEPARATOR, Path, PathBuf, Prefix}, sync::{Arc, LazyLock, Weak}, }; @@ -26,8 +27,8 @@ use tracing::Instrument; use turbo_rcstr::{RcStr, rcstr}; use turbo_tasks::{ CapturedEffect, Effect, EffectExt, EffectStateStorage, InvalidationReason, NonLocalValue, - ReadRef, ResolvedVc, TurboTasksApi, ValueToString, Vc, debug::ValueDebugFormat, parallel, - trace::TraceRawVcs, turbo_tasks_weak, turbobail, + OperationVc, ReadRef, ResolvedVc, TurboTasksApi, ValueToString, Vc, debug::ValueDebugFormat, + parallel, trace::TraceRawVcs, turbo_tasks_weak, turbobail, }; use turbo_tasks_hash::{hash_xxh3_hash64, hash_xxh3_hash128}; use turbo_unix_path::{normalize_path, sys_to_unix, unix_to_sys}; @@ -35,9 +36,9 @@ use turbo_unix_path::{normalize_path, sys_to_unix, unix_to_sys}; #[cfg(windows)] use crate::windows::{is_link_junction_point, to_verbatim_with_case_folded_disk}; use crate::{ - AnyhowWrapper, File, FileComparison, FileContent, FileMeta, FileSystem, FileSystemPath, - LinkContent, LinkTarget, PersistedFileContent, RawDirectoryContent, RawDirectoryEntry, - WriteLinkContent, WriteLinkTarget, WriteLinkTargetType, + AnyhowWrapper, DiskFileSystemMap, File, FileComparison, FileContent, FileMeta, FileSystem, + FileSystemPath, LinkContent, LinkTarget, PersistedFileContent, RawDirectoryContent, + RawDirectoryEntry, WriteLinkContent, WriteLinkTargetType, invalidation::Write, invalidator_map::InvalidatorMap, mutex_map::MutexMap, @@ -253,6 +254,7 @@ pub(crate) struct DiskFileSystemInner { #[turbo_tasks(debug_ignore, trace_ignore)] #[bincode(skip)] effect_state_storage: EffectStateStorage, + map: OperationVc, } impl DiskFileSystemInner { @@ -348,7 +350,7 @@ impl DiskFileSystemInner { /// Invalidates every tracked file in the filesystem. /// - /// Calls the given + /// Calls the given `reason` closure to find the [`InvalidationReason`]. pub(crate) fn invalidate_with_reason( &self, reason: impl Fn(&Path) -> R + Sync, @@ -444,21 +446,55 @@ impl DiskFileSystemInner { invalidator.invalidate_with_reason(&*turbo_tasks, reason) }); } +} - #[tracing::instrument(level = "info", name = "start filesystem watching", skip_all, fields(path = %self.root))] - async fn start_watching_internal(self: &Arc) -> Result<()> { - let root_path = self.root_path().to_path_buf(); - - // create the directory for the filesystem on disk, if it doesn't exist - retry_blocking(|| std::fs::create_dir_all(&root_path)) - .instrument(tracing::info_span!("create root directory", name = ?root_path)) - .concurrency_limited(&self.write_semaphore) - .await?; +#[turbo_tasks::value(transparent)] +struct OptionRcStr(Option); - DiskWatcher::start_watching(self.clone()).await?; +/// Canonicalizes successive prefixes of `target_sys_path`, from the system root toward the full +/// path, and passes each canonical prefix together with the untouched suffix to `visit`. +/// +/// Helper for [`DiskFileSystem::resolve_link_target_ancestry_slow_path`] and +/// [`DiskFileSystem::lookup_in_file_system_map`] +async fn visit_canonicalized_ancestry( + target_sys_path: &Path, + mut visit: impl FnMut(&Path, &Path) -> ControlFlow>, +) -> Result> { + // Canonicalization here is an untracked read of state the watcher can't see (outside the + // filesystem root), and is not portable across machines, hence it is `session_dependent`. + #[turbo_tasks::function(fs, session_dependent)] + async fn canonicalize_untracked(sys_path: RcStr) -> Vc { + Vc::cell( + retry_blocking(|| canonicalize_to_rcstr(Path::new(&*sys_path))) + .await + .ok(), + ) + } - Ok(()) + // Reversed, `ancestors` yields every prefix of the target, from the system root (e.g. `/` + // or `\\?\C:\`) down to the full target path. `skip(1)` skips the bare system root: it has + // no symlink/short-name/casing ambiguity to resolve. Each prefix borrows from + // `target_sys_path`, so no paths are copied here. + let ancestors: SmallVec<[&Path; 8]> = target_sys_path.ancestors().collect(); + for prefix in ancestors.into_iter().rev().skip(1) { + let Some(prefix_str) = prefix.to_str() else { + return Ok(None); + }; + let Some(canonical) = canonicalize_untracked(RcStr::from(prefix_str)) + .owned() + .await? + else { + return Ok(None); + }; + let rest = target_sys_path + .strip_prefix(prefix) + .expect("`ancestors` yields prefixes of `target_sys_path`"); + if let ControlFlow::Break(result) = visit(Path::new(canonical.as_str()), rest) { + return Ok(result); + } } + + Ok(None) } /// `DiskFileSystem` carries serializable fields (`name`, `root`, @@ -519,7 +555,7 @@ impl DiskFileSystem { } pub async fn start_watching(&self) -> Result<()> { - self.inner.start_watching_internal().await + DiskWatcher::start_watching(self.inner.clone()).await } pub async fn stop_watching(&self) { @@ -650,51 +686,44 @@ impl DiskFileSystem { vc_self: ResolvedVc, target_sys_path: &Path, ) -> Result> { - #[turbo_tasks::value(transparent)] - struct OptionRcStr(Option); - - /// Canonicalization here is an untracked read of state the watcher can't see (outside the - /// root), and is not portable across machines, hence it is `session_dependent`. - #[turbo_tasks::function(fs, session_dependent)] - async fn canonicalize_untracked(sys_path: RcStr) -> Vc { - Vc::cell( - retry_blocking(|| canonicalize_to_rcstr(Path::new(&*sys_path))) - .await - .ok(), - ) - } - let root_sys_path = self.inner.root_path(); - - // Reversed, `ancestors` yields every prefix of the target, from the system root (e.g. `/` - // or `\\?\C:\`) down to the full target path. `skip(1)` skips the bare system root: it has - // no symlink/short-name/casing ambiguity to resolve. Each prefix borrows from - // `target_sys_path`, so no paths are copied here. - let ancestors: SmallVec<[&Path; 8]> = target_sys_path.ancestors().collect(); - for prefix in ancestors.into_iter().rev().skip(1) { - let Some(prefix_str) = prefix.to_str() else { - // non-unicode: `read_link` will treat this as `LinkContent::Invalid` - return Ok(None); - }; - let Some(canonical) = canonicalize_untracked(RcStr::from(prefix_str)) - .owned() - .await? - else { - return Ok(None); - }; - let canonical = Path::new(canonical.as_str()); + visit_canonicalized_ancestry(target_sys_path, |canonical, rest| { if canonical.starts_with(root_sys_path) { // Reached the filesystem root. Keep the rest of the target as spelled and let // `try_from_sys_path` strip the root prefix lexically. - let rest = target_sys_path - .strip_prefix(prefix) - .expect("`ancestors` yields prefixes of `target_sys_path`"); - return Ok(self.try_from_sys_path(vc_self, &canonical.join(rest), None)); + ControlFlow::Break(self.try_from_sys_path(vc_self, &canonical.join(rest), None)) + } else { + ControlFlow::Continue(()) } + }) + .await + } + + /// Looks up a system path in any configured filesystem other than the current filesystem. + /// + /// Like [`Self::resolve_link_target_ancestry_slow_path`], the fallback handles paths whose + /// spelling differs from a configured root. + async fn lookup_in_file_system_map( + &self, + vc_self: ResolvedVc, + target_sys_path: &Path, + ) -> Result> { + let map = self.inner.map.connect().await?; + if !map.has_file_system_other_than(vc_self) { + return Ok(None); + } + if let Some(path) = map.lookup(target_sys_path) { + return Ok(Some(path)); } - // The whole path was consumed without reaching the filesystem root. - Ok(None) + visit_canonicalized_ancestry(target_sys_path, |canonical, rest| { + if let Some(path) = map.lookup(&canonical.join(rest)) { + ControlFlow::Break(Some(path)) + } else { + ControlFlow::Continue(()) + } + }) + .await } } @@ -721,30 +750,40 @@ pub(crate) fn format_absolute_fs_path(path: &Path, name: &str, root_path: &Path) impl DiskFileSystem { /// Create a new instance of `DiskFileSystem`. /// - /// `name` is a display name for the filesystem. This should be unique. `root` is the - /// [canonicalized][std::fs::canonicalize] root of the filesystem. + /// `name` is a display name for the filesystem. This should be unique. + /// + /// `root` is the [canonicalized][std::fs::canonicalize] root of the filesystem. It should have + /// a stable [cell identity][`ResolvedVc`] to avoid invalidating every path if the root changes. /// /// This API does not canonicalize itself, as that requires IO operations (e.g. symlink /// resolution) which should (ideally) not be cached. pub fn new(name: RcStr, root: Vc) -> Vc { - Self::new_internal(name, root, Vec::new(), DiskWatcherConfig::default()) + Self::new_internal( + name, + root, + Vec::new(), + DiskWatcherConfig::default(), + DiskFileSystemMap::empty(), + ) } - /// Create a new instance of `DiskFileSystem`. + /// An extended version of [`DiskFileSystem::new`]. /// - /// `name` is a display name for the filesystem. This should be unique. `root` is the - /// [canonicalized][std::fs::canonicalize] root of the filesystem. + /// `denied_paths` contains normalized Unix-style paths relative to `root` that + /// [`DiskFileSystem`] will treat as nonexistent, disallowing reads of files in those + /// directories. /// - /// This API does not canonicalize itself, as that requires IO operations (e.g. symlink - /// resolution) which should (ideally) not be cached. + /// `watcher_config` controls how filesystem changes are detected and reported. See + /// [`DiskWatcherConfig`]. /// - /// `denied_paths` is a list of paths that are not allowed to be accessed or navigated to. These - /// must be normalized unix-style paths, non-empty and relative to the fs root. + /// `map` provides other configured filesystems used to resolve symlink targets that leave + /// this filesystem's root. pub fn new_with_options( name: RcStr, root: Vc, denied_paths: Vec, watcher_config: DiskWatcherConfig, + map: OperationVc, ) -> Vc { for denied_path in &denied_paths { debug_assert!(!denied_path.is_empty(), "denied_path must not be empty"); @@ -753,7 +792,7 @@ impl DiskFileSystem { "denied_path must be normalized: {denied_path:?}" ); } - Self::new_internal(name, root, denied_paths, watcher_config) + Self::new_internal(name, root, denied_paths, watcher_config, map) } } @@ -765,6 +804,7 @@ impl DiskFileSystem { root: Vc, denied_paths: Vec, watcher_config: DiskWatcherConfig, + map: OperationVc, ) -> Result> { let root = root.owned().await?; let instance = DiskFileSystem { @@ -782,6 +822,7 @@ impl DiskFileSystem { turbo_tasks: turbo_tasks_weak(), tokio_handle: Handle::current(), effect_state_storage: EffectStateStorage::default(), + map, }), }; @@ -961,6 +1002,7 @@ impl FileSystem for DiskFileSystem { } let target = if target_sys_path.is_absolute() { + let raw = RcStr::from(sys_to_unix(target_sys_path.to_string_lossy().as_ref())); // First try a cheap, purely lexical conversion of the raw target. `relative_to` is // ignored for absolute targets. let mut target_fs_path = this.try_from_sys_path(self, &target_sys_path, None); @@ -974,6 +1016,11 @@ impl FileSystem for DiskFileSystem { .resolve_link_target_ancestry_slow_path(self, &target_sys_path) .await?; } + if target_fs_path.is_none() { + target_fs_path = this + .lookup_in_file_system_map(self, &target_sys_path) + .await?; + } let Some(target_fs_path) = target_fs_path else { // The target leaves the filesystem root (or is a dangling link whose parent @@ -988,6 +1035,7 @@ impl FileSystem for DiskFileSystem { }; // Rewrite from the sys root to the DiskFileSystem root. LinkTarget::Absolute { + raw, resolved: target_fs_path, } } else { @@ -1040,11 +1088,24 @@ impl FileSystem for DiskFileSystem { // in; resolving that needs the names of the root's own ancestors, which a // root-relative `FileSystemPath` doesn't carry. Rejecting it here is what lets // `LinkTarget` carry a resolved path at all. - let Some(resolved) = fs_path.parent().try_join(&raw) else { - return Ok(LinkContent::Invalid { - reason: rcstr!("the symlink target leaves the filesystem root"), - } - .cell()); + let resolved = if let Some(resolved) = fs_path.parent().try_join(&raw) { + resolved + } else { + let absolute_target = this + .to_sys_path_raw(&fs_path.parent()) + .join(&target_sys_path) + .normalize_lexically() + .ok(); + let Some(resolved) = (match absolute_target { + Some(path) => this.lookup_in_file_system_map(self, &path).await?, + None => None, + }) else { + return Ok(LinkContent::Invalid { + reason: rcstr!("the symlink target leaves the configured filesystem roots"), + } + .cell()); + }; + resolved }; LinkTarget::Relative { raw, resolved } }; @@ -1310,17 +1371,44 @@ impl FileSystem for DiskFileSystem { if this.inner.is_path_denied(&fs_path) { turbobail!("Cannot write link to denied path: {fs_path}"); } - let full_path = this.to_sys_path_raw(&fs_path); + let full_path = Arc::new(this.to_sys_path_raw(&fs_path)); validate_path_length(&full_path)?; - let content_hash = hash_xxh3_hash128(&*target.await?); + let content = target.await?; + + let target_fs = ResolvedVc::try_downcast_type::(content.target.fs) + .context("link target must use a disk filesystem")? + .await?; + let target_abs_sys_path = target_fs.to_sys_path_raw(&content.target); + let target_type = content.target_type.clone(); + let is_directory = matches!(target_type, WriteLinkTargetType::DirectoryOrJunctionPoint); + // Prefer to write relative links. + // + // Windows: Junction points require absolute paths. `pathdiff` may return an absolute path + // if paths cross drives. + let target_sys_path = if cfg!(windows) && is_directory { + None + } else { + full_path + .parent() + .and_then(|parent| pathdiff::diff_paths(&target_abs_sys_path, parent)) + }; + let target_sys_path = match target_sys_path { + Some(target_sys_path) if target_sys_path.as_os_str().is_empty() => PathBuf::from("."), + Some(target_sys_path) => target_sys_path, + None => target_abs_sys_path, + }; + let target_sys_path = Arc::new(target_sys_path); + let content_hash = + hash_xxh3_hash128((target_sys_path.as_os_str().as_encoded_bytes(), &target_type)); #[turbo_tasks::value(eq = "manual", cell = "new")] struct WriteLinkEffect { - full_path: Arc, fs: ResolvedVc, - target: ResolvedVc, + full_path: Arc, + target_sys_path: Arc, + target_type: WriteLinkTargetType, content_hash: u128, } @@ -1328,19 +1416,20 @@ impl FileSystem for DiskFileSystem { #[turbo_tasks::value_impl] impl Effect for WriteLinkEffect { async fn capture(&self) -> Result> { + // Untracked, a tracked read of this cell occurred in the write effect so if it + // somehow changes the effect will be re-emitted let inner = (*self.fs).untracked().await?.inner.clone(); - // Skip target materialization if the per-key effect state already records + // Skip the write entirely if the per-key effect state already records // `Applied { value_hash }` matching our hash. See `WriteEffect::capture`. - let key_bytes: Box<[u8]> = self.full_path.as_os_str().as_encoded_bytes().into(); + let key_bytes = self.full_path.as_os_str().as_encoded_bytes(); let content = if inner .effect_state_storage - .matches_applied(&key_bytes, self.content_hash) + .matches_applied(key_bytes, self.content_hash) { None } else { - // Untracked — see `WriteEffect::capture`. - Some((*self.target).untracked().await?) + Some((self.target_sys_path.clone(), self.target_type.clone())) }; Ok(Box::new(CapturedWriteLinkEffect { full_path: self.full_path.clone(), @@ -1356,7 +1445,7 @@ impl FileSystem for DiskFileSystem { struct CapturedWriteLinkEffect { full_path: Arc, inner: Arc, - content: Option>, + content: Option<(Arc, WriteLinkTargetType)>, content_hash: u128, } @@ -1382,59 +1471,30 @@ impl FileSystem for DiskFileSystem { } impl CapturedWriteLinkEffect { - async fn apply_inner(&self, content: &ReadRef) -> anyhow::Result<()> { + async fn apply_inner( + &self, + target: &(Arc, WriteLinkTargetType), + ) -> anyhow::Result<()> { let full_path = self.full_path.clone(); let _lock = self.inner.lock_path(full_path.clone()).await; - let WriteLinkContent { - target, - target_type, - } = &**content; + let (target, target_type) = target; + #[cfg(windows)] let is_directory = matches!(target_type, WriteLinkTargetType::DirectoryOrJunctionPoint); - let target = match target { - WriteLinkTarget::Absolute(target) => { - self.inner.root_path().join(unix_to_sys(target).as_ref()) - } - WriteLinkTarget::Relative(target) => { - let relative_target = PathBuf::from(unix_to_sys(target).as_ref()); - if cfg!(windows) && is_directory { - // Windows junction points must always be stored as absolute - full_path - .parent() - .unwrap_or(&full_path) - .join(relative_target) - } else { - relative_target - } - } - }; + #[cfg(not(windows))] + let _ = target_type; - let old_content = match retry_blocking(|| std::fs::read_link(&**full_path)) + let old_content = retry_blocking(|| std::fs::read_link(&**full_path)) .instrument(tracing::info_span!("read symlink before write", name = ?full_path)) .concurrency_limited(&self.inner.read_semaphore) .await - { - Ok(res) => Some((res.is_absolute(), res)), - Err(_) => None, - }; - #[cfg(not(windows))] - let is_equal = match &old_content { - Some((old_is_absolute, old_target)) => { - target == *old_target && target.is_absolute() == *old_is_absolute - } - None => false, - }; + .ok(); + let is_equal = old_content.as_deref() == Some(&**target); #[cfg(windows)] - let is_equal = match &old_content { - Some((old_is_absolute, old_target)) => { - target == *old_target - && target.is_absolute() == *old_is_absolute - && is_link_junction_point(&full_path).ok() == Some(is_directory) - } - None => false, - }; + let is_equal = + is_equal && is_link_junction_point(&full_path).ok() == Some(is_directory); if is_equal { return Ok(()); } @@ -1471,14 +1531,14 @@ impl FileSystem for DiskFileSystem { has_old_content = false; } #[cfg(all(not(windows), not(target_os = "wasi")))] - let io_result = std::os::unix::fs::symlink(&target, &**full_path); + let io_result = std::os::unix::fs::symlink(&**target, &**full_path); #[cfg(target_os = "wasi")] - let io_result = std::os::wasi::fs::symlink_path(&target, &**full_path); + let io_result = std::os::wasi::fs::symlink_path(&**target, &**full_path); #[cfg(windows)] let io_result = if is_directory { - std::os::windows::fs::junction_point(&target, &**full_path) + std::os::windows::fs::junction_point(&**target, &**full_path) } else { - std::os::windows::fs::symlink_file(&target, &**full_path) + std::os::windows::fs::symlink_file(&**target, &**full_path) }; io_result.map_err(|err| { match err.kind() { @@ -1543,9 +1603,10 @@ impl FileSystem for DiskFileSystem { } WriteLinkEffect { - full_path: Arc::new(full_path), fs: self, - target, + full_path, + target_sys_path, + target_type, content_hash, } .resolved_cell() @@ -1748,7 +1809,7 @@ mod tests { use crate::{DirectoryContent, FileContent, RawDirectoryContent}; use crate::{ DiskFileSystem, FileSystem, FileSystemEntryType, FileSystemPath, LinkContent, - LinkTarget, RealPathErrorType, WriteLinkContent, WriteLinkTarget, WriteLinkTargetType, + LinkTarget, RealPathErrorType, WriteLinkContent, WriteLinkTargetType, canonicalize_to_rcstr, }; @@ -1758,11 +1819,13 @@ mod tests { path: FileSystemPath, target: RcStr, ) -> anyhow::Result<()> { + let file_target = path.join(&format!("{target}/data.txt"))?; + let directory_target = path.join(&target)?; let write_file = |f| { fs.write_link( f, WriteLinkContent { - target: WriteLinkTarget::Relative(format!("{target}/data.txt").into()), + target: file_target.clone(), target_type: WriteLinkTargetType::FileNonPortable, } .cell(), @@ -1776,7 +1839,7 @@ mod tests { fs.write_link( f, WriteLinkContent { - target: WriteLinkTarget::Relative(target.clone()), + target: directory_target.clone(), target_type: WriteLinkTargetType::DirectoryOrJunctionPoint, } .cell(), @@ -1786,6 +1849,16 @@ mod tests { write_dir(path.join("symlink-dir")?).await?; write_dir(path.join("symlink-dir")?).await?; + fs.write_link( + path.join("symlink-parent")?, + WriteLinkContent { + target: path, + target_type: WriteLinkTargetType::DirectoryOrJunctionPoint, + } + .cell(), + ) + .await?; + Ok(()) } @@ -1833,6 +1906,11 @@ mod tests { read_to_string(path.join("symlink-dir/data.txt")).unwrap(), "foo" ); + #[cfg(not(windows))] + assert_eq!( + std::fs::read_link(path.join("symlink-parent")).unwrap(), + std::path::PathBuf::from(".") + ); // Write the same links again but with different targets read_strongly_consistent_and_apply_effects( @@ -2131,8 +2209,8 @@ mod tests { /// A relative target must stay inside the filesystem root at every step, not just at the /// end. Both of these step above the root; one comes back into it and one doesn't, but - /// neither can be resolved against a root-relative [`FileSystemPath`], so `read_link` - /// rejects both and every [`LinkContent::Link`] stays resolvable by construction. + /// neither resolves into a configured filesystem, so `read_link` rejects both and every + /// [`LinkContent::Link`] stays resolvable by construction. #[cfg(unix)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_read_escaping_relative_symlink() { @@ -2172,13 +2250,13 @@ mod tests { root_path: FileSystemPath, ) -> anyhow::Result<()> { // sub/link-reentrant -> ../..//root.txt, which steps above the root - // and back down into it. Resolving this would need the names of the root's own - // ancestors, which a root-relative path doesn't carry. + // and back down into it. It cannot be resolved through the configured filesystem + // map because no filesystem owns the path while it is outside the root. let reentrant = fs.read_link(root_path.join("sub/link-reentrant")?).await?; assert!(matches!( &*reentrant, LinkContent::Invalid { reason } - if reason == "the symlink target leaves the filesystem root" + if reason == "the symlink target leaves the configured filesystem roots" )); // sub/link-sideways -> ../../sibling/root.txt, which steps above the root and down @@ -2187,7 +2265,7 @@ mod tests { assert!(matches!( &*sideways, LinkContent::Invalid{reason} - if reason == "the symlink target leaves the filesystem root" + if reason == "the symlink target leaves the configured filesystem roots" )); // `\` is a legal filename character on unix, so a raw target may contain one. It @@ -2342,14 +2420,12 @@ mod tests { ) -> anyhow::Result<()> { // link-via-alias -> /alias/foo.txt (resolves to /foo.txt) let via_alias = fs.read_link(root_path.join("link-via-alias")?).await?; - assert_eq!( - *via_alias, + assert!(matches!( + &*via_alias, LinkContent::Link { - target: LinkTarget::Absolute { - resolved: root_path.join("foo.txt")?, - }, - } - ); + target: LinkTarget::Absolute { resolved, .. }, + } if resolved == &root_path.join("foo.txt")? + )); // link-outside -> /outside.txt (outside of the fs root) let outside = fs.read_link(root_path.join("link-outside")?).await?; @@ -2409,11 +2485,12 @@ mod tests { .map(|(symlink_idx, target_idx)| { let target = RcStr::from(format!("../_targets/{target_idx}")); let symlink_path = symlinks_dir.join(&symlink_idx.to_string()).unwrap(); + let target = symlinks_dir.join(&target).unwrap(); async move { fs.write_link( symlink_path, WriteLinkContent { - target: WriteLinkTarget::Relative(target), + target, target_type: WriteLinkTargetType::DirectoryOrJunctionPoint, } .cell(), @@ -2509,8 +2586,8 @@ mod tests { use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage}; use crate::{ - DirectoryContent, DiskFileSystem, DiskWatcherConfig, File as TurboFile, FileContent, - FileSystem, FileSystemPath, + DirectoryContent, DiskFileSystem, DiskFileSystemMap, DiskWatcherConfig, + File as TurboFile, FileContent, FileSystem, FileSystemPath, glob::{Glob, GlobOptions}, }; @@ -2568,6 +2645,7 @@ mod tests { Vc::cell(root), vec![denied_path], DiskWatcherConfig::default(), + DiskFileSystemMap::empty(), ); let root_path = fs.root().await?; @@ -2632,6 +2710,7 @@ mod tests { Vc::cell(root), vec![denied_path], DiskWatcherConfig::default(), + DiskFileSystemMap::empty(), ); let root_path = fs.root().await?; @@ -2695,6 +2774,7 @@ mod tests { Vc::cell(root), vec![denied_path], DiskWatcherConfig::default(), + DiskFileSystemMap::empty(), ); let root_path = fs.root().await?; @@ -2782,6 +2862,7 @@ mod tests { Vc::cell(root), vec![denied_path], DiskWatcherConfig::default(), + DiskFileSystemMap::empty(), ); let root_path = fs.root().await?; let allowed_file = root_path.join(&file_path)?; @@ -2802,6 +2883,7 @@ mod tests { Vc::cell(root), vec![denied_path], DiskWatcherConfig::default(), + DiskFileSystemMap::empty(), ); let root_path = fs.root().await?; diff --git a/turbopack/crates/turbo-tasks-fs/src/fs_map.rs b/turbopack/crates/turbo-tasks-fs/src/fs_map.rs new file mode 100644 index 000000000000..fcacef870bdf --- /dev/null +++ b/turbopack/crates/turbo-tasks-fs/src/fs_map.rs @@ -0,0 +1,101 @@ +use std::{ + collections::BTreeMap, + ops::Bound, + path::{Path, PathBuf}, +}; + +use turbo_rcstr::RcStr; +use turbo_tasks::{OperationVc, ResolvedVc, Vc}; +use turbo_unix_path::sys_to_unix; + +use crate::{DiskFileSystem, FileSystemPath}; + +/// An ordered set of canonical system roots and their owning filesystems. +/// +/// The roots must not overlap: no root may be an ancestor of another root. [`Self::lookup`] +/// relies on this invariant when selecting the nearest preceding root in path order. +#[turbo_tasks::value(shared)] +pub struct DiskFileSystemMap(BTreeMap>); + +impl FromIterator<(PathBuf, ResolvedVc)> for DiskFileSystemMap { + fn from_iter)>>(iter: T) -> Self { + let filesystems = BTreeMap::from_iter(iter); + let mut map = DiskFileSystemMap(BTreeMap::new()); + for (root, fs) in filesystems { + assert!( + map.lookup(&root).is_none(), + "filesystem root {} overlaps another filesystem root", + root.display() + ); + map.0.insert(root, fs); + } + map + } +} + +impl DiskFileSystemMap { + pub fn has_file_system_other_than(&self, current: ResolvedVc) -> bool { + self.0.values().any(|file_system| *file_system != current) + } + + /// Converts an absolute system path into a path owned by one of the installed filesystems. + /// + /// Returns `None` if the file path does not exist inside any other root, or if the relative + /// path would not be valid unicode. + pub fn lookup(&self, path: &Path) -> Option { + let (root, fs) = self.0.upper_bound(Bound::Included(path)).peek_prev()?; + let relative = path.strip_prefix(root).ok()?.to_str()?; + Some(FileSystemPath::new_normalized_unchecked( + ResolvedVc::upcast(*fs), + RcStr::from(sys_to_unix(relative)), + )) + } + + /// Creates a new empty `DiskFileSystemMap`, used when constructing a [`DiskFileSystem`] that + /// cannot traverse to any other roots outside of itself. + pub fn empty() -> OperationVc { + #[turbo_tasks::function(operation)] + pub fn operation() -> Vc { + DiskFileSystemMap(BTreeMap::new()).cell() + } + operation() + } +} + +#[cfg(test)] +mod tests { + use turbo_rcstr::rcstr; + use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage}; + + use super::*; + + #[tokio::test] + async fn component_safe_lookup() { + #[turbo_tasks::function(operation, root)] + async fn assert_component_safe_lookup() -> anyhow::Result<()> { + let fs = DiskFileSystem::new(rcstr!("root"), Vc::cell(rcstr!("/tmp/root"))) + .to_resolved() + .await?; + let map: DiskFileSystemMap = [(PathBuf::from("/tmp/root"), fs)].into_iter().collect(); + assert!(!map.has_file_system_other_than(fs)); + assert_eq!( + map.lookup(Path::new("/tmp/root/file")).unwrap().path, + "file" + ); + assert!(map.lookup(Path::new("/tmp/root-other/file")).is_none()); + Ok(()) + } + + let tt = turbo_tasks::TurboTasks::new(TurboTasksBackend::new( + BackendOptions::default(), + noop_backing_storage(), + )); + tt.run_once(async { + assert_component_safe_lookup() + .read_strongly_consistent() + .await + }) + .await + .unwrap(); + } +} diff --git a/turbopack/crates/turbo-tasks-fs/src/lib.rs b/turbopack/crates/turbo-tasks-fs/src/lib.rs index ea6334f2f126..b9659402301e 100644 --- a/turbopack/crates/turbo-tasks-fs/src/lib.rs +++ b/turbopack/crates/turbo-tasks-fs/src/lib.rs @@ -19,6 +19,7 @@ mod content; mod disk; pub mod embed; mod error; +mod fs_map; pub mod glob; mod globset; pub mod invalidation; @@ -56,10 +57,10 @@ pub(crate) use crate::{ pub use crate::{ content::{ File, FileContent, FileJsonContent, FileLine, FileLinesContent, FileMeta, LinkContent, - LinkTarget, Permissions, PersistedFileContent, WriteLinkContent, WriteLinkTarget, - WriteLinkTargetType, + LinkTarget, Permissions, PersistedFileContent, WriteLinkContent, WriteLinkTargetType, }, disk::{DiskFileSystem, canonicalize_to_rcstr, validate_path_length}, + fs_map::DiskFileSystemMap, null_fs::NullFileSystem, path::{ FileSystemPath, FileSystemPathOption, RealPathError, RealPathErrorType, diff --git a/turbopack/crates/turbo-tasks-fs/src/path.rs b/turbopack/crates/turbo-tasks-fs/src/path.rs index 8f76e0efec1c..f7c3fe47c0ff 100644 --- a/turbopack/crates/turbo-tasks-fs/src/path.rs +++ b/turbopack/crates/turbo-tasks-fs/src/path.rs @@ -11,7 +11,6 @@ use turbo_tasks::{ Completion, NonLocalValue, ResolvedVc, ValueToString, ValueToStringRef, Vc, trace::TraceRawVcs, turbobail, turbofmt, }; -use turbo_tasks_hash::HashAlgorithm; use turbo_unix_path::{ get_parent_path, get_relative_path_to, get_relative_request_to, join_path, normalize_path, }; @@ -430,15 +429,6 @@ impl FileSystemPath { self.fs().read(self.clone()).parse_json5() } - /// Hashes the file content (but not as a byte-exact content hash). This does NOT follow - /// symlinks, so use this when you only want the hash of the file itself, not whatever it - /// might point to. - /// - /// This is basically `isSymlink ? self.read_link().hash() : self.read().hash()`. - pub fn hash_file(&self, salt: Vc, algorithm: HashAlgorithm) -> Vc { - hash_file(self.clone(), salt, algorithm) - } - /// Reads content of a directory. /// /// DETERMINISM: Result is in random order. Either sort result or do not @@ -662,7 +652,7 @@ async fn realpath_with_links(path: FileSystemPath) -> Result = IndexSet::new(); - let mut visited: AutoSet = AutoSet::new(); + let mut visited: AutoSet = AutoSet::new(); // Pick some arbitrary symlink depth limit... similar to the ELOOP logic for realpath(3). // SYMLOOP_MAX is 40 for Linux: https://unix.stackexchange.com/q/721724 for _i in 0..40 { @@ -675,7 +665,7 @@ async fn realpath_with_links(path: FileSystemPath) -> Result = symlinks.into_iter().collect(); return Ok(error_result( original_path, @@ -763,25 +753,6 @@ async fn realpath_with_links(path: FileSystemPath) -> Result, - algorithm: HashAlgorithm, -) -> Result> { - match *path.get_type().await? { - FileSystemEntryType::File => Ok(path.read().hash(salt, algorithm)), - FileSystemEntryType::Symlink => Ok(path.read_link().hash(salt, algorithm)), - FileSystemEntryType::NotFound | FileSystemEntryType::Error => { - // Should this rather be `return None`? - turbobail!("Cannot hash content of missing path {path}") - } - FileSystemEntryType::Directory | FileSystemEntryType::Other => { - turbobail!("Cannot hash content of non-file path {path}") - } - } -} - #[cfg(test)] mod tests { use turbo_rcstr::rcstr; @@ -896,170 +867,4 @@ mod tests { .await .unwrap() } - - mod hash_file { - use std::{ - fs::{create_dir_all, write}, - path::Path, - }; - - use turbo_tasks::OperationVc; - - use super::*; - use crate::DiskFileSystem; - - /// Creates a symbolic link, mirroring the platform handling of the `read_glob` tests. On - /// Windows a link to a directory is created as a junction point, which requires an - /// absolute target. - fn symlink(target: &Path, link: &Path) -> std::io::Result<()> { - #[cfg(unix)] - { - std::os::unix::fs::symlink(target, link) - } - #[cfg(windows)] - { - if std::fs::metadata(target).is_ok_and(|metadata| metadata.is_dir()) { - assert!( - target.is_absolute(), - "a junction point needs an absolute target" - ); - std::os::windows::fs::junction_point(target, link) - } else { - std::os::windows::fs::symlink_file(target, link) - } - } - } - - /// Two directories that hold a file of the *same name* but with *different content*, each - /// with a symlink pointing at it through the *same* relative target. Plus the entry types - /// that `hash_file` has to tell apart. - fn create_fixture(root: &Path, outside: &Path) { - write(outside.join("outside.txt"), b"outside").unwrap(); - - create_dir_all(root.join("data-a")).unwrap(); - write(root.join("data-a/value.txt"), b"aaa").unwrap(); - symlink(Path::new("value.txt"), &root.join("data-a/link")).unwrap(); - symlink( - Path::new("../data-b/value.txt"), - &root.join("data-a/link-other"), - ) - .unwrap(); - - create_dir_all(root.join("data-b")).unwrap(); - write(root.join("data-b/value.txt"), b"bbbbbb").unwrap(); - symlink(Path::new("value.txt"), &root.join("data-b/link")).unwrap(); - - create_dir_all(root.join("dir")).unwrap(); - write(root.join("dir/inside.txt"), b"inside").unwrap(); - // the regression from #97507: reading *through* this link hits the directory - symlink(&root.join("dir"), &root.join("link-dir")).unwrap(); - // a link whose target doesn't exist - symlink(Path::new("nope.txt"), &root.join("dangling")).unwrap(); - // a link whose target leaves the filesystem root - symlink(&outside.join("outside.txt"), &root.join("escaping")).unwrap(); - } - - #[turbo_tasks::function(operation, root)] - async fn hash_file_operation(disk_root: RcStr, entry: RcStr) -> Result> { - let fs = DiskFileSystem::new(rcstr!("temp"), Vc::cell(disk_root)); - let path = fs.root().await?.join(&entry)?; - Ok(path.hash_file(Vc::cell(rcstr!("salt")), HashAlgorithm::Xxh3Hash128Hex)) - } - - /// `Ok` with the hash, or `Err` with the (flattened) error message. - async fn hash_of(disk_root: &RcStr, entry: RcStr) -> Result { - let operation: OperationVc = hash_file_operation(disk_root.clone(), entry); - match operation.read_strongly_consistent().await { - Ok(hash) => Ok((*hash).clone()), - Err(err) => Err(format!("{err:#}")), - } - } - - /// `hash_file` hashes a symlink *itself* rather than what it points at, so that a link to a - /// directory can be hashed at all and so that the hash matches what consumers write out - /// (they recreate a symlink as a symlink). See #97507. - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn hashes_by_entry_type() { - let scratch = tempfile::tempdir().unwrap(); - let outside = tempfile::tempdir().unwrap(); - create_fixture(scratch.path(), outside.path()); - - let tt = turbo_tasks::TurboTasks::new(TurboTasksBackend::new( - BackendOptions::default(), - noop_backing_storage(), - )); - let disk_root: RcStr = scratch.path().to_str().unwrap().into(); - tt.run_once(async move { - let file_a = hash_of(&disk_root, rcstr!("data-a/value.txt")).await; - let file_b = hash_of(&disk_root, rcstr!("data-b/value.txt")).await; - let link_a = hash_of(&disk_root, rcstr!("data-a/link")).await; - let link_b = hash_of(&disk_root, rcstr!("data-b/link")).await; - let link_other = hash_of(&disk_root, rcstr!("data-a/link-other")).await; - let link_dir = hash_of(&disk_root, rcstr!("link-dir")).await; - let dangling = hash_of(&disk_root, rcstr!("dangling")).await; - let escaping = hash_of(&disk_root, rcstr!("escaping")).await; - let dir = hash_of(&disk_root, rcstr!("dir")).await; - let missing = hash_of(&disk_root, rcstr!("gone.txt")).await; - - // A regular file hashes its content. - let file_a = file_a.expect("a file is hashable"); - let file_b = file_b.expect("a file is hashable"); - assert_ne!(file_a, file_b, "the two files have different content"); - - // A symlink is hashable, including one that points at a directory - reading - // through that link would fail with `Is a directory (os error 21)`. - let link_a = link_a.expect("a symlink to a file is hashable"); - let link_b = link_b.expect("a symlink to a file is hashable"); - let link_other = link_other.expect("a symlink to a file is hashable"); - let link_dir = link_dir.expect("a symlink to a directory is hashable"); - // A dangling link is still a link, and so is one that leaves the root (it is - // reported as `LinkContent::Invalid`). - let dangling = dangling.expect("a dangling symlink is hashable"); - let escaping = escaping.expect("a symlink leaving the root is hashable"); - - // The link is hashed, not the file it points at: `data-a/link` and `data-b/link` - // point at files with *different content* through the *same* target, so they hash - // the same... - assert_eq!( - link_a, link_b, - "the content of the target must not affect the hash of the link" - ); - // ...while a link with a different target hashes differently. - assert_ne!( - link_a, link_other, - "the target of the link must affect the hash of the link" - ); - // ...and a link never hashes like the file it points at. - assert_ne!(link_a, file_a); - - // All of the hashes above are distinct, i.e. nothing collapses into a shared - // "symlink" hash. - let hashes = [&link_a, &link_other, &link_dir, &dangling, &escaping]; - for (index, hash) in hashes.iter().enumerate() { - for other in &hashes[index + 1..] { - assert_ne!(hash, other, "every distinct link hashes distinctly"); - } - } - - // Entries that have no content to hash are errors today. `hash_file` carries an - // open question on whether these should return `None` instead - if that changes, - // these two assertions are the ones to revisit. - assert!( - dir.as_ref() - .is_err_and(|err| err.contains("Cannot hash content of non-file path")), - "a directory is not hashable, got {dir:?}" - ); - assert!( - missing - .as_ref() - .is_err_and(|err| err.contains("Cannot hash content of missing path")), - "a missing path is not hashable, got {missing:?}" - ); - - anyhow::Ok(()) - }) - .await - .unwrap() - } - } } diff --git a/turbopack/crates/turbo-tasks-fuzz/src/fs_watcher.rs b/turbopack/crates/turbo-tasks-fuzz/src/fs_watcher.rs index 341212b11f0c..efa77fd5bb95 100644 --- a/turbopack/crates/turbo-tasks-fuzz/src/fs_watcher.rs +++ b/turbopack/crates/turbo-tasks-fuzz/src/fs_watcher.rs @@ -21,7 +21,7 @@ use turbo_tasks::{ use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage}; use turbo_tasks_fs::{ DiskFileSystem, File, FileContent, FileSystem, FileSystemPath, WriteLinkContent, - WriteLinkTarget, WriteLinkTargetType, + WriteLinkTargetType, }; // `read_or_write_all_paths_operation` always writes the sentinel values to files/symlinks. We can @@ -349,7 +349,7 @@ async fn write_link( let path_str = path.path.clone(); invalidations.0.lock().unwrap().insert(path_str); let link_content = WriteLinkContent { - target: WriteLinkTarget::Relative(target), + target: path.parent().join(&target)?, target_type: if is_directory { WriteLinkTargetType::DirectoryOrJunctionPoint } else { diff --git a/turbopack/crates/turbo-tasks-fuzz/src/symlink_stress.rs b/turbopack/crates/turbo-tasks-fuzz/src/symlink_stress.rs index fe82fd985653..aa8031a51b4c 100644 --- a/turbopack/crates/turbo-tasks-fuzz/src/symlink_stress.rs +++ b/turbopack/crates/turbo-tasks-fuzz/src/symlink_stress.rs @@ -14,8 +14,7 @@ use turbo_tasks::{ }; use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage}; use turbo_tasks_fs::{ - DiskFileSystem, FileSystem, FileSystemPath, WriteLinkContent, WriteLinkTarget, - WriteLinkTargetType, + DiskFileSystem, FileSystem, FileSystemPath, WriteLinkContent, WriteLinkTargetType, }; #[derive(Args)] @@ -240,7 +239,7 @@ async fn write_symlink( ) -> anyhow::Result<()> { let symlink_path = symlinks_dir.join(&symlink_idx.to_string())?; let link_content = WriteLinkContent { - target: WriteLinkTarget::Relative(target), + target: symlink_path.parent().join(&target)?, target_type: WriteLinkTargetType::DirectoryOrJunctionPoint, }; symlink_path diff --git a/turbopack/crates/turbo-tasks/Cargo.toml b/turbopack/crates/turbo-tasks/Cargo.toml index d7b1e0511b3a..4656c709f96d 100644 --- a/turbopack/crates/turbo-tasks/Cargo.toml +++ b/turbopack/crates/turbo-tasks/Cargo.toml @@ -53,7 +53,7 @@ scattered-collect = { workspace = true } serde = { workspace = true, features = ["rc", "derive"] } serde_json = { workspace = true } shrink-to-fit = { workspace = true, features = ["indexmap", "serde_json", "smallvec", "nightly"] } -smallvec = { workspace = true } +smallvec = { workspace = true, features = ["serde", "impl_bincode"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["full"] } tokio-util = { workspace = true } diff --git a/turbopack/crates/turbopack-cli/src/util.rs b/turbopack/crates/turbopack-cli/src/util.rs index df54a1db37ba..7e999bf77e91 100644 --- a/turbopack/crates/turbopack-cli/src/util.rs +++ b/turbopack/crates/turbopack-cli/src/util.rs @@ -4,7 +4,9 @@ use anyhow::{Context, Result}; use bincode::{Decode, Encode}; use turbo_rcstr::{RcStr, rcstr}; use turbo_tasks::{Vc, trace::TraceRawVcs}; -use turbo_tasks_fs::{DiskFileSystem, DiskWatcherConfig, FileSystem, canonicalize_to_rcstr}; +use turbo_tasks_fs::{ + DiskFileSystem, DiskFileSystemMap, DiskWatcherConfig, FileSystem, canonicalize_to_rcstr, +}; #[turbo_tasks::task_input] #[derive(Clone, Debug, Hash, PartialEq, Eq, TraceRawVcs, Encode, Decode)] @@ -61,6 +63,7 @@ pub async fn project_fs( Vc::cell(project_dir), vec![denied_root_path], DiskWatcherConfig::default(), + DiskFileSystemMap::empty(), ); if watch { disk_fs.await?.start_watching().await?; diff --git a/turbopack/crates/turbopack-core/src/asset.rs b/turbopack/crates/turbopack-core/src/asset.rs index bb9801924f0f..e38e1224ea8a 100644 --- a/turbopack/crates/turbopack-core/src/asset.rs +++ b/turbopack/crates/turbopack-core/src/asset.rs @@ -4,7 +4,7 @@ use turbo_tasks::{ResolvedVc, Vc}; use turbo_tasks_fs::{ FileContent, FileJsonContent, FileLinesContent, FileSystemPath, WriteLinkContent, }; -use turbo_tasks_hash::{HashAlgorithm, deterministic_hash}; +use turbo_tasks_hash::HashAlgorithm; use crate::version::{VersionedAssetContent, VersionedContent}; @@ -126,10 +126,9 @@ impl AssetContent { pub async fn hash(&self, salt: Vc, algorithm: HashAlgorithm) -> Result> { Ok(match self { AssetContent::File(content) => content.hash(salt, algorithm), - AssetContent::Redirect(content) => Vc::cell(RcStr::from( - // no_hash_salt - deterministic_hash(&salt.await?, content, algorithm), - )), + AssetContent::Redirect(content) => { + Vc::cell(content.hash(&*salt.await?, algorithm).await?) + } }) } diff --git a/turbopack/crates/turbopack-core/src/file_source.rs b/turbopack/crates/turbopack-core/src/file_source.rs index 4e027051d859..05b9db02e88e 100644 --- a/turbopack/crates/turbopack-core/src/file_source.rs +++ b/turbopack/crates/turbopack-core/src/file_source.rs @@ -2,8 +2,8 @@ use anyhow::{Result, bail}; use turbo_rcstr::RcStr; use turbo_tasks::Vc; use turbo_tasks_fs::{ - FileContent, FileSystemEntryType, FileSystemPath, LinkContent, LinkTarget, WriteLinkContent, - WriteLinkTarget, WriteLinkTargetType, + FileContent, FileSystemEntryType, FileSystemPath, LinkContent, WriteLinkContent, + WriteLinkTargetType, }; use crate::{ @@ -70,12 +70,6 @@ impl Asset for FileSource { match file_type { FileSystemEntryType::Symlink => match &*self.path.read_link().await? { LinkContent::Link { target } => { - let write_target = match target { - LinkTarget::Absolute { resolved } => { - WriteLinkTarget::Absolute(resolved.path.clone()) - } - LinkTarget::Relative { raw, .. } => WriteLinkTarget::Relative(raw.clone()), - }; let target_fs_path = target.file_system_path(); let write_target_type = match *target_fs_path.get_type().await? { FileSystemEntryType::Directory => { @@ -89,12 +83,17 @@ impl Asset for FileSource { _ => WriteLinkTargetType::FileNonPortable, }; Ok(AssetContent::Redirect(WriteLinkContent { - target: write_target, + target: target_fs_path.clone(), target_type: write_target_type, }) .cell()) } - _ => bail!("Invalid symlink"), + LinkContent::NotFound => { + // This should not normally happen because the path was already identified as + // a symlink, but it may be removed between get_type and read_link. + Ok(AssetContent::File(FileContent::NotFound.resolved_cell()).cell()) + } + LinkContent::Invalid { reason } => bail!("Invalid symlink: {reason}"), }, FileSystemEntryType::File => { Ok(AssetContent::File(self.path.read().to_resolved().await?).cell()) diff --git a/turbopack/crates/turbopack-core/src/rebase.rs b/turbopack/crates/turbopack-core/src/rebase.rs index 24f31363eb53..34174fb46038 100644 --- a/turbopack/crates/turbopack-core/src/rebase.rs +++ b/turbopack/crates/turbopack-core/src/rebase.rs @@ -2,7 +2,7 @@ use std::hash::Hash; use anyhow::Result; use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc, turbobail}; -use turbo_tasks_fs::FileSystemPath; +use turbo_tasks_fs::{FileSystemPath, WriteLinkContent}; use crate::{ asset::{Asset, AssetContent}, @@ -77,7 +77,23 @@ impl Asset for RebasedAsset { #[turbo_tasks::function] async fn content(&self) -> Result> { if let Some(source) = *self.module.source().await? { - Ok(source.content()) + let source_content = source.content(); + let AssetContent::Redirect(redirect) = &*source_content.await? else { + return Ok(source_content); + }; + // treat symlinks targets as relative to the symlink. When we rebase the symlink, we + // should also rebase its target path + let redirect = WriteLinkContent { + target: FileSystemPath::rebase( + redirect.target.clone(), + self.input_dir.clone(), + self.output_dir.clone(), + ) + .owned() + .await?, + target_type: redirect.target_type.clone(), + }; + Ok(AssetContent::Redirect(redirect).cell()) } else { turbobail!("Module {} has no source", self.module.ident()); } diff --git a/turbopack/crates/turbopack-core/src/resolve/mod.rs b/turbopack/crates/turbopack-core/src/resolve/mod.rs index c6b1bc665869..880b9fabda51 100644 --- a/turbopack/crates/turbopack-core/src/resolve/mod.rs +++ b/turbopack/crates/turbopack-core/src/resolve/mod.rs @@ -1397,10 +1397,9 @@ async fn find_package( for resolve_modules in &options.modules { match resolve_modules { - ResolveModules::Nested(root, names) => { + ResolveModules::Nested(names) => { let mut lookup_path = lookup_path.clone(); - let mut lookup_path_value = lookup_path.clone(); - while lookup_path_value.is_inside_ref(root) { + loop { for name in names.iter() { let fs_path = lookup_path.join(name)?; if let Some(fs_path) = dir_exists( @@ -1430,12 +1429,10 @@ async fn find_package( } } } - lookup_path = lookup_path.parent(); - let new_context_value = lookup_path.clone(); - if new_context_value == lookup_path_value { + if lookup_path.is_root() { break; } - lookup_path_value = new_context_value; + lookup_path = lookup_path.parent(); } } ResolveModules::Path { @@ -3070,13 +3067,8 @@ async fn resolve_import_map_result( alias_lookup_path.clone(), request, match ty { - // TODO is that root correct? - ExternalType::CommonJs => { - node_cjs_resolve_options(alias_lookup_path.root().owned().await?) - } - ExternalType::EcmaScriptModule => { - node_esm_resolve_options(alias_lookup_path.root().owned().await?) - } + ExternalType::CommonJs => node_cjs_resolve_options(), + ExternalType::EcmaScriptModule => node_esm_resolve_options(), ExternalType::Script | ExternalType::Url | ExternalType::Global => options, }, ) @@ -3898,7 +3890,7 @@ mod tests { let extensions = custom_extensions .unwrap_or_else(|| vec![rcstr!(".ts"), rcstr!(".js"), rcstr!(".json")]); - let mut options_value = node_esm_resolve_options(lookup_path.clone()) + let mut options_value = node_esm_resolve_options() .with_fully_specified(fully_specified) .with_extensions(extensions) .owned() diff --git a/turbopack/crates/turbopack-core/src/resolve/node.rs b/turbopack/crates/turbopack-core/src/resolve/node.rs index 07591210c5c8..2886959cbec6 100644 --- a/turbopack/crates/turbopack-core/src/resolve/node.rs +++ b/turbopack/crates/turbopack-core/src/resolve/node.rs @@ -1,6 +1,5 @@ use turbo_rcstr::rcstr; use turbo_tasks::Vc; -use turbo_tasks_fs::FileSystemPath; use super::options::{ ConditionValue, ResolutionConditions, ResolveInPackage, ResolveIntoPackage, ResolveModules, @@ -8,7 +7,7 @@ use super::options::{ }; #[turbo_tasks::function] -pub fn node_cjs_resolve_options(root: FileSystemPath) -> Vc { +pub fn node_cjs_resolve_options() -> Vc { let conditions: ResolutionConditions = [ (rcstr!("node"), ConditionValue::Set), (rcstr!("require"), ConditionValue::Set), @@ -17,7 +16,7 @@ pub fn node_cjs_resolve_options(root: FileSystemPath) -> Vc { let extensions = vec![rcstr!(".js"), rcstr!(".json"), rcstr!(".node")]; ResolveOptions { extensions, - modules: vec![ResolveModules::Nested(root, vec![rcstr!("node_modules")])], + modules: vec![ResolveModules::Nested(vec![rcstr!("node_modules")])], into_package: vec![ ResolveIntoPackage::ExportsField { conditions: conditions.clone(), @@ -38,7 +37,7 @@ pub fn node_cjs_resolve_options(root: FileSystemPath) -> Vc { } #[turbo_tasks::function] -pub fn node_esm_resolve_options(root: FileSystemPath) -> Vc { +pub fn node_esm_resolve_options() -> Vc { let conditions: ResolutionConditions = [ (rcstr!("node"), ConditionValue::Set), (rcstr!("import"), ConditionValue::Set), @@ -48,7 +47,7 @@ pub fn node_esm_resolve_options(root: FileSystemPath) -> Vc { ResolveOptions { fully_specified: true, extensions, - modules: vec![ResolveModules::Nested(root, vec![rcstr!("node_modules")])], + modules: vec![ResolveModules::Nested(vec![rcstr!("node_modules")])], into_package: vec![ ResolveIntoPackage::ExportsField { conditions: conditions.clone(), diff --git a/turbopack/crates/turbopack-core/src/resolve/options.rs b/turbopack/crates/turbopack-core/src/resolve/options.rs index 5048a6100140..699339192736 100644 --- a/turbopack/crates/turbopack-core/src/resolve/options.rs +++ b/turbopack/crates/turbopack-core/src/resolve/options.rs @@ -29,9 +29,8 @@ pub struct ExcludedExtensions(#[bincode(with = "turbo_bincode::indexset")] pub F TraceRawVcs, Hash, PartialEq, Eq, Clone, Debug, ValueDebugFormat, NonLocalValue, Encode, Decode, )] pub enum ResolveModules { - /// when inside of path, use the list of directories to - /// resolve inside these - Nested(FileSystemPath, Vec), + /// Starting from the lookup path, look for modules in these directories at each parent. + Nested(Vec), /// look into that directory, unless the request has an excluded extension Path { dir: FileSystemPath, diff --git a/turbopack/crates/turbopack-core/src/resolve/pattern.rs b/turbopack/crates/turbopack-core/src/resolve/pattern.rs index c5d0104f4a5a..cc74aab825d0 100644 --- a/turbopack/crates/turbopack-core/src/resolve/pattern.rs +++ b/turbopack/crates/turbopack-core/src/resolve/pattern.rs @@ -1860,9 +1860,14 @@ pub async fn read_matches( FileSystemEntryType::Directory ) { - results.push(( + nested.push(( pos, - PatternMatch::Directory(prefix.clone().into(), fs_path), + read_matches( + target.file_system_path().clone(), + prefix.clone().into(), + true, + pattern, + ), )); } } diff --git a/turbopack/crates/turbopack-core/src/resolve/plugin.rs b/turbopack/crates/turbopack-core/src/resolve/plugin.rs index 36bdb86bd0db..e957aaf6ffd7 100644 --- a/turbopack/crates/turbopack-core/src/resolve/plugin.rs +++ b/turbopack/crates/turbopack-core/src/resolve/plugin.rs @@ -38,7 +38,11 @@ impl AfterResolvePluginCondition { pub fn matches(&self, fs_path: &FileSystemPath) -> bool { match self { AfterResolvePluginCondition::Glob { root, glob } => { - root.get_path_to(fs_path).is_some_and(|p| glob.matches(p)) + if root.fs == fs_path.fs { + root.get_path_to(fs_path).is_some_and(|p| glob.matches(p)) + } else { + glob.matches(&fs_path.path) + } } AfterResolvePluginCondition::Always => true, AfterResolvePluginCondition::Never => false, diff --git a/turbopack/crates/turbopack-ecmascript/src/references/external_module.rs b/turbopack/crates/turbopack-ecmascript/src/references/external_module.rs index 704b2d483ad7..aa25aa75ee7e 100644 --- a/turbopack/crates/turbopack-ecmascript/src/references/external_module.rs +++ b/turbopack/crates/turbopack-ecmascript/src/references/external_module.rs @@ -1,12 +1,12 @@ use std::{borrow::Cow, fmt::Display, io::Write}; -use anyhow::{Context, Result}; +use anyhow::Result; use bincode::{Decode, Encode}; use turbo_rcstr::{RcStr, rcstr}; use turbo_tasks::{ResolvedVc, TryJoinIterExt, ValueToStringRef, Vc, trace::TraceRawVcs}; use turbo_tasks_fs::{ - FileSystem, FileSystemPath, VirtualFileSystem, WriteLinkContent, WriteLinkTarget, - WriteLinkTargetType, rope::RopeBuilder, + FileSystem, FileSystemPath, VirtualFileSystem, WriteLinkContent, WriteLinkTargetType, + rope::RopeBuilder, }; use turbo_tasks_hash::{encode_hex, hash_xxh3_hash64}; use turbopack_core::{ @@ -497,22 +497,8 @@ impl Asset for ExternalsSymlinkAsset { // path: [output]/bench/app-router-server/.next/node_modules/lodash-ee4fa714b6d81ca3 // target: [project]/node_modules/.pnpm/lodash@3.10.1/node_modules/lodash - let output_root_to_project_root = this.chunking_context.output_root_to_root_path().await?; - let project_root_to_target = &this.target.path; - - let path = self.path().await?; - let path_to_output_root = path - .parent() - .get_relative_path_to(&*this.chunking_context.output_root().await?) - .context("path must be inside output root")?; - - let target = format!( - "{path_to_output_root}/{output_root_to_project_root}/{project_root_to_target}", - ) - .into(); - Ok(AssetContent::Redirect(WriteLinkContent { - target: WriteLinkTarget::Relative(target), + target: this.target.clone(), target_type: WriteLinkTargetType::DirectoryOrJunctionPoint, }) .cell()) diff --git a/turbopack/crates/turbopack-resolve/src/resolve.rs b/turbopack/crates/turbopack-resolve/src/resolve.rs index 921384e7337c..2e15b7168764 100644 --- a/turbopack/crates/turbopack-resolve/src/resolve.rs +++ b/turbopack/crates/turbopack-resolve/src/resolve.rs @@ -1,8 +1,8 @@ use anyhow::Result; use next_taskless::{BUN_EXTERNALS, EDGE_NODE_EXTERNALS, NODE_EXTERNALS}; use turbo_rcstr::rcstr; -use turbo_tasks::{ResolvedVc, Vc}; -use turbo_tasks_fs::{FileSystem, FileSystemPath}; +use turbo_tasks::Vc; +use turbo_tasks_fs::FileSystemPath; use turbopack_core::resolve::{ AliasMap, AliasPattern, ExternalTraced, ExternalType, FindContextFileResult, find_context_file, options::{ @@ -18,12 +18,10 @@ use crate::{ #[turbo_tasks::function] async fn base_resolve_options( - fs: ResolvedVc>, options_context: Vc, ) -> Result> { let opt = options_context.await?; let emulating = opt.emulate_environment; - let root = fs.root().owned().await?; let mut direct_mappings = AliasMap::new(); let node_externals = if let Some(environment) = emulating { environment.node_externals().owned().await? @@ -154,20 +152,14 @@ async fn base_resolve_options( extensions, modules: if let Some(environment) = emulating { if *environment.resolve_node_modules().await? { - vec![ResolveModules::Nested( - root.clone(), - vec![rcstr!("node_modules")], - )] + vec![ResolveModules::Nested(vec![rcstr!("node_modules")])] } else { Vec::new() } } else { let mut mods = Vec::new(); - if let Some(dir) = &opt.enable_node_modules { - mods.push(ResolveModules::Nested( - dir.clone(), - vec![rcstr!("node_modules")], - )); + if opt.enable_node_modules.is_some() { + mods.push(ResolveModules::Nested(vec![rcstr!("node_modules")])); } mods }, @@ -228,7 +220,7 @@ pub async fn resolve_options( } } - let resolve_options = base_resolve_options(*resolve_path.fs, options_context); + let resolve_options = base_resolve_options(options_context); let resolve_options = if options_context_value.enable_typescript { let find_tsconfig = async || { diff --git a/turbopack/crates/turbopack-resolve/src/typescript.rs b/turbopack/crates/turbopack-resolve/src/typescript.rs index b7af9e17108e..d674167eb38d 100644 --- a/turbopack/crates/turbopack-resolve/src/typescript.rs +++ b/turbopack/crates/turbopack-resolve/src/typescript.rs @@ -246,7 +246,7 @@ pub async fn tsconfig_resolve_options( let configs = read_tsconfigs( tsconfig.read(), ResolvedVc::upcast(FileSource::new(tsconfig.clone()).to_resolved().await?), - node_cjs_resolve_options(tsconfig.root().owned().await?), + node_cjs_resolve_options(), ) .await?; diff --git a/turbopack/crates/turbopack-tracing/tests/node-file-trace/integration/symlink-to-file/index.js b/turbopack/crates/turbopack-tracing/tests/node-file-trace/integration/symlink-to-file/index.js index 13db1694b819..c6acc1d5e2e4 100644 --- a/turbopack/crates/turbopack-tracing/tests/node-file-trace/integration/symlink-to-file/index.js +++ b/turbopack/crates/turbopack-tracing/tests/node-file-trace/integration/symlink-to-file/index.js @@ -1,3 +1,9 @@ +const { readlinkSync } = require('fs') + +if (readlinkSync(__dirname + '/linked.js') !== 'real.js') { + throw new Error('relative symlink target was not preserved') +} + const { compute } = require('./linked') console.log(compute())