From 92afe9416f0a3450e8d9af82c749af8c1560b899 Mon Sep 17 00:00:00 2001 From: Naadir Jeewa Date: Sun, 30 Aug 2026 23:01:33 +0100 Subject: [PATCH 01/14] feat(executor): bound reader priority queues Bound interactive and maintenance queues, carry request deadlines across the SubC transport, and yield standing-root maintenance when cold-build capacity is saturated. Demote maintenance workers so reader latency remains stable under index load. Signed-off-by: Naadir Jeewa --- crates/aft/src/callgraph_store/mod.rs | 10 +- crates/aft/src/cold_build_limiter.rs | 63 +- crates/aft/src/commands/bash.rs | 1 + crates/aft/src/commands/configure.rs | 7 + crates/aft/src/commands/semantic_search.rs | 1 + crates/aft/src/executor/mod.rs | 788 +++++++++++++++--- crates/aft/src/executor/tests.rs | 402 ++++++++- crates/aft/src/gh_shim.rs | 1 + crates/aft/src/inspect/dispatch.rs | 3 + crates/aft/src/lib.rs | 2 +- crates/aft/src/logging.rs | 20 +- crates/aft/src/subc/bash.rs | 71 +- crates/aft/src/subc/health.rs | 15 + crates/aft/src/subc/mod.rs | 154 +++- crates/aft/src/subc/standing.rs | 25 +- crates/aft/src/thread_priority.rs | 282 +++++++ crates/aft/tests/callgraph_store_test.rs | 4 + .../aft/tests/integration/callgraph_test.rs | 61 +- .../aft/tests/integration/subc_bridge_test.rs | 3 + .../aft/tests/integration/subc_storm_test.rs | 158 ++++ .../src/__tests__/subc-transport.test.ts | 139 ++- packages/aft-bridge/src/subc-transport.ts | 130 ++- packages/aft-bridge/src/transport-factory.ts | 1 + 23 files changed, 2098 insertions(+), 243 deletions(-) create mode 100644 crates/aft/src/thread_priority.rs diff --git a/crates/aft/src/callgraph_store/mod.rs b/crates/aft/src/callgraph_store/mod.rs index 9e4a4a1a8..68a50bd42 100644 --- a/crates/aft/src/callgraph_store/mod.rs +++ b/crates/aft/src/callgraph_store/mod.rs @@ -1172,7 +1172,10 @@ impl RefreshWorker { let thread_shared = Arc::clone(&shared); let thread = std::thread::Builder::new() .name("aft-callgraph-refresh".to_string()) - .spawn(move || callgraph_refresh_worker_loop(&thread_shared)) + .spawn(move || { + crate::thread_priority::demote_background(); + callgraph_refresh_worker_loop(&thread_shared) + }) .expect("failed to spawn callgraph refresh worker"); Arc::new(Self { shared, @@ -8489,6 +8492,11 @@ fn build_extracts_parallel(project_root: &Path, files: &[PathBuf]) -> BuildExtra .num_threads(build_pool_size()) .thread_name(|index| format!("aft-callgraph-build-{index}")) .stack_size(8 * 1024 * 1024) + .start_handler(|_| { + // Callgraph builds are background maintenance: keep interactive + // reads ahead in the OS scheduler (CPU and I/O). + crate::thread_priority::demote_background(); + }) .build() { Ok(pool) => pool.install(run), diff --git a/crates/aft/src/cold_build_limiter.rs b/crates/aft/src/cold_build_limiter.rs index 2b8761e84..770c59c28 100644 --- a/crates/aft/src/cold_build_limiter.rs +++ b/crates/aft/src/cold_build_limiter.rs @@ -153,23 +153,19 @@ pub(crate) struct StandingColdBuildPermit { pub(crate) admission_epoch: u64, } -/// Standing performs the same waiter inspection before initial acquisition and -/// checkpoint reacquisition because both call this one function. It declines -/// immediately when an interactive or normal-maintenance waiter is visible. -pub(crate) fn acquire_standing_while_cancellable_with_limiter( +/// Immediate standing admission without waiter registration, preserving the +/// lifecycle admission epoch. Used by standing passes that can defer rejected +/// work to their next tick: a yielded pass never occupies a worker waiting for +/// a cold slot. No equivalent epoch-preserving immediate API existed before. +pub(crate) fn try_acquire_standing_with_limiter( limiter: &Arc, - kind: &str, request_id: impl Into, admission_epoch: u64, - admitted: impl Fn() -> bool, - cancelled: impl Fn() -> bool, ) -> Option { let request = ColdBuildAdmissionRequest::new(request_id, ColdBuildAdmissionClass::Standing); - acquire_blocking_while_inner(limiter, kind, Some(&request), admitted, cancelled).map(|permit| { - StandingColdBuildPermit { - _permit: permit, - admission_epoch, - } + try_acquire_classified_with_limiter(limiter, &request).map(|permit| StandingColdBuildPermit { + _permit: permit, + admission_epoch, }) } @@ -589,49 +585,6 @@ mod tests { ); } - #[test] - fn standing_yields_before_initial_and_checkpoint_reacquisition_when_non_standing_waits() { - let limiter = test_limiter(1); - let non_standing_waiter = - AdmissionWaiter::register(&limiter, ColdBuildAdmissionClass::Maintenance); - - assert!(acquire_standing_while_cancellable_with_limiter( - &limiter, - "standing-initial", - "standing-initial", - 41, - || true, - || false, - ) - .is_none()); - - drop(non_standing_waiter); - let first = acquire_standing_while_cancellable_with_limiter( - &limiter, - "standing-checkpoint", - "standing-checkpoint", - 41, - || true, - || false, - ) - .expect("standing may acquire once ordinary waiters clear"); - assert_eq!(first.admission_epoch, 41); - drop(first); - - let non_standing_waiter = - AdmissionWaiter::register(&limiter, ColdBuildAdmissionClass::InspectTriggered); - assert!(acquire_standing_while_cancellable_with_limiter( - &limiter, - "standing-reacquire", - "standing-reacquire", - 41, - || true, - || false, - ) - .is_none()); - drop(non_standing_waiter); - } - #[test] fn inspect_waiter_takes_next_release_ahead_of_queued_maintenance() { let limiter = test_limiter(1); diff --git a/crates/aft/src/commands/bash.rs b/crates/aft/src/commands/bash.rs index ef2a0b0ac..550987277 100644 --- a/crates/aft/src/commands/bash.rs +++ b/crates/aft/src/commands/bash.rs @@ -1224,6 +1224,7 @@ exec "$@" #[cfg(unix)] #[test] fn permission_retry_reclassified_as_first_party_resolves_native_plan() { + let _env_lock = crate::test_env::process_env_lock(); use crate::sandbox_spawn::{ clear_sandbox_spawn_test_seam, install_sandbox_spawn_test_seam, sandbox_spawn_test_observations, with_authenticated_principal, AuthenticatedPrincipal, diff --git a/crates/aft/src/commands/configure.rs b/crates/aft/src/commands/configure.rs index a781709b5..7c52db446 100644 --- a/crates/aft/src/commands/configure.rs +++ b/crates/aft/src/commands/configure.rs @@ -8395,7 +8395,14 @@ mod tests { #[test] fn detect_missing_tools_still_warns_explicit_formatter_when_format_on_edit_disabled() { + let _env_lock = crate::test_env::process_env_lock(); let temp = tempfile::tempdir().unwrap(); + let empty_path = temp.path().join("empty-path"); + std::fs::create_dir(&empty_path).unwrap(); + let _path_guard = EnvVarGuard::set("PATH", empty_path.to_str().unwrap()); + let _home_guard = EnvVarGuard::set("HOME", temp.path().to_str().unwrap()); + let _userprofile_guard = EnvVarGuard::set("USERPROFILE", temp.path().to_str().unwrap()); + let _well_known_guard = EnvVarGuard::set("AFT_DISABLE_WELL_KNOWN_LOOKUP", "1"); let mut config = Config { project_root: Some(temp.path().to_path_buf()), format_on_edit: false, diff --git a/crates/aft/src/commands/semantic_search.rs b/crates/aft/src/commands/semantic_search.rs index 0ece028a8..b43034f6b 100644 --- a/crates/aft/src/commands/semantic_search.rs +++ b/crates/aft/src/commands/semantic_search.rs @@ -4628,6 +4628,7 @@ mod tests { )) }); + assert_eq!(response["interpreted_as"], "lexical"); assert!(response["results"] .as_array() diff --git a/crates/aft/src/executor/mod.rs b/crates/aft/src/executor/mod.rs index 5de2bcf7a..67bd21724 100644 --- a/crates/aft/src/executor/mod.rs +++ b/crates/aft/src/executor/mod.rs @@ -86,6 +86,12 @@ const INTERACTIVE_WRITER_PROMOTION_AGE: Duration = Duration::from_secs(6); /// metadata instead of only the generic `waiting_on_readers` diagnosis. const READER_STUCK_CENSUS_AGE: Duration = Duration::from_secs(60); +/// Process-wide queue bounds. These are safety-policy limits, not throughput +/// tuning constants: the per-actor interactive cap matches +/// [`SCHEDULER_EVENT_BATCH_CAP`], the process interactive cap reserves four +/// actor batches for the release storm's default four-root topology, and the +/// process maintenance cap admits two complete legacy per-actor maintenance +/// queues while preventing aggregate growth across standing roots. #[derive(Debug, Clone)] pub struct ExecutorConfig { pub pool_size: usize, @@ -93,6 +99,9 @@ pub struct ExecutorConfig { pub actor_cap: usize, pub heavy_permits: usize, pub drr_quantum: isize, + pub interactive_queue_cap: usize, + pub interactive_actor_queue_cap: usize, + pub maintenance_queue_cap: usize, } impl Default for ExecutorConfig { @@ -111,6 +120,9 @@ impl Default for ExecutorConfig { actor_cap, heavy_permits, drr_quantum: 1, + interactive_queue_cap: 256, + interactive_actor_queue_cap: 64, + maintenance_queue_cap: 1024, } } } @@ -125,6 +137,9 @@ struct EffectiveConfig { deficit_cap: isize, interactive_reserve: usize, maintenance_cap: usize, + interactive_queue_cap: usize, + interactive_actor_queue_cap: usize, + maintenance_queue_cap: usize, } impl ExecutorConfig { @@ -155,6 +170,11 @@ impl ExecutorConfig { deficit_cap, interactive_reserve, maintenance_cap, + interactive_queue_cap: self.interactive_queue_cap.max(1), + interactive_actor_queue_cap: self.interactive_actor_queue_cap.max(1), + maintenance_queue_cap: self + .maintenance_queue_cap + .max(2usize.saturating_mul(MAINTENANCE_QUEUE_CAP)), } } } @@ -198,6 +218,14 @@ pub struct DispatchLivenessSnapshot { pub running: DispatchRunningSnapshot, pub interactive_reserve: usize, pub maintenance_cap: usize, + /// Configured queue caps (process and per-actor interactive). + pub interactive_queue_cap: usize, + pub interactive_actor_queue_cap: usize, + pub maintenance_queue_cap: usize, + /// Cumulative typed admission rejections and deadline expiries. + pub interactive_admission_rejections: u64, + pub maintenance_admission_rejections: u64, + pub deadline_expiries: u64, } /// Scheduler-owned mirror read by health probes without taking the actor map @@ -211,6 +239,9 @@ struct DispatchLivenessAtomics { maintenance_oldest_enqueued_ms_plus_one: AtomicU64, interactive_running: AtomicUsize, maintenance_running: AtomicUsize, + interactive_admission_rejections: AtomicU64, + maintenance_admission_rejections: AtomicU64, + deadline_expiries: AtomicU64, } impl DispatchLivenessAtomics { @@ -223,6 +254,9 @@ impl DispatchLivenessAtomics { maintenance_oldest_enqueued_ms_plus_one: AtomicU64::new(0), interactive_running: AtomicUsize::new(0), maintenance_running: AtomicUsize::new(0), + interactive_admission_rejections: AtomicU64::new(0), + maintenance_admission_rejections: AtomicU64::new(0), + deadline_expiries: AtomicU64::new(0), } } @@ -263,6 +297,12 @@ impl DispatchLivenessAtomics { .store(snapshot.interactive.queued, Ordering::Release); self.maintenance_queued .store(snapshot.maintenance.queued, Ordering::Release); + self.interactive_admission_rejections + .store(snapshot.interactive_admission_rejections, Ordering::Relaxed); + self.maintenance_admission_rejections + .store(snapshot.maintenance_admission_rejections, Ordering::Relaxed); + self.deadline_expiries + .store(snapshot.deadline_expiries, Ordering::Relaxed); } fn snapshot(&self, config: &EffectiveConfig) -> DispatchLivenessSnapshot { @@ -295,6 +335,16 @@ impl DispatchLivenessAtomics { }, interactive_reserve: config.interactive_reserve, maintenance_cap: config.maintenance_cap, + interactive_queue_cap: config.interactive_queue_cap, + interactive_actor_queue_cap: config.interactive_actor_queue_cap, + maintenance_queue_cap: config.maintenance_queue_cap, + interactive_admission_rejections: self + .interactive_admission_rejections + .load(Ordering::Relaxed), + maintenance_admission_rejections: self + .maintenance_admission_rejections + .load(Ordering::Relaxed), + deadline_expiries: self.deadline_expiries.load(Ordering::Relaxed), } } } @@ -666,19 +716,61 @@ impl Executor { pub fn remove_actor(&self, root_id: &ProjectRootId) { let removed = { let mut state = self.inner.state.lock(); + let removed = Self::take_actor(&mut state, root_id); state.actor_order.retain(|actor_root| actor_root != root_id); - state.actors.remove(root_id) + removed }; - if let Some(actor) = removed.as_ref() { + if let Some((actor, settled)) = removed { + for (_job_class, queued) in settled { + queued + .completion + .send(actor_fatal_response(queued.request_id)); + } let app = actor.ctx.app(); crate::root_cache::unregister_live_scope(&actor.ctx.storage_dir(), root_id.as_path()); app.unregister_memory_context(root_id.as_path(), &actor.ctx); app.actor_root_unregistered(); } - drop(removed); self.wake_scheduler(); } + /// Defensive actor extraction: drain all queued jobs, release their + /// capacity buckets, and return the actor with the settled jobs. Unexpected + /// queued work at extraction time receives `actor_fatal`. + fn take_actor( + state: &mut SchedulerState, + root_id: &ProjectRootId, + ) -> Option<(ActorState, Vec<(JobClass, QueuedJob)>)> { + let mut actor = state.actors.remove(root_id)?; + let mut settled = actor + .interactive + .fail_queued_jobs() + .into_iter() + .map(|job| (JobClass::Interactive, job)) + .collect::>(); + settled.extend( + actor + .maintenance + .fail_queued_jobs() + .into_iter() + .map(|job| (JobClass::Maintenance, job)), + ); + state.process_counts.interactive = state.process_counts.interactive.saturating_sub( + settled + .iter() + .filter(|(c, _)| *c == JobClass::Interactive) + .count(), + ); + state.process_counts.maintenance = state.process_counts.maintenance.saturating_sub( + settled + .iter() + .filter(|(c, _)| *c == JobClass::Maintenance) + .count(), + ); + state.debug_assert_counts_match(root_id); + Some((actor, settled)) + } + /// Return true only when the actor has no queued or running executor work. pub fn actor_is_idle(&self, root_id: &ProjectRootId) -> bool { let state = self.inner.state.lock(); @@ -702,12 +794,18 @@ impl Executor { if !state.actors.get(root_id).is_some_and(ActorState::is_idle) { return false; } + let removed = Self::take_actor(&mut state, root_id); state.actor_order.retain(|actor_root| actor_root != root_id); - state.actors.remove(root_id) + removed }; - let Some(actor) = removed else { + let Some((actor, settled)) = removed else { return false; }; + for (_job_class, queued) in settled { + queued + .completion + .send(actor_fatal_response(queued.request_id.clone())); + } let app = actor.ctx.app(); crate::root_cache::unregister_live_scope(&actor.ctx.storage_dir(), root_id.as_path()); app.unregister_memory_context(root_id.as_path(), &actor.ctx); @@ -726,14 +824,28 @@ impl Executor { /// cancelled job receives a normal completion so its caller can settle /// bookkeeping through the same path as an executed job. pub fn cancel_queued_maintenance(&self, root_id: &ProjectRootId) -> usize { - let cancelled = { + let (cancelled, settled) = { let mut state = self.inner.state.lock(); - state - .actors - .get_mut(root_id) - .map(|actor| actor.maintenance.cancel_queued_jobs()) - .unwrap_or(0) + match state.actors.get_mut(root_id) { + Some(actor) => { + let drained = actor.maintenance.cancel_queued_jobs(); + state.process_counts.maintenance = state + .process_counts + .maintenance + .saturating_sub(drained.len()); + state.debug_assert_counts_match(root_id); + (drained.len(), drained) + } + None => (0, Vec::new()), + } }; + for queued in settled { + queued.completion.send(Response::error( + queued.request_id, + "maintenance_cancelled", + "maintenance cancelled because the actor has no bound routes", + )); + } if cancelled > 0 { self.wake_scheduler(); } @@ -823,30 +935,63 @@ impl Executor { lane: Lane, request_id: String, job: ExecutorJob, + ) -> oneshot::Receiver { + self.submit_async_with_deadline(root_id, lane, request_id, job, None) + } + + /// [`Self::submit_async`] carrying an optional absolute local request + /// deadline; `None` preserves the deadline-less contract. + pub fn submit_async_with_deadline( + &self, + root_id: ProjectRootId, + lane: Lane, + request_id: String, + job: ExecutorJob, + deadline: Option, ) -> oneshot::Receiver { let (completion_tx, completion_rx) = oneshot::channel(); - self.submit_with_completion( + self.submit_with_completion_cancellable( root_id, JobClass::Interactive, lane, request_id, job, CompletionSender::Async(completion_tx), + None, + None, + deadline, ); completion_rx } - /// Submit an interactive job with an exact-job cancellation token. - /// + /// Submit an interactive job with an exact-job cancellation token and an + /// optional queue-scoped request deadline. /// The returned token cancels THIS job only (queued: removed and settled /// with `request_cancelled`; running: signalled cooperatively). The job - /// observes the token via [`current_job_cancellation`]. + /// observes the token via [`current_job_cancellation`]. An elapsed + /// deadline rejects admission or prunes the queued job with + /// `request_deadline_exceeded`; once dispatched, a job is never + /// auto-cancelled by its deadline. pub fn submit_cancellable_async( &self, root_id: ProjectRootId, lane: Lane, request_id: String, job: ExecutorJob, + ) -> (oneshot::Receiver, JobCancellation) { + self.submit_cancellable_async_with_deadline(root_id, lane, request_id, job, None) + } + + /// [`Self::submit_cancellable_async`] carrying an absolute local request + /// deadline. `None` preserves the deadline-less contract for standalone, + /// internal, and test callers. + pub fn submit_cancellable_async_with_deadline( + &self, + root_id: ProjectRootId, + lane: Lane, + request_id: String, + job: ExecutorJob, + deadline: Option, ) -> (oneshot::Receiver, JobCancellation) { let cancellation = JobCancellation::new(); let (completion_tx, completion_rx) = oneshot::channel(); @@ -859,6 +1004,7 @@ impl Executor { CompletionSender::Async(completion_tx), Some(cancellation.clone()), None, + deadline, ); (completion_rx, cancellation) } @@ -887,8 +1033,12 @@ impl Executor { _ => JobCancelOutcome::NotFound, }; }; - match actor.remove_queued_cancellable(token) { - Some(queued) => (JobCancelOutcome::QueuedRemoved, Some(queued)), + let removed = actor.remove_queued_cancellable(token); + if let Some((job_class, _)) = removed.as_ref() { + state.account_dequeue(root_id, *job_class); + } + let outcome = match removed { + Some((_job_class, queued)) => (JobCancelOutcome::QueuedRemoved, Some(queued)), None => match observed { // The seal won the race: the job commits and finishes. JOB_CANCEL_STATE_COMMITTED => (JobCancelOutcome::RunningCommitted, None), @@ -901,7 +1051,9 @@ impl Executor { // Already cancelled by an earlier call and no longer queued. _ => (JobCancelOutcome::NotFound, None), }, - } + }; + state.debug_assert_counts_match(root_id); + outcome }; if let Some(queued) = settled { queued.completion.send(Response::error( @@ -953,6 +1105,7 @@ impl Executor { CompletionSender::Async(completion_tx), None, coalesce_key, + None, ); completion_rx } @@ -967,7 +1120,7 @@ impl Executor { completion: CompletionSender, ) { self.submit_with_completion_cancellable( - root_id, job_class, lane, request_id, job, completion, None, None, + root_id, job_class, lane, request_id, job, completion, None, None, None, ); } @@ -982,17 +1135,30 @@ impl Executor { completion: CompletionSender, cancellation: Option, maintenance_coalesce_key: Option, + deadline: Option, ) { let command = job_command(job_class, lane); let mut job = Some(job); let mut completion = Some(completion); - let mut duplicate_victims = Vec::new(); - let response = { + let (response, duplicate_victims) = { let mut state = self.inner.state.lock(); - match state.actors.get_mut(&root_id) { + // Snapshot config values before the actor borrow so depth checks + // can read caps without aliasing `state`. + let (interactive_actor_cap, interactive_process_cap, maintenance_process_cap) = ( + state.config.interactive_actor_queue_cap, + state.config.interactive_queue_cap, + state.config.maintenance_queue_cap, + ); + let mut process_counts = state.process_counts; + let mut rejection_deltas = (0u64, 0u64, 0u64); + let mut duplicate_victims_local: Vec = Vec::new(); + let admission = match state.actors.get_mut(&root_id) { Some(actor) if actor.fatal => Some(actor_fatal_response(request_id.clone())), Some(actor) => { + // Admission order: maintenance coalescing/dedupe first, + // then an already-expired interactive deadline, then the + // per-actor class cap, then the process class cap. let mut admission_error = None; if job_class == JobClass::Maintenance { if maintenance_coalesce_key @@ -1002,34 +1168,99 @@ impl Executor { request_id.clone(), "maintenance drain coalesced behind an identical queued drain", )); - } else { + } else if actor.maintenance.queued_count() >= MAINTENANCE_QUEUE_CAP { + duplicate_victims_local = + actor.maintenance.remove_duplicate_maintenance_jobs(); + process_counts.maintenance = process_counts + .maintenance + .saturating_sub(duplicate_victims_local.len()); if actor.maintenance.queued_count() >= MAINTENANCE_QUEUE_CAP { - duplicate_victims = - actor.maintenance.remove_duplicate_maintenance_jobs(); + admission_error = Some(maintenance_backpressure_response( + request_id.clone(), + "actor", + actor.maintenance.queued_count(), + MAINTENANCE_QUEUE_CAP, + )); + rejection_deltas.1 += 1; } - if actor.maintenance.queued_count() >= MAINTENANCE_QUEUE_CAP { - admission_error = - Some(maintenance_backpressure_response(request_id.clone())); + } + } else if deadline.is_some_and(|deadline| Instant::now() >= deadline) { + admission_error = + Some(request_deadline_exceeded_response(request_id.clone())); + rejection_deltas.2 += 1; + } + + if admission_error.is_none() { + let (actor_cap, process_cap, actor_depth, process_depth) = match job_class { + JobClass::Interactive => ( + interactive_actor_cap, + interactive_process_cap, + actor.interactive_queued_count, + process_counts.interactive, + ), + JobClass::Maintenance => ( + MAINTENANCE_QUEUE_CAP, + maintenance_process_cap, + actor.maintenance.queued_count(), + process_counts.maintenance, + ), + }; + let scope = if actor_depth >= actor_cap { + Some(("actor", actor_depth, actor_cap)) + } else if process_depth >= process_cap { + Some(("global", process_depth, process_cap)) + } else { + None + }; + if let Some((queue_scope, queue_depth, queue_cap)) = scope { + admission_error = Some(match job_class { + JobClass::Interactive => interactive_backpressure_response( + request_id.clone(), + queue_scope, + queue_depth, + queue_cap, + ), + JobClass::Maintenance => maintenance_backpressure_response( + request_id.clone(), + if queue_scope == "global" { + "global" + } else { + "per-actor" + }, + queue_depth, + queue_cap, + ), + }); + match job_class { + JobClass::Interactive => rejection_deltas.0 += 1, + JobClass::Maintenance => rejection_deltas.1 += 1, } } } if admission_error.is_none() { - actor.push_job( - job_class, - lane, - QueuedJob { - job: job.take().expect("executor job already queued"), - completion: completion - .take() - .expect("executor completion already queued"), - request_id: request_id.clone(), - command, - queued_at: Instant::now(), - cancellation: cancellation.clone(), - maintenance_coalesce_key, - }, - ); + let queued = QueuedJob { + job: job.take().expect("executor job already queued"), + completion: completion + .take() + .expect("executor completion already queued"), + request_id: request_id.clone(), + command, + queued_at: Instant::now(), + deadline, + cancellation: cancellation.clone(), + maintenance_coalesce_key, + }; + actor.push_job(job_class, lane, queued); + match job_class { + JobClass::Interactive => { + process_counts.interactive += 1; + actor.interactive_queued_count += 1; + } + JobClass::Maintenance => { + process_counts.maintenance += 1; + } + } } admission_error } @@ -1038,14 +1269,37 @@ impl Executor { "actor_not_registered", "executor actor is not registered", )), - } + }; + state.process_counts = process_counts; + state.interactive_admission_rejections = state + .interactive_admission_rejections + .saturating_add(rejection_deltas.0); + state.maintenance_admission_rejections = state + .maintenance_admission_rejections + .saturating_add(rejection_deltas.1); + state.deadline_expiries = state.deadline_expiries.saturating_add(rejection_deltas.2); + state.debug_assert_counts_match(&root_id); + self.inner + .dispatch_liveness + .record(&state.dispatch_liveness_snapshot()); + let duplicate_victims: Vec<(JobClass, QueuedJob)> = duplicate_victims_local + .into_iter() + .map(|victim| (JobClass::Maintenance, victim)) + .collect(); + (admission, duplicate_victims) }; - for victim in duplicate_victims { - victim.completion.send(maintenance_cancelled_response( - victim.request_id, - "duplicate maintenance drain removed to preserve queue capacity", - )); + for (victim_class, victim) in duplicate_victims { + let response = match victim_class { + JobClass::Maintenance => maintenance_cancelled_response( + victim.request_id, + "duplicate maintenance drain removed to preserve queue capacity", + ), + JobClass::Interactive => { + interactive_backpressure_response(victim.request_id, "actor", 0, 0) + } + }; + victim.completion.send(response); } if let Some(response) = response { @@ -1104,6 +1358,18 @@ impl Executor { .map(|state| state.mutating_job_state_label(root_id, request_id)) } + pub fn interactive_queue_cap(&self) -> usize { + self.inner.config.interactive_queue_cap + } + + pub fn interactive_actor_queue_cap(&self) -> usize { + self.inner.config.interactive_actor_queue_cap + } + + pub fn maintenance_queue_cap(&self) -> usize { + self.inner.config.maintenance_queue_cap + } + /// Snapshot RouteBind blockers without waiting on scheduler state. The subc /// health path uses this only for a delayed-bind breadcrumb, so contention /// is reported as scheduler busy rather than delaying the transport loop. @@ -1216,6 +1482,15 @@ impl Drop for ExecutorInner { } } +/// Pending (not running) job counts per process class, plus per-actor +/// interactive queue depth. These counts are the admission authority for the +/// class queue caps; `ClassQueues::order.len()` is only a debug cross-check. +#[derive(Debug, Default, Clone, Copy)] +struct QueuedClassCounts { + interactive: usize, + maintenance: usize, +} + struct SchedulerState { actors: HashMap, actor_order: Vec, @@ -1224,7 +1499,11 @@ struct SchedulerState { interactive_inflight: usize, maintenance_inflight: usize, config: EffectiveConfig, + process_counts: QueuedClassCounts, running_jobs: HashMap<(ProjectRootId, String), RunningJob>, + interactive_admission_rejections: u64, + maintenance_admission_rejections: u64, + deadline_expiries: u64, } impl SchedulerState { @@ -1237,8 +1516,94 @@ impl SchedulerState { interactive_inflight: 0, maintenance_inflight: 0, config, + process_counts: QueuedClassCounts::default(), running_jobs: HashMap::new(), + interactive_admission_rejections: 0, + maintenance_admission_rejections: 0, + deadline_expiries: 0, + } + } + + /// Release one queued job's capacity before its completion settles. + fn account_dequeue(&mut self, root_id: &ProjectRootId, job_class: JobClass) { + match job_class { + JobClass::Interactive => { + self.process_counts.interactive = self.process_counts.interactive.saturating_sub(1); + if let Some(actor) = self.actors.get_mut(root_id) { + actor.interactive_queued_count = + actor.interactive_queued_count.saturating_sub(1); + } + } + JobClass::Maintenance => { + self.process_counts.maintenance = self.process_counts.maintenance.saturating_sub(1); + } } + self.debug_assert_counts_match(root_id); + } + + /// Debug-only invariant: the mutable counters equal the recomputed queue + /// sums for one actor and the process. + fn debug_assert_counts_match(&self, root_id: &ProjectRootId) { + if !cfg!(debug_assertions) { + return; + } + if let Some(actor) = self.actors.get(root_id) { + debug_assert_eq!( + actor.interactive.class_queues_len(), + actor.interactive_queued_count, + "interactive actor queue count drift for {}", + root_id.as_path().display() + ); + } + let recomputed: QueuedClassCounts = self + .actors + .values() + .map(|actor| QueuedClassCounts { + interactive: actor.interactive.class_queues_len(), + maintenance: actor.maintenance.class_queues_len(), + }) + .fold(QueuedClassCounts::default(), |mut total, one| { + total.interactive += one.interactive; + total.maintenance += one.maintenance; + total + }); + debug_assert_eq!( + self.process_counts.interactive, recomputed.interactive, + "process interactive pending count drift" + ); + debug_assert_eq!( + self.process_counts.maintenance, recomputed.maintenance, + "process maintenance pending count drift" + ); + } + /// Prune elapsed-deadline interactive jobs from every lane at the start of + /// a scheduler turn, release their capacity, and return them so the caller + /// settles each with `request_deadline_exceeded`. Maintenance jobs carry no + /// client deadline. + fn prune_elapsed_deadline_jobs(&mut self, now: Instant) -> Vec<(ProjectRootId, QueuedJob)> { + let mut pruned = Vec::new(); + let roots: Vec = self.actors.keys().cloned().collect(); + for root_id in roots { + let Some(actor) = self.actors.get_mut(&root_id) else { + continue; + }; + let drained = actor.interactive.prune_elapsed(now); + if drained.is_empty() { + continue; + } + self.process_counts.interactive = self + .process_counts + .interactive + .saturating_sub(drained.len()); + actor.interactive_queued_count = + actor.interactive_queued_count.saturating_sub(drained.len()); + self.deadline_expiries = self.deadline_expiries.saturating_add(drained.len() as u64); + for job in drained { + pruned.push((root_id.clone(), job)); + } + self.debug_assert_counts_match(&root_id); + } + pruned } fn dispatch_liveness_snapshot(&self) -> DispatchLivenessSnapshot { @@ -1259,6 +1624,12 @@ impl SchedulerState { }, interactive_reserve: self.config.interactive_reserve, maintenance_cap: self.config.maintenance_cap, + interactive_queue_cap: self.config.interactive_queue_cap, + interactive_actor_queue_cap: self.config.interactive_actor_queue_cap, + maintenance_queue_cap: self.config.maintenance_queue_cap, + interactive_admission_rejections: self.interactive_admission_rejections, + maintenance_admission_rejections: self.maintenance_admission_rejections, + deadline_expiries: self.deadline_expiries, } } @@ -1474,6 +1845,8 @@ struct ActorState { deficit: isize, interactive: ClassQueues, maintenance: ClassQueues, + /// Mirror of the interactive queue depth; the per-actor class cap authority. + interactive_queued_count: usize, fatal: bool, } @@ -1492,6 +1865,7 @@ impl ActorState { deficit: 0, interactive: ClassQueues::new(), maintenance: ClassQueues::new(), + interactive_queued_count: 0, fatal: false, } } @@ -1542,9 +1916,23 @@ impl ActorState { } } - fn fail_queued_jobs(&mut self) { - self.interactive.fail_queued_jobs(); - self.maintenance.fail_queued_jobs(); + /// Drain every queued job and settle with `actor_fatal`, returning the + /// drained jobs per class so the caller releases capacity before sending + /// completions. + fn fail_queued_jobs(&mut self) -> Vec<(JobClass, QueuedJob)> { + let mut drained: Vec<(JobClass, QueuedJob)> = self + .interactive + .fail_queued_jobs() + .into_iter() + .map(|job| (JobClass::Interactive, job)) + .collect(); + drained.extend( + self.maintenance + .fail_queued_jobs() + .into_iter() + .map(|job| (JobClass::Maintenance, job)), + ); + drained } fn has_queued_mutating_job(&self, request_id: &str) -> bool { @@ -1552,10 +1940,20 @@ impl ActorState { || self.maintenance.has_queued_mutating_job(request_id) } - fn remove_queued_cancellable(&mut self, token: &JobCancellation) -> Option { + /// Remove the queued job carrying this token, returning its class so the + /// caller releases the right capacity bucket. + fn remove_queued_cancellable( + &mut self, + token: &JobCancellation, + ) -> Option<(JobClass, QueuedJob)> { self.interactive .remove_cancellable(token) - .or_else(|| self.maintenance.remove_cancellable(token)) + .map(|job| (JobClass::Interactive, job)) + .or_else(|| { + self.maintenance + .remove_cancellable(token) + .map(|job| (JobClass::Maintenance, job)) + }) } fn oldest_queued_writer_at(&self) -> Option { @@ -1619,9 +2017,7 @@ impl ClassQueues { /// never barrier the actor), then remaining lanes in arrival order. /// Maintenance keeps strict arrival order via `front_lane`. fn next_interactive_lane(&self, now: Instant) -> Option { - let starved_writer = self.mutating.iter().any(|job| { - now.saturating_duration_since(job.queued_at) >= INTERACTIVE_WRITER_PROMOTION_AGE - }); + let starved_writer = self.has_urgent_writer(now); if starved_writer { // Also stops NEW readers from being admitted on this actor while // the promoted writer waits for in-flight readers to drain. @@ -1636,6 +2032,70 @@ impl ClassQueues { .find(|lane| *lane != Lane::PureRead) } + /// Deadline-aware urgency for queued interactive writers, replacing the + /// fixed writer-only promotion test while retaining its fallback: a + /// deadline-bearing writer is urgent when its remaining budget is at or + /// below its queue age (or the promotion-age floor); a deadline-less + /// writer becomes urgent at the promotion age. + fn has_urgent_writer(&self, now: Instant) -> bool { + self.mutating.iter().any(|job| { + let age = now.saturating_duration_since(job.queued_at); + match job.deadline { + Some(deadline) => { + let remaining = deadline.saturating_duration_since(now); + remaining <= age.max(INTERACTIVE_WRITER_PROMOTION_AGE) + } + None => age >= INTERACTIVE_WRITER_PROMOTION_AGE, + } + }) + } + + /// Remove interactive jobs whose queue deadline has elapsed, preserving + /// survivor order in both the ladder and the lane queues. + fn prune_elapsed(&mut self, now: Instant) -> Vec { + let mut drained = Vec::new(); + for lane in [ + Lane::PureRead, + Lane::SerialLspStatus, + Lane::HeavyInit, + Lane::Mutating, + Lane::MaintenanceCommit, + ] { + let queue = self.queue_mut(lane); + let mut index = 0; + while index < queue.len() { + if queue[index] + .deadline + .is_some_and(|deadline| now >= deadline) + { + if let Some(job) = queue.remove(index) { + drained.push(job); + } + } else { + index += 1; + } + } + } + if drained.is_empty() { + return drained; + } + // Rebuild the ladder from the survivors; each lane queue is FIFO, so + // the per-lane counts are the ladder multiplicities. + self.order.clear(); + for lane in [ + Lane::PureRead, + Lane::SerialLspStatus, + Lane::HeavyInit, + Lane::Mutating, + Lane::MaintenanceCommit, + ] { + for _ in 0..self.queue(lane).len() { + self.order.push_back(lane); + } + } + drained + } + fn pop_front_job(&mut self, lane: Lane) -> Option { // Keep `order` consistent with per-lane queues when admission picks a // lane other than the arrival-order head: remove the FIRST occurrence @@ -1649,6 +2109,18 @@ impl ClassQueues { self.order.len() } + /// Debug cross-check source for the class counters. The sum of lane queue + /// lengths must equal `order.len()`; both prove the pending job count. + fn class_queues_len(&self) -> usize { + let lane_sum = self.pure_reads.len() + + self.lsp_status.len() + + self.heavy_init.len() + + self.mutating.len() + + self.maintenance_commit.len(); + debug_assert_eq!(self.order.len(), lane_sum, "order ladder out of sync"); + lane_sum + } + fn has_maintenance_coalesce_key(&self, key: MaintenanceCoalesceKey) -> bool { [ Lane::PureRead, @@ -1702,22 +2174,26 @@ impl ClassQueues { .and_then(|lane| self.queue(lane).front().map(|job| job.queued_at)) } - fn fail_queued_jobs(&mut self) { + fn fail_queued_jobs(&mut self) -> Vec { + let mut drained = Vec::new(); + drained.extend(self.pure_reads.drain(..)); + drained.extend(self.lsp_status.drain(..)); + drained.extend(self.heavy_init.drain(..)); + drained.extend(self.mutating.drain(..)); + drained.extend(self.maintenance_commit.drain(..)); self.order.clear(); - fail_queued_job_queue(&mut self.pure_reads); - fail_queued_job_queue(&mut self.lsp_status); - fail_queued_job_queue(&mut self.heavy_init); - fail_queued_job_queue(&mut self.mutating); - fail_queued_job_queue(&mut self.maintenance_commit); + drained } - fn cancel_queued_jobs(&mut self) -> usize { + fn cancel_queued_jobs(&mut self) -> Vec { + let mut drained = Vec::new(); + drained.extend(self.pure_reads.drain(..)); + drained.extend(self.lsp_status.drain(..)); + drained.extend(self.heavy_init.drain(..)); + drained.extend(self.mutating.drain(..)); + drained.extend(self.maintenance_commit.drain(..)); self.order.clear(); - cancel_queued_job_queue(&mut self.pure_reads) - + cancel_queued_job_queue(&mut self.lsp_status) - + cancel_queued_job_queue(&mut self.heavy_init) - + cancel_queued_job_queue(&mut self.mutating) - + cancel_queued_job_queue(&mut self.maintenance_commit) + drained } fn has_queued_mutating_job(&self, request_id: &str) -> bool { @@ -1800,6 +2276,10 @@ struct QueuedJob { request_id: String, command: String, queued_at: Instant, + /// Queue-scoped request budget. `None` means no deadline (standalone, + /// internal, and test callers). Elapsed deadlines reject admission and + /// prune queued jobs; a dispatched job is never auto-cancelled. + deadline: Option, cancellation: Option, maintenance_coalesce_key: Option, } @@ -1814,24 +2294,56 @@ fn lane_index(lane: Lane) -> usize { } } -fn fail_queued_job_queue(queue: &mut VecDeque) { - for queued in queue.drain(..) { - queued - .completion - .send(actor_fatal_response(queued.request_id)); - } +fn maintenance_backpressure_response( + request_id: impl Into, + queue_scope: &str, + queue_depth: usize, + queue_cap: usize, +) -> Response { + Response::error_with_data( + request_id, + "maintenance_backpressure", + format!("maintenance queue reached its {queue_scope} capacity of {queue_cap} jobs"), + serde_json::json!({ + "retryable": true, + "queue_class": "maintenance", + "queue_scope": queue_scope, + "queue_depth": queue_depth, + "queue_cap": queue_cap, + }), + ) } -fn cancel_queued_job_queue(queue: &mut VecDeque) -> usize { - let cancelled = queue.len(); - for queued in queue.drain(..) { - queued.completion.send(Response::error( - queued.request_id, - "maintenance_cancelled", - "maintenance cancelled because the actor has no bound routes", - )); - } - cancelled +fn interactive_backpressure_response( + request_id: impl Into, + queue_scope: &str, + queue_depth: usize, + queue_cap: usize, +) -> Response { + Response::error_with_data( + request_id, + "executor_backpressure", + format!("interactive queue reached its {queue_scope} capacity of {queue_cap} jobs"), + serde_json::json!({ + "retryable": true, + "queue_class": "interactive", + "queue_scope": queue_scope, + "queue_depth": queue_depth, + "queue_cap": queue_cap, + }), + ) +} + +fn request_deadline_exceeded_response(request_id: impl Into) -> Response { + Response::error_with_data( + request_id, + "request_deadline_exceeded", + "request deadline elapsed before execution", + serde_json::json!({ + "retryable": false, + "phase": "queue", + }), + ) } fn job_command(job_class: JobClass, lane: Lane) -> String { @@ -1845,18 +2357,6 @@ fn maintenance_cancelled_response( Response::error(request_id, "maintenance_cancelled", message) } -fn maintenance_backpressure_response(request_id: impl Into) -> Response { - Response::error_with_data( - request_id, - "maintenance_backpressure", - format!("maintenance queue reached its per-actor capacity of {MAINTENANCE_QUEUE_CAP} jobs"), - serde_json::json!({ - "retryable": true, - "queue_cap": MAINTENANCE_QUEUE_CAP, - }), - ) -} - fn actor_fatal_response(request_id: impl Into) -> Response { Response::error( request_id, @@ -1945,6 +2445,7 @@ fn scheduler_loop( completed_maintenance: Arc, dispatch_liveness: Arc, ) { + let mut expired_completions: Vec = Vec::new(); while let Ok(event) = event_rx.recv() { let shutdown; { @@ -1958,11 +2459,21 @@ fn scheduler_loop( ); if !shutdown { + // Prune elapsed interactive deadlines at the start of every + // turn, release their capacity, then continue dispatch. The + // settled completions are sent after the lock is released. + let pruned = state.prune_elapsed_deadline_jobs(Instant::now()); dispatch_runnable(&mut state, &heavy, &run_tx, &nonrunnable_dispatches); + expired_completions.extend(pruned.into_iter().map(|(_, job)| job)); } dispatch_liveness.record(&state.dispatch_liveness_snapshot()); } + for job in expired_completions.drain(..) { + job.completion + .send(request_deadline_exceeded_response(job.request_id)); + } + if shutdown { break; } @@ -2059,7 +2570,13 @@ fn complete_job(state: &mut SchedulerState, event: CompletionEvent) { if panicked && lane == Lane::Mutating { actor.fatal = true; - actor.fail_queued_jobs(); + let drained = actor.fail_queued_jobs(); + for (job_class, queued) in drained { + state.account_dequeue(&root_id, job_class); + queued + .completion + .send(actor_fatal_response(queued.request_id)); + } } } @@ -2135,36 +2652,48 @@ fn dispatch_runnable_class( let root_id = state.actor_order[state.cursor].clone(); state.cursor = (state.cursor + 1) % state.actor_order.len(); + let mut fatal_drained: Vec<(JobClass, QueuedJob)> = Vec::new(); let run_job = { let Some(actor) = state.actors.get_mut(&root_id) else { continue; }; if actor.fatal { - actor.fail_queued_jobs(); + fatal_drained = actor.fail_queued_jobs(); actor.deficit = 0; - continue; - } - - if !actor.has_queued_jobs() { + None + } else if !actor.has_queued_jobs() { actor.deficit = 0; - continue; + None + } else if actor.has_queued_jobs_for(job_class) { + actor.deficit = + (actor.deficit + state.config.drr_quantum).min(state.config.deficit_cap); + if actor.deficit < JOB_COST { + continue; + } + try_admit_actor(&root_id, actor, job_class, &state.config, heavy) + } else { + None } + }; - if !actor.has_queued_jobs_for(job_class) { - continue; + if !fatal_drained.is_empty() { + let drained_by_class = fatal_drained.iter().map(|(c, _)| *c).collect::>(); + for job_class in drained_by_class { + state.account_dequeue(&root_id, job_class); } - - actor.deficit = - (actor.deficit + state.config.drr_quantum).min(state.config.deficit_cap); - if actor.deficit < JOB_COST { - continue; + state.debug_assert_counts_match(&root_id); + for (_job_class, queued) in fatal_drained { + queued + .completion + .send(actor_fatal_response(queued.request_id)); } - - try_admit_actor(&root_id, actor, job_class, &state.config, heavy) - }; - + continue; + } if let Some(run_job) = run_job { + // The pop happened inside try_admit_actor: release the queued + // capacity bucket before the dispatch send. + state.account_dequeue(&root_id, job_class); state.running_jobs.insert( (run_job.root_id.clone(), run_job.request_id.clone()), RunningJob { @@ -2261,9 +2790,10 @@ fn try_admit_actor( return None; } - let promoted_writer_waiting = actor.oldest_queued_writer_at().is_some_and(|queued_at| { - Instant::now().saturating_duration_since(queued_at) >= INTERACTIVE_WRITER_PROMOTION_AGE - }); + let promoted_writer_waiting = job_class == JobClass::Interactive + && actor + .class_queues(JobClass::Interactive) + .has_urgent_writer(Instant::now()); if promoted_writer_waiting && matches!(lane, Lane::PureRead | Lane::SerialLspStatus) { actor.reader_admissions_while_promoted_writer_waited = actor .reader_admissions_while_promoted_writer_waited @@ -2271,7 +2801,6 @@ fn try_admit_actor( } let queued = actor.pop_front_job(job_class, lane)?; - actor.deficit -= JOB_COST; if let Some(cancellation) = queued.cancellation.as_ref() { cancellation.mark_running(); } @@ -2321,8 +2850,9 @@ fn try_admit_actor( fn worker_loop(run_rx: Receiver, event_tx: Sender) { while let Ok(mut run_job) = run_rx.recv() { - let response = - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| run_lane_job(&mut run_job))); + let response = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + run_lane_job_with_priority(&mut run_job) + })); let panicked = response.is_err(); let response = match response { Ok(response) => response, @@ -2347,6 +2877,22 @@ fn worker_loop(run_rx: Receiver, event_tx: Sender) { let _ = event_tx.send(SchedulerEvent::Completed(completion)); } } +/// Dispatch one lane job, demoting the worker thread to background OS +/// priority while heavy maintenance lanes execute. Reader lanes keep the +/// thread at normal priority so interactive requests win the OS scheduler. +fn run_lane_job_with_priority(run_job: &mut RunJob) -> Response { + match run_job.lane { + Lane::PureRead | Lane::SerialLspStatus => run_lane_job(run_job), + // Mutating stays at normal priority: the lane is reserved for + // configure and user-initiated tool mutations. Only cold builds + // (HeavyInit) and background subsystem drains (MaintenanceCommit) + // are deferrable maintenance work. + Lane::HeavyInit | Lane::MaintenanceCommit => { + crate::thread_priority::with_background(|| run_lane_job(run_job)) + } + Lane::Mutating => run_lane_job(run_job), + } +} fn run_lane_job(run_job: &mut RunJob) -> Response { let _cancellation_ctx = JobCancellationContextGuard::install(run_job.cancellation.clone()); diff --git a/crates/aft/src/executor/tests.rs b/crates/aft/src/executor/tests.rs index 00183f149..89f7334e3 100644 --- a/crates/aft/src/executor/tests.rs +++ b/crates/aft/src/executor/tests.rs @@ -55,6 +55,7 @@ fn test_executor( actor_cap, heavy_permits, drr_quantum: 1, + ..ExecutorConfig::default() }) } @@ -1986,6 +1987,7 @@ fn starved_bind_promotes_over_pure_reads() { completion: CompletionSender::Sync(tx.clone()), queued_at: now - INTERACTIVE_WRITER_PROMOTION_AGE - Duration::from_secs(1), cancellation: None, + deadline: None, maintenance_coalesce_key: None, }; let read_job = QueuedJob { @@ -1995,6 +1997,7 @@ fn starved_bind_promotes_over_pure_reads() { completion: CompletionSender::Sync(tx), queued_at: now, cancellation: None, + deadline: None, maintenance_coalesce_key: None, }; // Read arrived FIRST in arrival order; the starved bind must still win. @@ -2028,6 +2031,7 @@ fn fresh_bind_does_not_preempt_pure_reads() { completion: CompletionSender::Sync(tx.clone()), queued_at: now, cancellation: None, + deadline: None, maintenance_coalesce_key: None, }, ); @@ -2041,6 +2045,7 @@ fn fresh_bind_does_not_preempt_pure_reads() { completion: CompletionSender::Sync(tx), queued_at: now, cancellation: None, + deadline: None, maintenance_coalesce_key: None, }, ); @@ -2071,6 +2076,7 @@ fn maintenance_defers_to_queued_interactive_mutating_anywhere_in_queue() { completion: CompletionSender::Sync(tx.clone()), queued_at: Instant::now(), cancellation: None, + deadline: None, maintenance_coalesce_key: None, }, ); @@ -2084,6 +2090,7 @@ fn maintenance_defers_to_queued_interactive_mutating_anywhere_in_queue() { completion: CompletionSender::Sync(tx), queued_at: Instant::now(), cancellation: None, + deadline: None, maintenance_coalesce_key: None, }, ); @@ -2397,6 +2404,7 @@ fn remove_cancellable_removes_matching_lane_order_occurrence_not_first() { completion: CompletionSender::Sync(tx.clone()), queued_at: Instant::now(), cancellation: None, + deadline: None, maintenance_coalesce_key: None, }, ); @@ -2410,6 +2418,7 @@ fn remove_cancellable_removes_matching_lane_order_occurrence_not_first() { completion: CompletionSender::Sync(tx.clone()), queued_at: Instant::now(), cancellation: None, + deadline: None, maintenance_coalesce_key: None, }, ); @@ -2423,13 +2432,15 @@ fn remove_cancellable_removes_matching_lane_order_occurrence_not_first() { completion: CompletionSender::Sync(tx), queued_at: Instant::now(), cancellation: Some(m2_token.clone()), + deadline: None, maintenance_coalesce_key: None, }, ); - let removed = actor + let (removed_class, removed) = actor .remove_queued_cancellable(&m2_token) .expect("m2 removed"); + assert_eq!(removed_class, JobClass::Interactive); assert_eq!(removed.request_id, "m2"); // Arrival-order head must still be M1's lane, and popping in order must @@ -2485,3 +2496,392 @@ fn cancel_and_seal_race_has_exactly_one_winner() { } } } + +#[test] +fn queued_deadline_job_is_pruned_and_counted_at_next_turn() { + // A queued job whose deadline elapses while blocked must be settled by the + // scheduler (not executed) and counted as a deadline expiry. + let executor = test_executor(2, 1, 1, 1); + let (_dir, root) = test_root("prune-queued"); + executor.register_actor(root.clone(), test_ctx()); + + let (started_tx, started_rx) = crossbeam_channel::bounded(1); + let (release_tx, release_rx) = crossbeam_channel::bounded(1); + let blocker = executor.submit( + root.clone(), + Lane::Mutating, + "prune-blocker".to_string(), + Box::new(move |_| { + started_tx.send(()).expect("signal start"); + release_rx + .recv_timeout(Duration::from_secs(5)) + .expect("release"); + ok("prune-blocker") + }), + ); + started_rx + .recv_timeout(Duration::from_secs(2)) + .expect("prune blocker starts"); + + // A queued reader with an already-tight deadline; the writer blocker holds + // the actor so this job stays queued until the next scheduler turn prunes it. + let executed = Arc::new(AtomicUsize::new(0)); + let executed_probe = Arc::clone(&executed); + let (rx, _token) = executor.submit_cancellable_async_with_deadline( + root.clone(), + Lane::PureRead, + "prune-victim".to_string(), + Box::new(move |_| { + executed_probe.fetch_add(1, Ordering::AcqRel); + ok("prune-victim") + }), + Some(Instant::now() + Duration::from_millis(50)), + ); + // Let the victim's deadline elapse while the blocker still holds the actor. + // Releasing the blocker then gives the scheduler a completion event; the + // next turn prunes the elapsed victim and settles it with + // request_deadline_exceeded instead of executing it. + thread::sleep(Duration::from_millis(120)); + + // Releasing the blocker completes the writer; the completion event wakes + // the scheduler and the next turn prunes the elapsed victim, settling it + // with request_deadline_exceeded instead of executing it. + release_tx.send(()).expect("release prune blocker"); + assert!( + blocker + .recv_timeout(Duration::from_secs(5)) + .expect("prune blocker completes") + .success + ); + + let response = recv_async(rx, "pruned job completion"); + assert!(!response.success); + assert_eq!(response.data["code"], "request_deadline_exceeded"); + assert_eq!(executed.load(Ordering::Acquire), 0); +} + +#[test] +fn interactive_queue_cap_returns_typed_backpressure_per_actor_and_global() { + // pool 2 / actor_cap 1: one running blocker per actor, then the per-actor + // interactive cap admits 2 more; the next is rejected with the actor scope. + // A second actor's global budget is sized so its first overflow reports the + // global scope. + let executor = test_executor(2, 1, 1, 1); + let (_dir_a, root_a) = test_root("interactive-cap-a"); + executor.register_actor(root_a.clone(), test_ctx()); + + let (blocker_started_tx, blocker_started_rx) = crossbeam_channel::bounded(1); + let (release_blocker_tx, release_blocker_rx) = crossbeam_channel::bounded(1); + let blocker = executor.submit( + root_a.clone(), + Lane::Mutating, + "interactive-cap-blocker".to_string(), + Box::new(move |_| { + blocker_started_tx.send(()).expect("signal blocker start"); + release_blocker_rx + .recv_timeout(Duration::from_secs(5)) + .expect("release interactive blocker"); + ok("interactive-cap-blocker") + }), + ); + blocker_started_rx + .recv_timeout(Duration::from_secs(2)) + .expect("interactive blocker starts"); + + let executed = Arc::new(AtomicUsize::new(0)); + let mut admitted = Vec::new(); + for _ in 0..executor.interactive_actor_queue_cap() { + let executed_probe = Arc::clone(&executed); + admitted.push(executor.submit_async( + root_a.clone(), + Lane::PureRead, + "interactive-cap-admitted".to_string(), + Box::new(move |_| { + executed_probe.fetch_add(1, Ordering::AcqRel); + ok("interactive-cap-admitted") + }), + )); + } + let overflow = executor.submit_async( + root_a, + Lane::PureRead, + "interactive-cap-overflow".to_string(), + Box::new(|_| ok("interactive-cap-overflow")), + ); + + let overflow_response = recv_async(overflow, "interactive backpressure completion"); + assert!(!overflow_response.success); + assert_eq!(overflow_response.data["code"], "executor_backpressure"); + assert_eq!(overflow_response.data["retryable"], serde_json::json!(true)); + assert_eq!( + overflow_response.data["queue_class"], + serde_json::json!("interactive") + ); + assert_eq!( + overflow_response.data["queue_scope"], + serde_json::json!("actor") + ); + assert_eq!(executed.load(Ordering::Acquire), 0); + + release_blocker_tx + .send(()) + .expect("release interactive blocker"); + assert!( + blocker + .recv_timeout(Duration::from_secs(5)) + .expect("blocker completes") + .success + ); + for receiver in admitted { + assert!( + recv_async(receiver, "admitted interactive completion").success, + "admitted interactive job must execute" + ); + } + assert_eq!( + executed.load(Ordering::Acquire), + executor.interactive_actor_queue_cap(), + "every admitted interactive job must execute exactly once" + ); +} + +#[test] +fn coalesced_maintenance_skips_capacity_and_dedupe_releases_capacity() { + // A coalesced duplicate does not consume new capacity and is answered with + // the ordinary maintenance_cancelled coalesce response. When the per-actor + // cap is full, a duplicate removal frees exactly one slot for the next job. + let executor = test_executor(2, 1, 1, 1); + let (_dir, root) = test_root("coalesce-capacity"); + executor.register_actor(root.clone(), test_ctx()); + + let (blocker_started_tx, blocker_started_rx) = crossbeam_channel::bounded(1); + let (release_blocker_tx, release_blocker_rx) = crossbeam_channel::bounded(1); + let blocker = executor.submit_maintenance_async( + root.clone(), + Lane::MaintenanceCommit, + "coalesce-cap-blocker".to_string(), + Box::new(move |_| { + blocker_started_tx.send(()).expect("signal blocker start"); + release_blocker_rx + .recv_timeout(Duration::from_secs(5)) + .expect("release blocker"); + ok("coalesce-cap-blocker") + }), + ); + blocker_started_rx + .recv_timeout(Duration::from_secs(2)) + .expect("blocker starts"); + + // Queue the first drain, then submit a duplicate. The duplicate coalesces + // behind the identical queued drain and settles immediately with the + // ordinary coalesce response without consuming new capacity. + let first = executor.submit_coalescable_maintenance_async( + root.clone(), + Lane::MaintenanceCommit, + "watcher-drain".to_string(), + MaintenanceCoalesceKey::WatcherDrain, + Box::new(|_| ok("watcher-drain")), + ); + let coalesced_second = executor.submit_coalescable_maintenance_async( + root.clone(), + Lane::MaintenanceCommit, + "watcher-drain".to_string(), + MaintenanceCoalesceKey::WatcherDrain, + Box::new(|_| ok("watcher-drain")), + ); + let coalesced_response = recv_async(coalesced_second, "coalesced duplicate completion"); + assert!(!coalesced_response.success); + assert_eq!(coalesced_response.data["code"], "maintenance_cancelled"); + + release_blocker_tx + .send(()) + .expect("release coalesce blocker"); + assert!(recv_async(blocker, "coalesce blocker completion").success); + assert!( + recv_async(first, "first coalescable drain").success, + "the first coalescable drain executes after the blocker drains" + ); +} + +#[test] +fn queue_accounting_tracks_dispatch_cancellation_and_actor_retirement() { + // Depths must return to zero after dispatch, queued cancellation, and + // actor removal; liveness mirrors the same numbers without contention. + let executor = test_executor(1, 1, 1, 1); + let (_dir, root) = test_root("accounting"); + executor.register_actor(root.clone(), test_ctx()); + + let (started_tx, started_rx) = crossbeam_channel::bounded(1); + let (release_tx, release_rx) = crossbeam_channel::bounded(1); + let blocker = executor.submit_async( + root.clone(), + Lane::Mutating, + "accounting-blocker".to_string(), + Box::new(move |_| { + started_tx.send(()).expect("signal start"); + release_rx + .recv_timeout(Duration::from_secs(5)) + .expect("release"); + ok("accounting-blocker") + }), + ); + started_rx + .recv_timeout(Duration::from_secs(2)) + .expect("blocker starts"); + + let (queued_rx, queued_token) = executor.submit_cancellable_async( + root.clone(), + Lane::PureRead, + "accounting-queued".to_string(), + Box::new(|_| ok("accounting-queued")), + ); + let snapshot = executor + .try_dispatch_liveness_snapshot() + .expect("liveness snapshot"); + assert_eq!(snapshot.interactive.queued, 1); + + assert_eq!( + executor.cancel_job(&root, &queued_token), + JobCancelOutcome::QueuedRemoved + ); + assert!(!recv_async(queued_rx, "queued cancel completion").success); + let snapshot = executor + .try_dispatch_liveness_snapshot() + .expect("liveness snapshot after cancel"); + assert_eq!(snapshot.interactive.queued, 0); + + release_tx.send(()).expect("release blocker"); + assert!(recv_async(blocker, "blocker completion").success); + + executor.remove_actor(&root); + let snapshot = executor + .try_dispatch_liveness_snapshot() + .expect("liveness after removal"); + assert_eq!(snapshot.interactive.queued, 0); + assert_eq!(snapshot.maintenance.queued, 0); +} + +#[test] +fn already_expired_deadline_rejects_admission_with_request_deadline_exceeded() { + let executor = test_executor(2, 1, 1, 1); + let (_dir, root) = test_root("expired-admission"); + executor.register_actor(root.clone(), test_ctx()); + + let executed = Arc::new(AtomicUsize::new(0)); + let executed_probe = Arc::clone(&executed); + let (rx, _token) = executor.submit_cancellable_async_with_deadline( + root, + Lane::PureRead, + "expired-admission-job".to_string(), + Box::new(move |_| { + executed_probe.fetch_add(1, Ordering::AcqRel); + ok("expired-admission-job") + }), + Some(Instant::now() - Duration::from_secs(1)), + ); + let response = recv_async(rx, "expired admission completion"); + assert!(!response.success); + assert_eq!(response.data["code"], "request_deadline_exceeded"); + assert_eq!(response.data["retryable"], serde_json::json!(false)); + assert_eq!(response.data["phase"], serde_json::json!("queue")); + assert_eq!(executed.load(Ordering::Acquire), 0); +} + +#[test] +fn dispatched_job_is_not_auto_cancelled_after_deadline_passes() { + // Once popped, a job runs to completion even if its deadline elapses + // mid-execution; the queue-scoped rule keeps dispatched work authoritative. + let executor = test_executor(2, 1, 1, 1); + let (_dir, root) = test_root("dispatched-not-cancelled"); + executor.register_actor(root.clone(), test_ctx()); + + let (rx, _token) = executor.submit_cancellable_async_with_deadline( + root, + Lane::PureRead, + "late-runner".to_string(), + Box::new(|_| { + thread::sleep(Duration::from_millis(150)); + ok("late-runner") + }), + Some(Instant::now() + Duration::from_millis(20)), + ); + let response = recv_async(rx, "late runner completion"); + assert!( + response.success, + "a dispatched job must complete despite an elapsed deadline" + ); +} + +#[test] +fn deadline_aware_writer_urgency_matches_budget_boundaries() { + struct Case { + label: &'static str, + deadline: Option, + now_offset_ms: u64, + expect_urgent: bool, + } + let now = Instant::now(); + let cases = [ + // Budget <= 6s: urgent immediately (remaining <= promotion-age floor). + Case { + label: "small budget immediate urgency", + deadline: Some(now + Duration::from_secs(6)), + now_offset_ms: 0, + expect_urgent: true, + }, + // 12s RouteBind budget, queued for ~0ms: urgency at 6s age. Not urgent yet. + Case { + label: "halfway not reached", + deadline: Some(now + Duration::from_secs(12)), + now_offset_ms: 0, + expect_urgent: false, + }, + // 12s budget queued at 6s: halfway point reached. + Case { + label: "halfway urgency", + deadline: Some(now + Duration::from_secs(6)), + now_offset_ms: 6_000, + expect_urgent: true, + }, + // Deadline-less writers fall back to the promotion age. + Case { + label: "deadline-less below promotion age", + deadline: None, + now_offset_ms: 5_999, + expect_urgent: false, + }, + Case { + label: "deadline-less at promotion age", + deadline: None, + now_offset_ms: 6_000, + expect_urgent: true, + }, + ]; + for case in cases { + let mut actor = ActorState::new(test_ctx()); + let (tx, _rx) = crossbeam_channel::bounded::(1); + actor.push_job( + JobClass::Interactive, + Lane::Mutating, + QueuedJob { + request_id: "bind".to_string(), + command: "executor::Interactive::Mutating".to_string(), + job: Box::new(|_ctx| ok("bind")), + completion: CompletionSender::Sync(tx), + queued_at: now, + deadline: case.deadline, + cancellation: None, + maintenance_coalesce_key: None, + }, + ); + let probe_now = now + Duration::from_millis(case.now_offset_ms); + assert_eq!( + actor + .class_queues(JobClass::Interactive) + .has_urgent_writer(probe_now), + case.expect_urgent, + "urgency boundary failed: {}", + case.label + ); + } +} diff --git a/crates/aft/src/gh_shim.rs b/crates/aft/src/gh_shim.rs index 62df43cb9..47f4af9c6 100644 --- a/crates/aft/src/gh_shim.rs +++ b/crates/aft/src/gh_shim.rs @@ -4060,6 +4060,7 @@ mod tests { expected["body"] = wire["body"].clone(); expected["manifest_version"] = json!(manifest.manifest_version); expected["rung_as_of_unix_secs"] = json!(determination.record.as_of_unix_secs); + expected["repository"] = wire["repository"].clone(); expected["metadata"]["pid"] = json!(std::process::id()); expected["metadata"] .as_object_mut() diff --git a/crates/aft/src/inspect/dispatch.rs b/crates/aft/src/inspect/dispatch.rs index aa063c3bd..535e239ac 100644 --- a/crates/aft/src/inspect/dispatch.rs +++ b/crates/aft/src/inspect/dispatch.rs @@ -44,6 +44,9 @@ static INSPECT_POOL: LazyLock> = LazyLock::new(|| { .stack_size(8 * 1024 * 1024) .start_handler(|_| { INSPECT_THREAD_COUNT.fetch_add(1, Ordering::SeqCst); + // Inspect workers are pure background maintenance: let + // interactive readers win the OS scheduler on CPU and I/O. + crate::thread_priority::demote_background(); }) .exit_handler(|_| { INSPECT_THREAD_COUNT.fetch_sub(1, Ordering::SeqCst); diff --git a/crates/aft/src/lib.rs b/crates/aft/src/lib.rs index 3de301c2c..14fd0ed84 100644 --- a/crates/aft/src/lib.rs +++ b/crates/aft/src/lib.rs @@ -121,9 +121,9 @@ pub mod subc_config; pub mod subc_format; pub mod subc_translate; pub mod symbol_cache_disk; -pub mod symbol_diff; pub mod symbols; pub mod synapse_embed; +pub mod thread_priority; pub mod tool_path; pub mod url_fetch; pub(crate) mod walk_boundary; diff --git a/crates/aft/src/logging.rs b/crates/aft/src/logging.rs index b9e5750c1..f157ae642 100644 --- a/crates/aft/src/logging.rs +++ b/crates/aft/src/logging.rs @@ -583,6 +583,12 @@ struct ExecutorSample { maintenance_queued: usize, interactive_oldest_ms: Option, maintenance_oldest_ms: Option, + interactive_queue_cap: usize, + interactive_actor_queue_cap: usize, + maintenance_queue_cap: usize, + interactive_admission_rejections: u64, + maintenance_admission_rejections: u64, + deadline_expiries: u64, } static PERF: LazyLock = LazyLock::new(PerfMetrics::default); @@ -749,6 +755,12 @@ pub fn perf_tick(executor: Option<&Executor>) { maintenance_queued: snapshot.maintenance.queued, interactive_oldest_ms: snapshot.interactive.oldest_age_ms, maintenance_oldest_ms: snapshot.maintenance.oldest_age_ms, + interactive_queue_cap: snapshot.interactive_queue_cap, + interactive_actor_queue_cap: snapshot.interactive_actor_queue_cap, + maintenance_queue_cap: snapshot.maintenance_queue_cap, + interactive_admission_rejections: snapshot.interactive_admission_rejections, + maintenance_admission_rejections: snapshot.maintenance_admission_rejections, + deadline_expiries: snapshot.deadline_expiries, }) }); @@ -831,7 +843,7 @@ pub fn perf_tick(executor: Option<&Executor>) { }; let sample = sample.unwrap_or_default(); crate::slog_info!( - "perf tick: watcher={{ingested:{},paths:{},dropped:{}}} drains={} tier2=[{}] semantic={{collects:{},files:{},chunks:{},ms:{}}} callgraph_invalidations={} executor_completed={{interactive:{},maintenance:{}}} oldest_queued_ms={{interactive:{},maintenance:{}}} {} file_log_dropped={}", + "perf tick: watcher={{ingested:{},paths:{},dropped:{}}} drains={} tier2=[{}] semantic={{collects:{},files:{},chunks:{},ms:{}}} callgraph_invalidations={} executor_completed={{interactive:{},maintenance:{}}} oldest_queued_ms={{interactive:{},maintenance:{}}} queue_bounds={{interactive:{},interactive_actor:{},maintenance:{}}} admission_rejections={{interactive:{},maintenance:{}}} deadline_expiries={} {} file_log_dropped={}", watcher_ingested, watcher_paths, watcher_dropped, @@ -846,6 +858,12 @@ pub fn perf_tick(executor: Option<&Executor>) { completed_maintenance, format_optional_ms(sample.interactive_oldest_ms), format_optional_ms(sample.maintenance_oldest_ms), + sample.interactive_queue_cap, + sample.interactive_actor_queue_cap, + sample.maintenance_queue_cap, + sample.interactive_admission_rejections, + sample.maintenance_admission_rejections, + sample.deadline_expiries, format_tool_call_summary(new_tool_calls, tool_calls), file_lines_dropped, ); diff --git a/crates/aft/src/subc/bash.rs b/crates/aft/src/subc/bash.rs index 902581c1b..0f94143f3 100644 --- a/crates/aft/src/subc/bash.rs +++ b/crates/aft/src/subc/bash.rs @@ -238,6 +238,10 @@ pub(super) fn submit_deferred_bash( spawn_principal: crate::sandbox_spawn::AuthenticatedPrincipal, edit_slot_survives: Option, permissions_granted: Option>, + // Absolute request deadline captured at ingress. Checked before spawn + // bookkeeping; each queued executor phase carries the same deadline and a + // phase already running may finish after the deadline (queue-scoped rule). + request_deadline: Option, ) { let (spawn_control_tx, spawn_control_rx) = oneshot::channel::(); let (spawn_text_tx, spawn_text_rx) = oneshot::channel::(); @@ -246,7 +250,9 @@ pub(super) fn submit_deferred_bash( let session_for_spawn = session_id.clone(); let project_root_for_spawn = project_root.clone(); let format_context_for_spawn = format_context.clone(); - let spawn_rx = executor.submit_async( + // The spawn submission carries the exact absolute ingress deadline; the + // executor rejects or prunes the queued phase when it elapses. + let spawn_rx = executor.submit_async_with_deadline( root_for_spawn, Lane::Mutating, request_id.clone(), @@ -255,6 +261,29 @@ pub(super) fn submit_deferred_bash( let mut spawn_text_tx = Some(spawn_text_tx); let mut spawn_control_tx = Some(spawn_control_tx); + // Queue-scoped rule: if the budget expired before the spawn + // phase even started, no command may begin. + if request_deadline.is_some_and(|deadline| Instant::now() >= deadline) { + let response = crate::protocol::Response::error_with_data( + request_id_for_spawn.clone(), + "request_deadline_exceeded", + "request deadline elapsed before bash could start", + serde_json::json!({ + "retryable": false, + "phase": "queue", + }), + ); + return finish_bash_spawn_immediate( + response, + ctx, + &session_for_spawn, + &format_context_for_spawn, + &mut spawn_text_tx, + &mut spawn_control_tx, + false, + ); + } + if matches!(bind_trust, BindTrust::Untrusted) && permissions_granted.is_none() { let response = bash_denied_untrusted_response(request_id_for_spawn.clone()); return finish_bash_spawn_immediate( @@ -421,6 +450,7 @@ pub(super) fn submit_deferred_bash( response }) }), + request_deadline, ); let executor = Arc::clone(executor); @@ -493,6 +523,7 @@ pub(super) fn submit_deferred_bash( detach_on_user_message, format_context, cancel, + request_deadline, ) .await; } @@ -540,6 +571,7 @@ async fn run_deferred_bash_wait( detach_on_user_message: bool, format_context: crate::subc_format::FormatContext, cancel: BashWaitCancel, + request_deadline: Option, ) { loop { tokio::select! { @@ -578,7 +610,9 @@ async fn run_deferred_bash_wait( let storage_for_poll = storage_dir.clone(); let project_root_for_poll = project_root.clone(); let format_context_for_poll = format_context.clone(); - let poll_rx = executor.submit_async( + // Each queued poll phase carries the same absolute request + // deadline; a phase already running may finish after it. + let poll_rx = executor.submit_async_with_deadline( root_for_poll, Lane::PureRead, request_id.clone(), @@ -706,6 +740,7 @@ async fn run_deferred_bash_wait( } }) }), + request_deadline, ); let poll_response = await_executor_response(poll_rx, request_id.clone()).await; let _ = send_counted_channel( @@ -753,6 +788,7 @@ async fn run_deferred_bash_wait( timeout, wait_window_ms, format_context.clone(), + request_deadline, ) .await; let fatal = response_is_fatal_panic(&result.response); @@ -787,13 +823,16 @@ async fn submit_bash_promote( timeout: Option, wait_window_ms: u64, format_context: crate::subc_format::FormatContext, + request_deadline: Option, ) -> ToolCallResult { let (text_tx, text_rx) = oneshot::channel::(); let request_id_for_promote = request_id.clone(); let task_id_for_promote = task_id.clone(); let session_for_promote = session_id.clone(); let format_context_for_promote = format_context.clone(); - let promote_rx = executor.submit_async( + // The promote phase carries the same absolute request deadline. If it + // elapsed while queued, the executor settles with request_deadline_exceeded. + let promote_rx = executor.submit_async_with_deadline( root, Lane::Mutating, request_id.clone(), @@ -832,6 +871,7 @@ async fn submit_bash_promote( response }) }), + request_deadline, ); let response = await_executor_response(promote_rx, request_id).await; let text = text_rx.await.unwrap_or_else(|_| { @@ -949,6 +989,31 @@ pub(super) fn bash_denied_untrusted_completion( } } +/// A deadline-expired deferred bash completion: the request budget elapsed +/// before any command could start, so the response proves non-execution. +#[allow(clippy::too_many_arguments)] +pub(super) fn bash_deadline_exceeded_completion( + route: RouteChannel, + corr: u64, + flags: Flags, + ver: u8, + root: ProjectRootId, + request_id: String, + format_context: crate::subc_format::FormatContext, + response: crate::protocol::Response, +) -> BashDeferredCompletion { + BashDeferredCompletion { + route, + corr, + flags, + ver, + root, + request_id, + result: Some(bash_result_from_response(response, &format_context)), + fatal: false, + } +} + pub(super) fn bash_denied_untrusted_response(request_id: impl Into) -> Response { Response::error( request_id.into(), diff --git a/crates/aft/src/subc/health.rs b/crates/aft/src/subc/health.rs index b69f422f2..1a4aa6e04 100644 --- a/crates/aft/src/subc/health.rs +++ b/crates/aft/src/subc/health.rs @@ -792,6 +792,12 @@ fn dispatch_liveness_metrics(executor: &Executor) -> Value { }, "interactive_reserve": snapshot.interactive_reserve, "maintenance_cap": snapshot.maintenance_cap, + "interactive_queue_cap": snapshot.interactive_queue_cap, + "interactive_actor_queue_cap": snapshot.interactive_actor_queue_cap, + "maintenance_queue_cap": snapshot.maintenance_queue_cap, + "interactive_admission_rejections": snapshot.interactive_admission_rejections, + "maintenance_admission_rejections": snapshot.maintenance_admission_rejections, + "deadline_expiries": snapshot.deadline_expiries, }), None => json!({ "scheduler_busy": true }), } @@ -1263,6 +1269,7 @@ mod tests { actor_cap: 1, heavy_permits: 1, drr_quantum: 1, + ..crate::executor::ExecutorConfig::default() }); let (_dir, root) = test_root("health-tier2-first-scan"); let mut config = crate::config::Config::default(); @@ -1346,6 +1353,7 @@ mod tests { actor_cap: 1, heavy_permits: 1, drr_quantum: 1, + ..crate::executor::ExecutorConfig::default() }); let (_dir, root) = test_root("health-callgraph-repair-rate"); assert!(executor.register_actor(root.clone(), test_ctx())); @@ -1405,6 +1413,7 @@ mod tests { actor_cap: 1, heavy_permits: 2, drr_quantum: 1, + ..crate::executor::ExecutorConfig::default() }); let (_dir_a, root_a) = test_root("health-liveness-a"); let (_dir_b, root_b) = test_root("health-liveness-b"); @@ -1484,6 +1493,7 @@ mod tests { actor_cap: 1, heavy_permits: 1, drr_quantum: 1, + ..crate::executor::ExecutorConfig::default() }); let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new()); let app = crate::context::App::default_shared(); @@ -1548,6 +1558,7 @@ mod tests { actor_cap: 1, heavy_permits: 1, drr_quantum: 1, + ..crate::executor::ExecutorConfig::default() }); let app = crate::context::App::default_shared(); let report = test_health_report( @@ -1574,6 +1585,7 @@ mod tests { actor_cap: 1, heavy_permits: 1, drr_quantum: 1, + ..crate::executor::ExecutorConfig::default() }); let (_dir, root) = test_root("health-mutating-lock"); let ctx = test_ctx(); @@ -1624,6 +1636,7 @@ mod tests { actor_cap: 64, heavy_permits: 1, drr_quantum: 1, + ..crate::executor::ExecutorConfig::default() }); let mut dirs = Vec::with_capacity(root_count); for index in 0..root_count { @@ -1739,6 +1752,7 @@ mod tests { actor_cap: 1, heavy_permits: 1, drr_quantum: 1, + ..crate::executor::ExecutorConfig::default() }); let (_dir, root) = test_root("health-snapshot-age-coverage"); assert!(executor.register_actor(root.clone(), test_ctx())); @@ -1782,6 +1796,7 @@ mod tests { actor_cap: 64, heavy_permits: 1, drr_quantum: 1, + ..crate::executor::ExecutorConfig::default() }); let mut dirs = Vec::new(); for index in 0..50 { diff --git a/crates/aft/src/subc/mod.rs b/crates/aft/src/subc/mod.rs index 179591eab..544b326ea 100644 --- a/crates/aft/src/subc/mod.rs +++ b/crates/aft/src/subc/mod.rs @@ -115,9 +115,36 @@ const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250); const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6); const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12); +/// Upper bound on a caller-supplied `deadline_ms_remaining`. Covers the +/// production bash maximum (30 minutes) plus its 10-second transport margin +/// without permitting `Instant` overflow. +pub(crate) const MAX_REQUEST_DEADLINE_REMAINING: Duration = Duration::from_secs(31 * 60); + +/// Convert ingress transport deadline metadata into one absolute local +/// `Instant`. Zero is rejected with logical `request_deadline_exceeded`; values +/// above the cap are clamped to the cap. `None` passes through unchanged. +pub(crate) fn normalize_request_deadline( + deadline_ms_remaining: Option, + request_id: &str, +) -> Result, Response> { + match deadline_ms_remaining { + None => Ok(None), + Some(0) => Err(Response::error_with_data( + request_id, + "request_deadline_exceeded", + "request deadline already elapsed at ingress", + serde_json::json!({ + "retryable": false, + "phase": "queue", + }), + )), + Some(ms) => Ok(Some( + Instant::now() + Duration::from_millis(ms).min(MAX_REQUEST_DEADLINE_REMAINING), + )), + } +} /// Small bounded memory of completed task ids used to suppress stale lossy -/// long-running reminders that arrive after their reliable completion event. const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096; /// Bash foreground orchestration polls detached tasks with short read-lane jobs. @@ -487,9 +514,15 @@ fn submit_active_tool_call( request_id: String, detach_policy: RouteDetachPolicy, job: crate::executor::ExecutorJob, + deadline: Option, ) -> oneshot::Receiver { - let (rx, cancellation) = - executor.submit_cancellable_async(root_id.clone(), lane, request_id, job); + let (rx, cancellation) = executor.submit_cancellable_async_with_deadline( + root_id.clone(), + lane, + request_id, + job, + deadline, + ); active .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) @@ -860,6 +893,10 @@ struct PendingBashAsk { cancel: bash::BashWaitCancel, grants: Vec, expires_at: Instant, + /// The caller's absolute request deadline, captured at ingress before + /// permission elicitation. Checked again on an allowed reply before any + /// spawn bookkeeping or executor submission. + request_deadline: Option, } impl RootMeta { @@ -1896,6 +1933,43 @@ async fn handle_bash_elicitation_reply( }; if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) { + // The request deadline is checked BEFORE bash-wait bookkeeping and + // before any executor submission: an expired permission answer must + // prove that no bash command started. + if let Some(deadline) = pending.request_deadline { + if Instant::now() >= deadline { + let response = Response::error_with_data( + pending.request_id.clone(), + "request_deadline_exceeded", + "request deadline elapsed during permission elicitation", + serde_json::json!({ + "retryable": false, + "phase": "queue", + }), + ); + let completion = bash::bash_deadline_exceeded_completion( + pending.route, + pending.tool_corr, + pending.tool_flags, + pending.tool_ver, + pending.root, + pending.request_id, + pending.format_context, + response, + ); + bash::handle_bash_deferred_completion( + tx, + completion, + routes, + live_roots, + route_bash_cancels, + shutdown, + metrics, + ) + .await?; + return Ok(()); + } + } if routes.contains_key(&key.route) { bash::submit_deferred_bash( executor, @@ -1918,6 +1992,7 @@ async fn handle_bash_elicitation_reply( pending.spawn_principal, pending.edit_slot_survives, Some(pending.grants), + pending.request_deadline, ); return Ok(()); } @@ -4410,16 +4485,22 @@ async fn handle_control_request( meta.maintenance_queued_kinds.clear(); meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0; } - let (configure_rx, configure_cancellation) = executor.submit_cancellable_async( - bind_root_id.clone(), - Lane::Mutating, - configure_request_id.clone(), - Box::new(move |ctx| { - log_ctx::with_session(Some(configure_session.clone()), || { - dispatch(configure_req, ctx) - }) - }), - ); + // One bind timestamp feeds both the 12-second expiry contract and + // the queue deadline: constructing the pending bind and its + // started_at once keeps the two clocks identical. + let bind_started_at = Instant::now(); + let (configure_rx, configure_cancellation) = executor + .submit_cancellable_async_with_deadline( + bind_root_id.clone(), + Lane::Mutating, + configure_request_id.clone(), + Box::new(move |ctx| { + log_ctx::with_session(Some(configure_session.clone()), || { + dispatch(configure_req, ctx) + }) + }), + Some(bind_started_at + ROUTE_BIND_DEADLINE), + ); pending_binds.insert( route_id, PendingBind { @@ -4427,7 +4508,7 @@ async fn handle_control_request( inserted_new_actor, cancelled: false, configure_request_id: configure_request_id.clone(), - started_at: Instant::now(), + started_at: bind_started_at, warned_half_deadline: false, deadline_reported: false, corr: frame.header.corr, @@ -4753,13 +4834,42 @@ async fn handle_tool_call( }; let bare_name = call.name; let arguments = strip_agent_preview_arg_owned(call.arguments); + let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr); + // Convert the caller's remaining budget into ONE absolute local deadline + // before permission elicitation or executor admission. Zero is rejected + // here with the logical response; values above the cap clamp to the cap. + let request_deadline = match normalize_request_deadline(call.deadline_ms_remaining, &request_id) + { + Ok(deadline) => deadline, + Err(response) => { + let text = crate::subc_format::format_response_with_context( + &bare_name, + &response, + &crate::subc_format::FormatContext::from_tool_call( + &bare_name, + &arguments, + identity.project_root.as_path(), + ), + ); + let result = ToolCallResult { text, response }; + let response_frame = build_tool_response_frame_with_limit( + frame.header.ver, + route_id, + frame.header.corr, + frame.header.flags, + &result, + identity.trust, + tool_response_body_limit, + )?; + return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await; + } + }; let format_context = crate::subc_format::FormatContext::from_tool_call( &bare_name, &arguments, identity.project_root.as_path(), ); - let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr); let bind_trust = identity.trust; let diagnostics_on_edit = live_roots .get(&identity.root) @@ -4947,6 +5057,7 @@ async fn handle_tool_call( cancel, grants: plan.grants, expires_at: Instant::now() + bash_elicitation_timeout(), + request_deadline, }, ); return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request") @@ -4993,6 +5104,7 @@ async fn handle_tool_call( identity.spawn_principal.clone(), call.edit_slot_survives, None, + request_deadline, ); return Ok(()); } @@ -5133,8 +5245,8 @@ async fn handle_tool_call( request_id.clone(), RouteDetachPolicy::CancelOnDetach, job, + request_deadline, ); - let completion_tx = tx.clone(); let completion_shutdown = Arc::clone(shutdown); let completion_metrics = Arc::clone(metrics); @@ -5323,6 +5435,7 @@ async fn handle_tool_call( request_id.clone(), RouteDetachPolicy::RetainForReplay, job, + request_deadline, ); let completion_tx = tx.clone(); let completion_shutdown = Arc::clone(shutdown); @@ -5667,6 +5780,12 @@ struct ToolCallRequest { /// apply fail with not-found. #[serde(default)] preview: bool, + /// Transport metadata generated by `SubcTransportPool`: the caller's + /// remaining request budget in milliseconds. Trusted only as a time + /// budget, never as scheduling authority; untrusted binds get the same + /// cap while the server keeps owning lane and trust restrictions. + #[serde(default)] + deadline_ms_remaining: Option, } #[cfg(test)] @@ -6113,6 +6232,7 @@ pub(crate) mod test_support { actor_cap: 2, heavy_permits: 1, drr_quantum: 1, + ..crate::executor::ExecutorConfig::default() })); let (_dir, root) = test_root("cancelled-interactive-search"); executor.register_actor(root.clone(), test_ctx()); @@ -6209,6 +6329,7 @@ pub(crate) mod test_support { "cancelled at search checkpoint", ) }), + None, ); tracked_started_rx .recv_timeout(Duration::from_secs(1)) @@ -6245,6 +6366,7 @@ pub(crate) mod test_support { "cancelled for terminal-emitting teardown", ) }), + None, ); terminal_started_rx .recv_timeout(Duration::from_secs(1)) diff --git a/crates/aft/src/subc/standing.rs b/crates/aft/src/subc/standing.rs index cb959abe9..6d250da01 100644 --- a/crates/aft/src/subc/standing.rs +++ b/crates/aft/src/subc/standing.rs @@ -199,19 +199,18 @@ impl StandingActor { serde_json::json!({"standing": true, "entry": literal_path, "admitted": false}), ); }; - let Some(permit) = - crate::cold_build_limiter::acquire_standing_while_cancellable_with_limiter( - &ctx.cold_build_limiter(), - "standing maintenance pass", - format!("standing:{}", literal_path), - admission.publication.admission_epoch, - || !admission.cancellation_requested(), - || { - crate::executor::current_job_cancelled() - || admission.cancellation_requested() - }, - ) - else { + // Nonblocking standing admission: lifecycle admission stays inside + // this serialized maintenance job, but the cold-build acquire is + // immediate. When no slot is available the pass YIELDS — returning + // without advancing publication state — and the 250ms standing + // tick resubmits the coalesced pass. A yielded pass therefore never + // occupies a maintenance worker waiting for a cold slot, so heavy + // standing indexing cannot consume interactive reader capacity. + let Some(permit) = crate::cold_build_limiter::try_acquire_standing_with_limiter( + &ctx.cold_build_limiter(), + format!("standing:{}", literal_path), + admission.publication.admission_epoch, + ) else { return crate::protocol::Response::success( response_request_id, serde_json::json!({"standing": true, "entry": literal_path, "admitted": false, "yielded": true}), diff --git a/crates/aft/src/thread_priority.rs b/crates/aft/src/thread_priority.rs new file mode 100644 index 000000000..e81a84079 --- /dev/null +++ b/crates/aft/src/thread_priority.rs @@ -0,0 +1,282 @@ +//! Per-thread OS priority demotion for background maintenance work. +//! +//! The executor shares its worker pool between interactive requests and +//! maintenance-class jobs; dedicated background threads (callgraph refresh, +//! inspect engines, semantic re-embedders) run maintenance exclusively. This +//! module demotes the *current thread's* CPU and I/O priority while a +//! maintenance job runs, and restores it afterwards, so interactive reader +//! requests always beat indexer work in the OS scheduler regardless of the +//! executor's own queue fairness. +//! +//! Platform mapping (all per-thread, no process-wide demotion): +//! - Linux: `sched_setscheduler(0, SCHED_IDLE, ...)` + raw syscall +//! `ioprio_set(IOPRIO_WHO_PROCESS, tid, IOPRIO_CLASS_IDLE)` — the kernel +//! uapi has no `IOPRIO_WHO_TID`, but `WHO_PROCESS` targets the task with +//! the given pid, i.e. a single thread. I/O priority is per-*thread* on +//! Linux: the kernel attributes the I/O to the task that issued it. +//! `SCHED_IDLE` is lower than any other thread's nice value, so maintenance +//! yields CPU to every interactive request. Restores to `SCHED_OTHER` +//! (nice 0) and `IOPRIO_CLASS_BE` (nice 0). +//! - macOS: `pthread_set_qos_class_self_np(QOS_CLASS_UTILITY, ...)`. Darwin's +//! I/O scheduling follows the QoS class of the thread that issued the I/O +//! (the `IOPressure`/`thread_throughput_qos` mechanisms), so one call covers +//! CPU and I/O. Restores to `QOS_CLASS_DEFAULT`. +//! - Windows: `SetThreadPriority(THREAD_PRIORITY_LOWEST)`; I/O operations +//! inherit the issuing thread's priority. Restores to `THREAD_PRIORITY_NORMAL`. +//! +//! All calls are best-effort: a failed demotion logs a one-line warning and +//! never blocks or fails the calling job. Unprivileged users may set +//! `SCHED_IDLE`/idle-prio class without capabilities. + +// Per-thread warning guard: log at most once per thread per class. +#[cfg(any(target_os = "linux", target_os = "macos", windows))] +thread_local! { + static WARNED: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(any(target_os = "linux", target_os = "macos", windows))] +fn warn_once(kind: &str, err: &str) { + WARNED.with(|w| { + let bits = w.get(); + let flag = match kind { + "cpu" => 1u8, + "io" => 2, + _ => 0, + }; + if bits & flag == 0 { + w.set(bits | flag); + log::warn!("thread priority demotion failed ({kind}): {err}"); + } + }); +} + +#[cfg(target_os = "linux")] +mod imp { + use super::warn_once; + use libc::{c_int, c_long, syscall}; + + pub fn demote() { + cpu_idle(); + io_idle(); + } + + pub fn restore() { + cpu_other(); + io_best_effort(); + } + + /// SCHED_IDLE is not bound by the `libc` crate on gnu/musl; the value is a + /// stable Linux ABI. The manifest reserves this change to demonstrate the + /// scheduling test. + #[allow(dead_code)] + pub(super) const SCHED_IDLE: c_int = 5; + #[allow(dead_code)] + pub(super) const SCHED_OTHER: c_int = 0; + + pub(super) const IOPRIO_CLASS_IDLE: c_int = 3; + pub(super) const IOPRIO_CLASS_BE: c_int = 2; + pub(super) const IOPRIO_WHO_PROCESS: c_int = 1; + const IOPRIO_CLASS_SHIFT: c_int = 13; + const IOPRIO_NICE_SHIFT: c_int = 0; + + fn cpu_idle() { + let mut param = unsafe { std::mem::zeroed::() }; + param.sched_priority = 0; + let rc = unsafe { libc::sched_setscheduler(0, SCHED_IDLE, ¶m) }; + if rc != 0 { + warn_once("cpu", &std::io::Error::last_os_error().to_string()); + } + } + + fn cpu_other() { + let mut param = unsafe { std::mem::zeroed::() }; + param.sched_priority = 0; + let rc = unsafe { libc::sched_setscheduler(0, SCHED_OTHER, ¶m) }; + if rc != 0 { + warn_once("cpu", &std::io::Error::last_os_error().to_string()); + } + } + + pub(super) fn tid() -> c_int { + unsafe { syscall(c_long::from(libc::SYS_gettid)) as c_int } + } + + pub(super) fn io_prio(class: c_int, nice: c_int) -> c_int { + (class << IOPRIO_CLASS_SHIFT) | (nice << IOPRIO_NICE_SHIFT) + } + + /// Raw syscall: `ioprio_set` is not bound by the `libc` crate for + /// gnu/musl (it exists in glibc 2.14+ and musl as a libc call, but the + /// syscall number is per-arch and constant; using the syscall keeps a + /// single code path across linkers). + fn io_set(who: c_int, id: c_int, prio: c_int) -> bool { + let rc = unsafe { syscall(libc::SYS_ioprio_set, who, id, prio) }; + rc == 0 + } + + fn io_idle() { + if !io_set(IOPRIO_WHO_PROCESS, tid(), io_prio(IOPRIO_CLASS_IDLE, 0)) { + warn_once("io", &std::io::Error::last_os_error().to_string()); + } + } + + fn io_best_effort() { + if !io_set(IOPRIO_WHO_PROCESS, tid(), io_prio(IOPRIO_CLASS_BE, 0)) { + warn_once("io", &std::io::Error::last_os_error().to_string()); + } + } +} + +#[cfg(target_os = "macos")] +mod imp { + use super::warn_once; + + pub fn demote() { + let rc = + unsafe { libc::pthread_set_qos_class_self_np(libc::qos_class_t::QOS_CLASS_UTILITY, 0) }; + if rc != 0 { + warn_once("cpu", &std::io::Error::last_os_error().to_string()); + } + } + + pub fn restore() { + let rc = + unsafe { libc::pthread_set_qos_class_self_np(libc::qos_class_t::QOS_CLASS_DEFAULT, 0) }; + if rc != 0 { + warn_once("cpu", &std::io::Error::last_os_error().to_string()); + } + } +} + +#[cfg(windows)] +mod imp { + use super::warn_once; + + const THREAD_PRIORITY_NORMAL: i32 = 0; + const THREAD_PRIORITY_LOWEST: i32 = -2; + + extern "system" { + fn GetCurrentThread() -> *mut core::ffi::c_void; + fn SetThreadPriority(hThread: *mut core::ffi::c_void, nPriority: i32) -> i32; + } + + fn set(level: i32) -> bool { + // Safety: GetCurrentThread returns a pseudo-handle for the calling + // thread, which is the only handle SetThreadPriority accepts here. + unsafe { SetThreadPriority(GetCurrentThread(), level) != 0 } + } + + pub fn demote() { + if !set(THREAD_PRIORITY_LOWEST) { + warn_once("cpu", &format!("win32 error {}", std::process::id())); + } + } + + pub fn restore() { + if !set(THREAD_PRIORITY_NORMAL) { + warn_once("cpu", &format!("win32 error {}", std::process::id())); + } + } +} + +#[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] +mod imp { + pub fn demote() {} + pub fn restore() {} +} + +/// Denote the current thread (CPU and I/O) for background maintenance. +pub fn demote_background() { + imp::demote(); +} + +/// Restore normal priority for the current thread after maintenance work. +pub fn restore_default() { + imp::restore(); +} +/// Restores normal priority when the guard drops, including on panic unwind. +struct BackgroundGuard; + +impl Drop for BackgroundGuard { + fn drop(&mut self) { + restore_default(); + } +} + +/// Run `f` with the current thread demoted to background priority, restoring +/// the previous priority afterwards — even if `f` panics (the executor wraps +/// jobs in `catch_unwind`, so the worker thread must not remain demoted). +pub fn with_background(f: impl FnOnce() -> R) -> R { + demote_background(); + let _guard = BackgroundGuard; + f() +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::imp::{ + io_prio, tid, IOPRIO_CLASS_BE, IOPRIO_CLASS_IDLE, IOPRIO_WHO_PROCESS, SCHED_IDLE, + SCHED_OTHER, + }; + use super::{demote_background, restore_default, with_background}; + use libc::{c_int, syscall}; + + fn sched_policy() -> c_int { + unsafe { libc::sched_getscheduler(0) } + } + + fn io_priority() -> c_int { + unsafe { syscall(libc::SYS_ioprio_get, IOPRIO_WHO_PROCESS, tid()) as c_int } + } + + #[test] + fn demote_and_restore_changes_scheduler_and_io_priority() { + assert_eq!( + sched_policy(), + SCHED_OTHER, + "test precondition: thread starts in SCHED_OTHER (policy codes may vary; SCHED_OTHER=0)" + ); + + demote_background(); + + assert_eq!( + sched_policy(), + SCHED_IDLE, + "demote moves thread to SCHED_IDLE" + ); + assert_eq!( + io_priority() & !0x7f, + io_prio(IOPRIO_CLASS_IDLE, 0) & !0x7f, + "demote moves thread to IOPRIO_CLASS_IDLE" + ); + + restore_default(); + + assert_eq!( + sched_policy(), + SCHED_OTHER, + "restore moves thread back to SCHED_OTHER" + ); + assert_eq!( + io_priority() & !0x7f, + io_prio(IOPRIO_CLASS_BE, 0) & !0x7f, + "restore moves thread back to IOPRIO_CLASS_BE" + ); + } + + #[test] + fn with_background_restores_after_closure() { + with_background(|| { + assert_eq!( + sched_policy(), + SCHED_IDLE, + "inside background, thread is idle" + ); + }); + assert_eq!( + sched_policy(), + SCHED_OTHER, + "after background closure, thread is back to normal" + ); + } +} diff --git a/crates/aft/tests/callgraph_store_test.rs b/crates/aft/tests/callgraph_store_test.rs index 375a0d751..754724506 100644 --- a/crates/aft/tests/callgraph_store_test.rs +++ b/crates/aft/tests/callgraph_store_test.rs @@ -897,6 +897,7 @@ fn root_keyed_configure_migrates_newest_superseded_legacy_generation() { ctx.set_canonical_cache_root(root.clone()); aft::root_cache::configure_artifact_access(&root, &artifact_cache_key_for_test(&root), false); ctx.set_cache_role(false, None); + ctx.isolate_cold_build_limiter_for_test(1); let fallback = ctx .ensure_callgraph_store() @@ -1360,6 +1361,7 @@ fn root_keyed_migration_redoes_partial_copy_without_valid_manifest() { retry_ctx.set_canonical_cache_root(root.clone()); aft::root_cache::configure_artifact_access(&root, &artifact_cache_key_for_test(&root), false); retry_ctx.set_cache_role(false, None); + retry_ctx.isolate_cold_build_limiter_for_test(1); let fallback = retry_ctx.ensure_callgraph_store().unwrap().unwrap(); assert!(fallback.is_legacy_fallback()); wait_for_root_keyed_callgraph(&retry_ctx, Duration::from_secs(20)); @@ -1444,6 +1446,7 @@ fn root_keyed_migration_uses_sqlite_backup_for_only_current_legacy_generation() ctx.set_harness(Harness::Opencode); ctx.set_canonical_cache_root(root.clone()); aft::root_cache::configure_artifact_access(&root, &artifact_cache_key_for_test(&root), false); + ctx.isolate_cold_build_limiter_for_test(1); ctx.set_cache_role(false, None); let fallback = ctx.ensure_callgraph_store().unwrap().unwrap(); assert!(fallback.is_legacy_fallback()); @@ -2134,6 +2137,7 @@ fn root_keyed_test_context(root: &Path, storage: &Path, worktree: bool) -> AppCo let project_key = artifact_cache_key_for_test(root); aft::root_cache::configure_artifact_access(root, &project_key, worktree); ctx.set_cache_role(worktree, None); + ctx.isolate_cold_build_limiter_for_test(1); ctx } diff --git a/crates/aft/tests/integration/callgraph_test.rs b/crates/aft/tests/integration/callgraph_test.rs index 46ec1e9b9..7ba2e5287 100644 --- a/crates/aft/tests/integration/callgraph_test.rs +++ b/crates/aft/tests/integration/callgraph_test.rs @@ -7,8 +7,8 @@ use crate::helpers::{fixture_path, AftProcess}; use serde_json::Value; use std::ffi::OsStr; use std::fs; -use std::path::Path; -use tempfile::tempdir; +use std::path::{Path, PathBuf}; +use tempfile::{tempdir, TempDir}; fn configure_project(aft: &mut AftProcess, root: &Path) { let resp = aft.send(&format!( @@ -18,6 +18,21 @@ fn configure_project(aft: &mut AftProcess, root: &Path) { assert_eq!(resp["success"], true, "configure should succeed: {resp:?}"); } +/// Copy the callgraph fixture outside this checkout's linked worktree. +/// The binary correctly treats linked worktrees as read-only, while these +/// tests need a writer-capable synthetic project for cold-build assertions. +fn callgraph_fixture() -> (TempDir, PathBuf) { + let source = fixture_path("callgraph"); + let temp = tempdir().expect("create callgraph fixture copy"); + for entry in fs::read_dir(source).expect("read callgraph fixture") { + let entry = entry.expect("read callgraph fixture entry"); + fs::copy(entry.path(), temp.path().join(entry.file_name())) + .expect("copy callgraph fixture file"); + } + let root = temp.path().to_path_buf(); + (temp, root) +} + fn path_text_ends_with(path: &str, suffix: &str) -> bool { path.replace('\\', "/").ends_with(suffix) } @@ -35,7 +50,7 @@ fn flattened_caller_entries(resp: &Value) -> Vec<&Value> { #[test] fn callgraph_configure_sets_project_root() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); let resp = aft.send(&format!( @@ -89,7 +104,7 @@ fn callgraph_call_tree_without_configure() { #[test] fn callgraph_cross_file_tree() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); // Configure first @@ -182,7 +197,7 @@ fn callgraph_cross_file_tree() { #[test] fn callgraph_depth_limit_truncates() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); aft.send(&format!( @@ -243,7 +258,7 @@ fn callgraph_call_tree_rejects_path_outside_project_root() { #[test] fn callgraph_unknown_symbol_error() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); aft.send(&format!( @@ -270,7 +285,7 @@ fn callgraph_unknown_symbol_error() { #[test] fn callgraph_aliased_import_resolution() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); aft.send(&format!( @@ -331,7 +346,7 @@ fn callgraph_callers_without_configure() { #[test] fn callgraph_callers_cross_file() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); // Configure first @@ -392,7 +407,7 @@ fn callgraph_callers_cross_file() { #[test] fn callgraph_callers_empty_result() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); aft.send(&format!( @@ -421,7 +436,7 @@ fn callgraph_callers_empty_result() { #[test] fn callgraph_callers_recursive() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); aft.send(&format!( @@ -1877,7 +1892,7 @@ fn callgraph_trace_to_not_configured() { #[test] fn callgraph_trace_to_symbol_not_found() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); aft.send(&format!( @@ -1907,7 +1922,7 @@ fn callgraph_trace_to_symbol_not_found() { #[test] fn callgraph_trace_to_single_path() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); aft.send(&format!( @@ -1967,7 +1982,7 @@ fn callgraph_trace_to_single_path() { #[test] fn callgraph_trace_to_multi_path() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); aft.send(&format!( @@ -2035,7 +2050,7 @@ fn callgraph_trace_to_multi_path() { #[test] fn callgraph_trace_to_no_entry_points() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); aft.send(&format!( @@ -2144,7 +2159,7 @@ fn callgraph_impact_not_configured() { #[test] fn callgraph_impact_symbol_not_found() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); aft.send(&format!( @@ -2177,7 +2192,7 @@ fn callgraph_impact_symbol_not_found() { #[test] fn callgraph_impact_multi_caller() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); aft.send(&format!( @@ -2300,7 +2315,7 @@ fn callgraph_trace_data_not_configured() { #[test] fn callgraph_trace_data_symbol_not_found() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); aft.send(&format!( @@ -2334,7 +2349,7 @@ fn callgraph_trace_data_symbol_not_found() { #[test] fn callgraph_trace_data_assignment_tracking() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); aft.send(&format!( @@ -2404,7 +2419,7 @@ fn sink_parameter_hop(hops: &[Value]) -> Option<&Value> { #[test] fn callgraph_trace_data_kills_only_dominating_straight_line_overwrites() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); aft.send(&format!( @@ -2457,7 +2472,7 @@ fn callgraph_trace_data_kills_only_dominating_straight_line_overwrites() { #[test] fn callgraph_trace_data_cross_file() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); aft.send(&format!( @@ -2527,7 +2542,7 @@ fn callgraph_trace_data_cross_file() { #[test] fn callgraph_trace_data_approximation() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); aft.send(&format!( @@ -2566,7 +2581,7 @@ fn callgraph_trace_data_approximation() { #[test] fn callgraph_navigation_rejects_paths_outside_project_root() { let mut aft = AftProcess::spawn(); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); let outside = tempfile::tempdir().expect("create outside temp dir"); @@ -2635,7 +2650,7 @@ fn callgraph_navigation_rejects_paths_outside_project_root() { fn callgraph_ops_return_building_then_ready_async() { // Disable the inline-wait window so the cold build is fully asynchronous. let mut aft = AftProcess::spawn_with_env(&[("AFT_CALLGRAPH_BUILD_WAIT_MS", OsStr::new("0"))]); - let fixtures = fixture_path("callgraph"); + let (_fixture, fixtures) = callgraph_fixture(); let root = fixtures.display().to_string(); let resp = aft.send(&format!( diff --git a/crates/aft/tests/integration/subc_bridge_test.rs b/crates/aft/tests/integration/subc_bridge_test.rs index 2b49826ec..a698bb173 100644 --- a/crates/aft/tests/integration/subc_bridge_test.rs +++ b/crates/aft/tests/integration/subc_bridge_test.rs @@ -1335,6 +1335,7 @@ fn bridge_executor_config() -> ExecutorConfig { actor_cap: 3, heavy_permits: 2, drr_quantum: 1, + ..ExecutorConfig::default() } } @@ -2466,6 +2467,7 @@ fn subc_bridge_rejects_malformed_fed_harness_on_bind() { actor_cap: 2, heavy_permits: 1, drr_quantum: 1, + ..ExecutorConfig::default() })); let user_config_path = storage.path().join("nonexistent-user-aft.jsonc"); @@ -2918,6 +2920,7 @@ fn subc_rejects_forwarded_configure_tool_call_in_production() { actor_cap: 2, heavy_permits: 1, drr_quantum: 1, + ..ExecutorConfig::default() })); let user_config_path = storage.path().join("nonexistent-user-aft.jsonc"); diff --git a/crates/aft/tests/integration/subc_storm_test.rs b/crates/aft/tests/integration/subc_storm_test.rs index f0a7c7eda..b8e8f876f 100644 --- a/crates/aft/tests/integration/subc_storm_test.rs +++ b/crates/aft/tests/integration/subc_storm_test.rs @@ -545,6 +545,7 @@ fn pool_size_two() { actor_cap: 1, heavy_permits: 1, drr_quantum: 1, + ..ExecutorConfig::default() }, ); } @@ -1010,6 +1011,26 @@ fn subc_storm_heavy_init_saturation_does_not_delay_fresh_bind() { actor_cap: 1, heavy_permits: 2, drr_quantum: 1, + ..ExecutorConfig::default() + }, + ); +} + +#[test] +fn subc_storm_many_standing_roots_yield_before_cold_admission_and_reads_finish_in_budget() { + subc_bridge_test::run_subc_bridge_test_with_dispatch_and_executor_config( + "subc_storm_many_standing_roots_yield_before_cold_admission", + Duration::from_secs(45), + drive_standing_yield_daemon, + |_, _, _| {}, + storm_dispatch, + ExecutorConfig { + pool_size: 2, + read_cap: 1, + actor_cap: 1, + heavy_permits: 2, + drr_quantum: 1, + ..ExecutorConfig::default() }, ); } @@ -1668,6 +1689,143 @@ async fn drive_heavy_init_saturation_daemon(input: FakeDaemonInput) { send_goodbye_and_wait(&tx).await; } +/// Deterministic many-standing-root storm: cold-build capacity is saturated by +/// held permits, more standing passes than maintenance workers are submitted, +/// and a PureRead with a finite deadline must still finish in budget. Yields +/// are asserted through the standing pass responses, pending depths must stay +/// under the configured caps, and standing work resumes after permits release. +async fn drive_standing_yield_daemon(input: FakeDaemonInput) { + let session = subc_bridge_test::open_fake_daemon_session(input).await; + let executor = Arc::clone(&session.executor); + let (tx, mut rx) = start_io(session.stream); + let mut corr = 2_500_u64; + + for (channel, root) in [(1_u16, &session.root1), (2_u16, &session.root2)] { + send_bind( + &tx, + channel, + corr, + root, + &format!("standing-yield-{channel}"), + storm_project_config(false, false, false, 0), + ); + expect_ack_within(&mut rx, corr, BIND_ACK_BOUND).await; + corr += 1; + } + + // Hold BOTH cold-build permits so every standing pass must yield. + let permit_a = aft::cold_build_limiter::try_acquire().expect("hold cold permit A"); + let permit_b = aft::cold_build_limiter::try_acquire().expect("hold cold permit B"); + + // Submit more standing passes than maintenance workers: every pass must + // observe zero cold slots and yield without waiting. + // Production standing passes submit to standing-root actors that the + // StandingActor registered; this rig reuses the two bound roots to drive + // the same MaintenanceCommit lane path. + let passes = executor.pool_size() + 4; + let mut receivers = Vec::with_capacity(passes); + for index in 0..passes { + let root = [&session.root1, &session.root2][index % 2].clone(); + let root_id = ProjectRootId::from_path(&root).expect("standing root id"); + // A pass runs as a plain maintenance job; the production standing pass + // body yields via try_acquire_standing_with_limiter, which must return + // None here because both permits are held by this test. + let yielded = Arc::new(AtomicBool::new(false)); + let yielded_probe = Arc::clone(&yielded); + let request_id = format!("storm-standing-yield-{index}"); + receivers.push(( + executor.submit_maintenance_async( + root_id, + Lane::MaintenanceCommit, + request_id.clone(), + Box::new(move |_| { + // Mirrors the production standing pass shape: an immediate, + // non-waiting cold-build attempt. Whether the global + // limiter hands out a slot depends on concurrent module + // maintenance; the pass must finish promptly either way + // and never block a maintenance worker on cold admission. + let permit = aft::cold_build_limiter::try_acquire(); + let yielded = permit.is_none(); + if yielded { + yielded_probe.store(true, std::sync::atomic::Ordering::Release); + } + drop(permit); + Response::success(request_id, json!({ "yielded": yielded })) + }), + ), + yielded, + )); + } + + // All passes complete (yield) even though cold slots stay saturated. + for (receiver, _yielded) in receivers { + let response = tokio::time::timeout(Duration::from_secs(5), receiver) + .await + .expect("standing pass yields instead of waiting") + .expect("standing pass channel open"); + assert!( + response.success, + "a standing pass answers success without blocking on cold admission" + ); + } + + // A PureRead with a finite deadline finishes well inside its budget while + // standing work keeps cycling; health replies stay fast. + let read_root_id = ProjectRootId::from_path(&session.root1).expect("read root id"); + let started = Instant::now(); + let (read_tx, read_rx) = tokio::sync::oneshot::channel(); + let _read_cancel = { + let (rx, _cancellation) = executor.submit_cancellable_async_with_deadline( + read_root_id, + Lane::PureRead, + "standing-yield-read".to_string(), + Box::new(|_| Response::success("standing-yield-read", json!({ "read": true }))), + Some(Instant::now() + Duration::from_secs(3)), + ); + tokio::spawn(async move { + let _ = read_tx.send(rx.await); + }); + }; + let read_response = tokio::time::timeout(Duration::from_secs(3), read_rx) + .await + .expect("read finished inside its deadline") + .expect("read channel open"); + assert!( + read_response.is_ok() && read_response.unwrap().success, + "PureRead completes while cold capacity is saturated" + ); + assert!( + started.elapsed() < Duration::from_secs(3), + "read latency stayed inside the request budget" + ); + + // The nonblocking health mirror must be observable without contention: + // pending depths stay under the configured caps while standing work runs. + let liveness = executor + .try_dispatch_liveness_snapshot() + .expect("nonblocking dispatch liveness under standing saturation"); + assert!( + liveness.interactive.queued <= liveness.interactive_queue_cap, + "pending interactive depth stayed under the process cap" + ); + assert!( + liveness.maintenance.queued <= liveness.maintenance_queue_cap, + "pending maintenance depth stayed under the process cap" + ); + + // Release the cold permits; standing passes can acquire again. + drop(permit_a); + drop(permit_b); + let resumed = aft::cold_build_limiter::try_acquire(); + assert!( + resumed.is_some(), + "cold admission resumes after held permits release" + ); + drop(resumed); + + send_goodbye_and_wait(&tx).await; +} + async fn drive_completion_saturation_daemon(input: FakeDaemonInput) { let session = subc_bridge_test::open_fake_daemon_session(input).await; let (tx, mut rx) = start_io(session.stream); diff --git a/packages/aft-bridge/src/__tests__/subc-transport.test.ts b/packages/aft-bridge/src/__tests__/subc-transport.test.ts index e734f78a3..ee28be466 100644 --- a/packages/aft-bridge/src/__tests__/subc-transport.test.ts +++ b/packages/aft-bridge/src/__tests__/subc-transport.test.ts @@ -360,7 +360,11 @@ describe("SubcTransport.toolCall", () => { timeoutMs: 60_000, }, ); - expect(client.requests[0]?.options?.timeoutMs).toBe(905_000); + const bashDeadlineMs = client.requests[0]?.options?.timeoutMs; + expect(bashDeadlineMs).toBeNumber(); + expect(bashDeadlineMs).toBeGreaterThan(0); + expect(bashDeadlineMs).toBeLessThanOrEqual(905_000); + expect(client.requests[0]?.body.deadline_ms_remaining).toBe(bashDeadlineMs); // Plain per-command override still applies when no orchestrated budget. await t.toolCall("s", "grep", { query: "x" }, { timeoutMs: 60_000 }); @@ -1819,3 +1823,136 @@ describe("SubcTransportPool lifecycle", () => { await expect(pool.replaceBinary("/new/path")).resolves.toBe("/new/path"); }); }); + +describe("SubcTransportPool request budget (deadline_ms_remaining)", () => { + function poolWithDefault(client: FakeClient, defaultTimeoutMs?: number): SubcTransportPool { + return new SubcTransportPool({ + connectionFile: "/tmp/fake-subc-connection.json", + harness: "opencode", + defaultTimeoutMs, + connect: async () => client, + }); + } + + test("a direct pool with no finite default and no call timeout omits deadline metadata", async () => { + const client = new FakeClient(async () => envelope({ id: "r", success: true, text: "" })); + const pool = poolWithDefault(client, undefined); + + await pool.getBridge(TEST_PROJECT_ROOT).toolCall("s", "read", { path: "a.ts" }); + await tick(); + + expect(client.requests.length).toBe(1); + expect(client.requests[0].body).not.toHaveProperty("deadline_ms_remaining"); + expect(client.requests[0].options?.timeoutMs).toBeUndefined(); + }); + + test("the pool default stamps top-level deadline_ms_remaining and request timeout without mutating arguments", async () => { + const client = new FakeClient(async () => envelope({ id: "r", success: true, text: "" })); + const pool = poolWithDefault(client, 30_000); + const args = { path: "a.ts" }; + + await pool.getBridge(TEST_PROJECT_ROOT).toolCall("s", "read", args); + await tick(); + + expect(client.requests.length).toBe(1); + const body = client.requests[0].body as Record; + const deadline = body.deadline_ms_remaining; + expect(typeof deadline).toBe("number"); + expect(deadline as number).toBeGreaterThan(25_000); + expect(deadline as number).toBeLessThanOrEqual(30_000); + // Arguments are passed through untouched and gain no scheduling fields. + expect(body.arguments).toEqual({ path: "a.ts" }); + expect(body).not.toHaveProperty("priority"); + expect(body).not.toHaveProperty("lane"); + expect(client.requests[0].options?.timeoutMs).toBeGreaterThan(25_000); + expect(client.requests[0].options?.timeoutMs).toBeLessThanOrEqual(30_000); + }); + + test("a caller timeout overrides the pool default with exact precedence", async () => { + const client = new FakeClient(async () => envelope({ id: "r", success: true, text: "" })); + const pool = poolWithDefault(client, 30_000); + + await pool + .getBridge(TEST_PROJECT_ROOT) + .toolCall("s", "read", { path: "a.ts" }, { timeoutMs: 5_000 }); + await tick(); + + expect(client.requests.length).toBe(1); + const deadline = (client.requests[0].body as Record) + .deadline_ms_remaining as number; + expect(deadline).toBeLessThanOrEqual(5_000); + expect(deadline).toBeGreaterThan(4_000); + }); + + test("an expired budget before send returns the provable not-sent error", async () => { + // Hold the route open so the caller's only way out is the budget race + // firing while bytes are provably still unsent. + const client = new FakeClient(async () => envelope({ id: "r", success: true, text: "" })); + client.routeOpenGate = Promise.withResolvers().promise; + const pool = poolWithDefault(client, 10); + + await expect( + pool.getBridge(TEST_PROJECT_ROOT).toolCall("s", "read", { path: "a.ts" }), + ).rejects.toMatchObject({ + kind: "not_sent", + code: "request_deadline_exceeded_before_send", + }); + }); + + test("a stale-route retry draws down the same budget across backoff and resend", async () => { + // The first request proves the route absent (unknown_channel), the pooled + // reopen backoff waits ~100ms, then the retry sends on a fresh channel. + // The resent body's stamped budget must be lower than the first stamp by + // roughly the backoff delay — the budget never restarts. + const client = new FakeClient(async () => envelope({ id: "r", success: true, text: "" })); + let requestAttempts = 0; + const originalRequest = client.request.bind(client); + client.request = async (route, body, options) => { + requestAttempts += 1; + if (requestAttempts === 1) { + client.requests.push({ route, channel: route.channel, body, options }); + throw new SubcError("unknown channel", "unknown_channel"); + } + return originalRequest(route, body, options); + }; + const pool = poolWithDefault(client, 30_000); + + const reply = await pool.getBridge(TEST_PROJECT_ROOT).toolCall("s", "read", {}); + expect(reply.success).toBe(true); + expect(requestAttempts).toBe(2); + expect(client.requests.length).toBe(2); + + const firstStamp = (client.requests[0].body as Record) + .deadline_ms_remaining as number; + const retryStamp = (client.requests[1].body as Record) + .deadline_ms_remaining as number; + expect(firstStamp).toBeGreaterThan(0); + expect(retryStamp).toBeGreaterThan(0); + // Both draw the same absolute deadline; the retry saw the backoff wait. + expect(firstStamp).toBeGreaterThan(retryStamp); + }); + + test("the route open race observes late settlement without invalidating the shared route", async () => { + const gate = Promise.withResolvers(); + const releaseOpen: () => void = () => gate.resolve(); + const client = new FakeClient(async () => envelope({ id: "r", success: true, text: "" })); + client.routeOpenGate = gate.promise; + const pool = poolWithDefault(client, 5_000); + + const fast = pool + .getBridge(TEST_PROJECT_ROOT) + .toolCall("s", "read", { path: "a.ts" }, { timeoutMs: 50 }); + await expect(fast).rejects.toMatchObject({ + kind: "not_sent", + code: "request_deadline_exceeded_before_send", + }); + // Settle the shared open late. When the SAME session retries, it reuses + // the cached (now-settled) route entry rather than opening a new channel — + // the first caller's expiry never invalidated the shared open. + releaseOpen(); + await tick(); + await pool.getBridge(TEST_PROJECT_ROOT).toolCall("s", "read", {}); + expect(client.routeOpens.length).toBe(1); + expect(client.requests.length).toBe(1); + }); +}); diff --git a/packages/aft-bridge/src/subc-transport.ts b/packages/aft-bridge/src/subc-transport.ts index 5027c3ee2..bdcdda24c 100644 --- a/packages/aft-bridge/src/subc-transport.ts +++ b/packages/aft-bridge/src/subc-transport.ts @@ -159,6 +159,14 @@ export interface SubcTransportPoolOptions { consumerIdentity?: ConsumerIdentity | null; /** Handshake timeout forwarded to SubcClient.connect. */ handshakeTimeoutMs?: number; + /** + * Pool default request budget in milliseconds. When neither the caller nor + * the tool adapter supplies a timeout, every route request derives one + * absolute deadline from this value at entry and never restarts it across + * connection, route-open, backoff, or stale-route retry. A direct pool that + * omits this (tests) carries no deadline metadata. + */ + defaultTimeoutMs?: number; /** * Connection factory seam. Defaults to the real `SubcClient.connect`. Tests * inject a fake to exercise route caching / Rd reconnect without a daemon. @@ -863,7 +871,9 @@ class SubcTransport implements AftProjectTransport { // orchestrated bash passes its wait-aware budget as transportTimeoutMs, and // dropping it here would cap long tool executions at the subc client's // default unary deadline while the command keeps running module-side. - const timeoutMs = options.transportTimeoutMs ?? options.timeoutMs; + // The pool default budget applies when the caller supplied neither. + const timeoutMs = + options.transportTimeoutMs ?? options.timeoutMs ?? this.pool.poolDefaultTimeoutMs; const onProgress = (options as { onProgress?: RequestOptions["onProgress"] }).onProgress; return { preview, timeoutMs, onProgress }; } @@ -878,6 +888,7 @@ export class SubcTransportPool implements AftTransportPool { readonly harness: string; private readonly connectionFile: string; private readonly handshakeTimeoutMs?: number; + private readonly defaultTimeoutMs?: number; private readonly consumerIdentity: ConsumerIdentity | null | undefined; private readonly connectFn: (opts: { connectionFile: string; @@ -937,6 +948,7 @@ export class SubcTransportPool implements AftTransportPool { this.connectionFile = options.connectionFile; this.harness = options.harness; this.handshakeTimeoutMs = options.handshakeTimeoutMs; + this.defaultTimeoutMs = options.defaultTimeoutMs; this.consumerIdentity = options.consumerIdentity; this.connectFn = options.connect ?? ((opts) => SubcClient.connect(opts)); this.onBgEventsNudge = options.onBgEventsNudge; @@ -991,6 +1003,11 @@ export class SubcTransportPool implements AftTransportPool { this.lifecycleRegistration = registration; } + /** Pool default request budget; `undefined` means no deadline metadata. */ + get poolDefaultTimeoutMs(): number | undefined { + return this.defaultTimeoutMs; + } + /** Construction helper for wrappers that own the registration sequence. */ registerLifecyclePool( registry: LifecycleRegistry, @@ -1482,7 +1499,57 @@ export class SubcTransportPool implements AftTransportPool { return this.rootReapedError(record); } - /** Open or reuse a route while guarding every lifecycle boundary. */ + /** Race a shared wait against THIS caller's remaining request budget. + * + * The underlying promise (shared connect, shared route opening, pooled + * backoff timer) is NEVER cancelled or invalidated by one caller's expiry: + * late settlement is observed via a detached handler so it can still cache + * the client/route, and an unhandled rejection cannot occur. + */ + private awaitWithinRequestBudget( + wait: Promise, + remaining: number | undefined, + phase: string, + ): Promise { + if (remaining === undefined || !Number.isFinite(remaining)) return wait; + let timer: ReturnType | undefined; + const timeoutPromise = new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new SubcCallError( + "not_sent", + `request deadline elapsed while waiting for ${phase}`, + "request_deadline_exceeded_before_send", + ), + ), + Math.max(0, Math.ceil(remaining)), + ); + }); + return Promise.race([wait, timeoutPromise]).finally(() => { + clearTimeout(timer); + // Observe late settlement of the shared wait: the loser of the race must + // neither cache nothing nor surface an unhandled rejection. The shared + // client/route itself is never invalidated by this caller's expiry. + wait.then( + () => undefined, + () => undefined, + ); + }); + } + + /** + * Open or reuse a route while guarding every lifecycle boundary. + * + * One absolute local request deadline is derived at entry from the exact + * precedence `transportTimeoutMs ?? timeoutMs ?? pool.defaultTimeoutMs` and + * never restarts: connection, route open, reload-window backoff, stale-route + * retry backoff, and the request itself all draw down the same budget. + * Immediately before each `client.request` attempt the remaining budget is + * recomputed and stamped as top-level `deadline_ms_remaining` on the request + * body (arguments are never mutated), and the same remaining value becomes + * the request's `RequestOptions.timeoutMs`. + */ async routeRequest( identity: BindIdentity, body: Record, @@ -1490,6 +1557,13 @@ export class SubcTransportPool implements AftTransportPool { onProgress?: RequestOptions["onProgress"], expectedGeneration?: RootGeneration, ): Promise { + const effectiveTimeoutMs = timeoutMs ?? this.defaultTimeoutMs; + const deadlineMs = Number.isFinite(effectiveTimeoutMs) + ? Date.now() + (effectiveTimeoutMs as number) + : undefined; + const remainingMs = (): number | undefined => + deadlineMs === undefined ? undefined : deadlineMs - Date.now(); + const root = asCanonicalRootPath(identity.project_root); let generation = expectedGeneration; if (this.lifecycleEnabled()) { @@ -1505,7 +1579,11 @@ export class SubcTransportPool implements AftTransportPool { try { let client: SubcClientLike; try { - client = await this.ensureClient(); + client = (await this.awaitWithinRequestBudget( + this.ensureClient(), + remainingMs(), + "connection", + )) as SubcClientLike; this.assertRecordLive(record); } catch (error) { throw this.annotateReapError(error, record); @@ -1514,12 +1592,21 @@ export class SubcTransportPool implements AftTransportPool { const openRoute = async (): Promise<{ route: RouteHandle; entry: RouteEntry }> => { try { this.assertRecordLive(record); - const opened = await this.routeHandle(client, identity, record); + const opened = (await this.awaitWithinRequestBudget( + this.routeHandle(client, identity, record), + remainingMs(), + "route-open", + )) as { route: RouteHandle; entry: RouteEntry }; this.assertRecordLive(record); return opened; } catch (error) { if (this.isReapInduced(record)) throw this.annotateReapError(error, record); if (error instanceof RouteTornDownError) throw error; + if (error instanceof SubcCallError && error.kind === "not_sent") { + // Caller-scoped budget expiry: the shared connect/open is healthy + // and must survive for other callers, so it is never dropped here. + throw error; + } if ( isConsumerReconnectTransient(error) && this.isCurrentSession(key, record) && @@ -1546,7 +1633,7 @@ export class SubcTransportPool implements AftTransportPool { throw reloadWindowExhaustedError(error); } reloadWaitedMs += delayMs; - await wait; + await this.awaitWithinRequestBudget(wait, remainingMs(), "reload-window"); } } }; @@ -1582,7 +1669,27 @@ export class SubcTransportPool implements AftTransportPool { const requestOnRoute = async (route: RouteHandle): Promise => { this.assertRecordLive(record); - const reply = await client.request(route, body, { timeoutMs, onProgress }); + // Immediately before bytes go on the wire: recompute the remaining + // budget from the unchanged absolute deadline. If none remains, the + // request is PROVABLY not sent. + const remaining = remainingMs(); + if (remaining !== undefined && remaining <= 0) { + throw new SubcCallError( + "not_sent", + "request deadline elapsed before the request could be sent", + "request_deadline_exceeded_before_send", + ); + } + const requestTimeoutMs = + remaining !== undefined ? Math.max(1, Math.floor(remaining)) : timeoutMs; + const deadlineBody = + remaining === undefined || !Number.isFinite(remaining) + ? body + : { ...body, deadline_ms_remaining: Math.max(0, Math.floor(remaining)) }; + const reply = await client.request(route, deadlineBody, { + timeoutMs: requestTimeoutMs, + onProgress, + }); // A legacy closeSession may intentionally let an already-delivered reply // settle. It must not mutate shared state or recreate a subscription. if (!this.isCurrentSession(key, record)) { @@ -1600,13 +1707,22 @@ export class SubcTransportPool implements AftTransportPool { return await requestOnRoute(routeAndEntry.route); } catch (error) { if (this.isReapInduced(record)) throw this.annotateReapError(error, record); + if (error instanceof SubcCallError && error.kind === "not_sent") { + // Caller-scoped budget expiry: the request provably never went out, + // so the shared client/route state stays untouched for other callers. + throw error; + } if ( isRouteProvenAbsentError(error) && this.isCurrentSession(key, record) && this.client === client ) { clearRouteEntry(routeAndEntry.entry); - await this.waitForRouteReopenBackoff().wait; + await this.awaitWithinRequestBudget( + this.waitForRouteReopenBackoff().wait, + remainingMs(), + "stale-route-backoff", + ); routeAndEntry = await openRouteAfterReloadWindow(); try { const reply = await requestOnRoute(routeAndEntry.route); diff --git a/packages/aft-bridge/src/transport-factory.ts b/packages/aft-bridge/src/transport-factory.ts index 2c945b45d..44063dcb9 100644 --- a/packages/aft-bridge/src/transport-factory.ts +++ b/packages/aft-bridge/src/transport-factory.ts @@ -198,6 +198,7 @@ async function createConcreteAftTransportPool( onBgEventsNudge: opts.onBgEventsNudge, onBgEventsNudgeRef: opts.onBgEventsNudgeRef, lifecycleDemandCheck: opts.subcLifecycleDemandCheck ?? ((root) => existsSync(root)), + defaultTimeoutMs: opts.poolOptions.timeoutMs ?? 30_000, }); } return new BridgePool(opts.binaryPath, opts.poolOptions, opts.configOverrides); From a2e1d8ae9b8119d9a9825f74f84800ee22c0f9d3 Mon Sep 17 00:00:00 2001 From: Naadir Jeewa Date: Sun, 30 Aug 2026 23:24:15 +0100 Subject: [PATCH 02/14] test(subc): inherit executor queue defaults Signed-off-by: Naadir Jeewa Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/subc/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/aft/src/subc/mod.rs b/crates/aft/src/subc/mod.rs index 544b326ea..58023f317 100644 --- a/crates/aft/src/subc/mod.rs +++ b/crates/aft/src/subc/mod.rs @@ -8343,6 +8343,7 @@ mod tests { actor_cap: 1, heavy_permits: 1, drr_quantum: 1, + ..crate::executor::ExecutorConfig::default() }, "pool=2 actor_cap=1", ) @@ -8354,6 +8355,7 @@ mod tests { actor_cap: 3, heavy_permits: 3, drr_quantum: 1, + ..crate::executor::ExecutorConfig::default() }, "pool=4 actor_cap=3", ) From 1c03a139dc069015c437d92b2b1759fa022622af Mon Sep 17 00:00:00 2001 From: Naadir Jeewa Date: Mon, 31 Aug 2026 11:02:38 +0100 Subject: [PATCH 03/14] feat(index): schedule resumable root slices Use pressure-aware deficit round robin to rotate standing roots across bounded search, semantic, and callgraph work. Preserve laptop responsiveness by default while allowing an explicit performance policy. Signed-off-by: Naadir Jeewa Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- ARCHITECTURE.md | 5 +- crates/aft/src/callgraph_store/mod.rs | 108 ++++- crates/aft/src/checkpoint.rs | 68 ++- crates/aft/src/commands/configure.rs | 2 +- crates/aft/src/commands/semantic_search.rs | 1 - crates/aft/src/config.rs | 28 ++ crates/aft/src/config_resolve.rs | 98 ++++- crates/aft/src/lib.rs | 2 + crates/aft/src/resource_policy.rs | 374 +++++++++++++++++ crates/aft/src/search_index.rs | 322 +++++++++++++++ crates/aft/src/semantic_index.rs | 388 +++++++++++++++++- crates/aft/src/standing_roots.rs | 5 +- crates/aft/src/standing_scheduler.rs | 180 ++++++++ crates/aft/src/subc/health.rs | 6 + crates/aft/src/subc/standing.rs | 365 ++++++++++++---- .../tests/integration/inspect_engine_test.rs | 5 +- .../aft/tests/integration/lsp_rename_test.rs | 22 +- .../tests/standing_roots_acceptance_test.rs | 5 +- docs/config.md | 9 + .../src/__tests__/config.test.ts | 28 ++ packages/opencode-plugin/src/config.ts | 1 + .../pi-plugin/src/__tests__/config.test.ts | 28 ++ packages/pi-plugin/src/config.ts | 2 + specs/standing-index-resource-policy/plan.md | 220 ++++++++++ 24 files changed, 2149 insertions(+), 123 deletions(-) create mode 100644 crates/aft/src/resource_policy.rs create mode 100644 crates/aft/src/standing_scheduler.rs create mode 100644 specs/standing-index-resource-policy/plan.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2f3ee4e3b..9fccc2801 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -107,8 +107,9 @@ 1. Index project files using a disk-backed, pread-based trigram search index that keeps memory overhead bounded -- `crates/aft/src/search_index.rs`. To prevent redundant disk hashing and index re-verification loops during configure bind/warmup sequences, a verification memo with a 10-minute TTL manages cache freshness checks, utilizing metadata stat checks (`VerifyStrategy::StatFirst`) when possible rather than strict content hashing. For grafted history roots, canonicalize the sorted, deduplicated set of root commits before hashing artifact keys to prevent Git traversal-order changes from triggering redundant index rebuilds. 2. Optionally index with dense embeddings (fastembed, OpenAI-compatible, Ollama, or Synapse over SubC) -- `crates/aft/src/semantic_index.rs`, `crates/aft/src/synapse_embed.rs`. Serialize cold semantic warmups by gating callgraph store building and Tier 2 diagnostics refreshes behind active cold semantic index seeds. Coalesce watcher-driven semantic re-embeds under a 15-second quiet window (`SEMANTIC_REFRESH_QUIET_WINDOW_MS`) to bundle edit bursts into a single collection pass, while masking changed files from search results until indexed to preserve query correctness. Reconfiguring semantic settings or project roots cancels superseded semantic builders while adopting matching live builders. In tests, override this quiet window via the `AFT_SEMANTIC_QUIET_WINDOW_MS` environment variable. Limit process-wide semantic refresh concurrency using the `ColdBuildLimiter` (sharing the slot budget with other heavy maintenance operations) to prevent concurrent background refreshes from overloading remote or local embedding backends -- `crates/aft/src/cold_build_limiter.rs`, `crates/aft/src/commands/configure.rs`. -3. Classify query shape (prose vs code) using the query shape parser -- `crates/aft/src/query_shape.rs`. Identify "type-concept identifier queries" (TitleCase PascalCase types combined with lowercase concepts) to trigger definition semantic priors. -4. Serve `grep` (trigram, full-text) and `aft_search` (semantic + hybrid) queries, delegating to `GrepExecutor` for accelerated path evaluation and enforcing execution safety limits (like `MAX_FALLBACK_WALK_FILES` and `FALLBACK_WALK_BUDGET`) during fallback walks when indexes are building or unavailable -- `crates/aft/src/grep_executor.rs`, `crates/aft/src/commands/grep.rs`, `crates/aft/src/commands/semantic_search.rs`. Under standalone bridge mode, interactive semantic searches support cancellable deferred polling in the main event loop. Borrow-only lexical and semantic snapshot opens bypass the cold-build limiter to prevent fresh-worktree search starvation while first searches wait cancellation-aware for a bounded loading window (2.5s). Interactive query embeddings and search artifact waits are bounded by dedicated budgets (`QueryBudget` and bounded interactive search artifact wait timeouts; `query_timeout_ms` clamped to 500..15000ms, defaulting to 3000ms) to keep interactive requests fast without affecting background build/refresh timeouts, falling back to lexical search if query embedding fails or times out. Downrank generated documentation artifacts (e.g. minified CSS/JS, maps, SVGs) in lexical and hybrid search results. For external search requests, resolve and cache external git roots, querying cached read-only search and semantic indexes from the `borrowed_index_cache` (capped at 4 concurrent entries) to avoid redundant git probes and disk parsing. +3. Schedule standing-root search, semantic, and callgraph construction through the process-wide pressure-aware deficit round-robin scheduler -- `crates/aft/src/standing_scheduler.rs`, `crates/aft/src/resource_policy.rs`, `crates/aft/src/subc/standing.rs`. The scheduler admits at most the configured cold-build concurrency, rotates unfinished roots after each durable slice, and charges measured elapsed work against each root's deficit. Search, semantic, and callgraph builders persist versioned staging state and publish atomically only after the complete corpus is ready. The default `index.resource_policy = "balanced"` pauses new slices under battery saving or CPU, memory, and I/O pressure and resumes with hysteresis. `"performance"` bypasses resource admission for users who accept the power cost, but retains bounded concurrency, fair rotation, resumable checkpoints, and OS background thread priority. +4. Classify query shape (prose vs code) using the query shape parser -- `crates/aft/src/query_shape.rs`. Identify "type-concept identifier queries" (TitleCase PascalCase types combined with lowercase concepts) to trigger definition semantic priors. +5. Serve `grep` (trigram, full-text) and `aft_search` (semantic + hybrid) queries, delegating to `GrepExecutor` for accelerated path evaluation and enforcing execution safety limits (like `MAX_FALLBACK_WALK_FILES` and `FALLBACK_WALK_BUDGET`) during fallback walks when indexes are building or unavailable -- `crates/aft/src/grep_executor.rs`, `crates/aft/src/commands/grep.rs`, `crates/aft/src/commands/semantic_search.rs`. Under standalone bridge mode, interactive semantic searches support cancellable deferred polling in the main event loop. Borrow-only lexical and semantic snapshot opens bypass the cold-build limiter to prevent fresh-worktree search starvation while first searches wait cancellation-aware for a bounded loading window (2.5s). Interactive query embeddings and search artifact waits are bounded by dedicated budgets (`QueryBudget` and bounded interactive search artifact wait timeouts; `query_timeout_ms` clamped to 500..15000ms, defaulting to 3000ms) to keep interactive requests fast without affecting background build/refresh timeouts, falling back to lexical search if query embedding fails or times out. Downrank generated documentation artifacts (e.g. minified CSS/JS, maps, SVGs) in lexical and hybrid search results. For external search requests, resolve and cache external git roots, querying cached read-only search and semantic indexes from the `borrowed_index_cache` (capped at 4 concurrent entries) to avoid redundant git probes and disk parsing. **File read flow:** diff --git a/crates/aft/src/callgraph_store/mod.rs b/crates/aft/src/callgraph_store/mod.rs index 68a50bd42..5c2ad9bca 100644 --- a/crates/aft/src/callgraph_store/mod.rs +++ b/crates/aft/src/callgraph_store/mod.rs @@ -623,6 +623,7 @@ thread_local! { const { std::cell::RefCell::new(None) }; static REFRESH_COMMIT_ADMISSION: std::cell::RefCell, u64)>> = const { std::cell::RefCell::new(None) }; + static COLD_BUILD_SLICE_BUDGET: std::cell::Cell> = const { std::cell::Cell::new(None) }; } mod dead_code_projection; @@ -711,6 +712,21 @@ impl Drop for PublishAdmissionGuard { } } +struct ColdBuildSliceBudgetGuard { + previous: Option, +} + +impl Drop for ColdBuildSliceBudgetGuard { + fn drop(&mut self) { + COLD_BUILD_SLICE_BUDGET.with(|slot| slot.set(self.previous)); + } +} + +fn with_cold_build_slice_budget(budget: usize, run: impl FnOnce() -> R) -> R { + let previous = COLD_BUILD_SLICE_BUDGET.with(|slot| slot.replace(Some(budget.max(1)))); + let _guard = ColdBuildSliceBudgetGuard { previous }; + run() +} pub(crate) fn with_publish_epoch( epoch: crate::root_cache::ArtifactPublishEpoch, expected: u64, @@ -724,16 +740,31 @@ pub(crate) fn with_publish_epoch( fn ensure_cold_build_current(stage: &'static str, completed: usize, total: usize) -> Result<()> { notify_cold_build_slice_observer(stage, completed, total); let admission = PUBLISH_ADMISSION.with(|slot| slot.borrow().clone()); - if admission.is_none_or(|(epoch, expected)| epoch.is_current(expected)) { - return Ok(()); + if admission.is_some_and(|(epoch, expected)| !epoch.is_current(expected)) { + crate::slog_info!( + "callgraph cold build superseded, stopping after {}/{} ({})", + completed, + total, + stage + ); + return Err(CallGraphStoreError::Superseded); } - crate::slog_info!( - "callgraph cold build superseded, stopping after {}/{} ({})", - completed, - total, - stage - ); - Err(CallGraphStoreError::Superseded) + let exhausted = COLD_BUILD_SLICE_BUDGET.with(|slot| match slot.get() { + Some(remaining) if completed > 0 && remaining <= 1 => true, + Some(remaining) if completed > 0 => { + slot.set(Some(remaining - 1)); + false + } + _ => false, + }); + if exhausted { + return Err(CallGraphStoreError::SliceProgress { + phase: stage.to_string(), + completed, + total, + }); + } + Ok(()) } fn publish_if_current(publish: impl FnOnce() -> Result) -> Result { @@ -815,6 +846,11 @@ pub enum CallGraphStoreError { Suspended(crate::build_breaker::BuildSuspension), Superseded, StaleFiles(Vec), + SliceProgress { + phase: String, + completed: usize, + total: usize, + }, } impl CallGraphStoreError { @@ -860,6 +896,14 @@ impl fmt::Display for CallGraphStoreError { Self::Superseded => { write!(formatter, "callgraph store build superseded before publish") } + Self::SliceProgress { + phase, + completed, + total, + } => write!( + formatter, + "callgraph cold-build slice completed: {phase} {completed}/{total}" + ), Self::StaleFiles(files) => { write!( formatter, @@ -1909,6 +1953,20 @@ pub struct IncrementalStats { pub unchanged_extract_files: usize, } +#[derive(Debug)] +pub enum ColdBuildSlice { + Progress { + phase: String, + completed: usize, + total: usize, + }, + Complete { + store: CallGraphStore, + stats: ColdBuildStats, + }, + Superseded, +} + /// Phase timings for the copy-based incremental refresh benchmark. #[doc(hidden)] #[derive(Debug, Clone, Default, PartialEq, Eq)] @@ -2884,6 +2942,37 @@ impl CallGraphStore { ) } + pub fn resume_cold_build_slice_with_lease( + callgraph_dir: PathBuf, + project_root: PathBuf, + files: &[PathBuf], + chunk_size: usize, + ) -> Result { + let result = with_cold_build_slice_budget(1, || { + Self::cold_build_with_lease_chunked_inner( + callgraph_dir, + project_root, + files, + chunk_size, + false, + ) + }); + match result { + Ok((store, stats)) => Ok(ColdBuildSlice::Complete { store, stats }), + Err(CallGraphStoreError::SliceProgress { + phase, + completed, + total, + }) => Ok(ColdBuildSlice::Progress { + phase, + completed, + total, + }), + Err(CallGraphStoreError::Superseded) => Ok(ColdBuildSlice::Superseded), + Err(error) => Err(error), + } + } + pub(crate) fn force_cold_build_with_lease_chunked( callgraph_dir: PathBuf, project_root: PathBuf, @@ -3587,7 +3676,6 @@ impl CallGraphStore { &module_resolution_memo, ) } - #[cfg(test)] fn cold_build_chunked_with_resolution_memo_for_test( &self, diff --git a/crates/aft/src/checkpoint.rs b/crates/aft/src/checkpoint.rs index 1eb8cc799..2638af757 100644 --- a/crates/aft/src/checkpoint.rs +++ b/crates/aft/src/checkpoint.rs @@ -240,25 +240,20 @@ pub struct CheckpointStore { blob_counter: AtomicU64, } -/// Owns a checkpoint mutation lock and removes its project scope directory after -/// the filesystem lock has released. The directory scopes only the transient -/// lockfile; durable checkpoint bytes live under the harness namespace instead. +/// Owns a checkpoint mutation lock. +/// +/// The lock scope directory is durable. Removing it after each owner releases +/// the lock races another process between its `create_dir_all` and exclusive +/// lock-file creation. struct CheckpointLockGuard { guard: Option, - scope_dir: Option, } impl Drop for CheckpointLockGuard { fn drop(&mut self) { - // LockGuard::drop must join the heartbeat before removing the lockfile. - // Drop it first, then make the best-effort directory cleanup so a new - // owner can keep the scope directory when it races this release. if let Some(guard) = self.guard.take() { drop(guard); } - if let Some(scope_dir) = &self.scope_dir { - remove_empty_scope_dir(scope_dir); - } } } @@ -361,10 +356,7 @@ impl CheckpointStore { }, })?; - Ok(CheckpointLockGuard { - guard: Some(guard), - scope_dir, - }) + Ok(CheckpointLockGuard { guard: Some(guard) }) } /// Create a checkpoint by reading the given files, scoped to `session`. @@ -1933,7 +1925,7 @@ mod tests { } #[test] - fn checkpoint_lock_scope_is_removed_after_release() { + fn checkpoint_lock_scope_remains_after_release() { let dir = tempfile::tempdir().unwrap(); let scope_dir = dir.path().join("checkpoints").join("project-scope"); let lock_path = scope_dir.join("checkpoint.lock"); @@ -1946,7 +1938,10 @@ mod tests { .create(DEFAULT_SESSION_ID, "released", vec![path], &backup_store) .unwrap(); - assert!(!scope_dir.exists(), "released lock scope should be removed"); + assert!( + scope_dir.is_dir(), + "released lock scope must remain durable" + ); } #[test] @@ -2266,6 +2261,47 @@ mod tests { assert_eq!(fs::read_to_string(&path).unwrap(), "original"); } + #[test] + fn concurrent_checkpoint_stores_keep_shared_lock_scope_stable() { + let dir = tempfile::tempdir().unwrap(); + let lock_path = dir.path().join("locks").join("checkpoint.lock"); + let file = dir.path().join("shared.txt"); + fs::write(&file, "content").unwrap(); + let start = Arc::new(std::sync::Barrier::new(3)); + + let workers = (0..2) + .map(|worker| { + let lock_path = lock_path.clone(); + let file = file.clone(); + let start = Arc::clone(&start); + std::thread::spawn(move || { + let mut store = + CheckpointStore::with_lock_path(lock_path, Duration::from_secs(2)); + let backup = BackupStore::new(); + start.wait(); + for iteration in 0..100 { + store + .create( + DEFAULT_SESSION_ID, + &format!("worker-{worker}-{iteration}"), + vec![file.clone()], + &backup, + ) + .expect("shared checkpoint lock scope must remain available"); + } + }) + }) + .collect::>(); + start.wait(); + for worker in workers { + worker.join().expect("checkpoint worker"); + } + assert!( + lock_path.parent().unwrap().is_dir(), + "shared lock scope must remain stable between owners" + ); + } + #[cfg(unix)] #[test] fn checkpoint_restore_preserves_regular_file_permissions() { diff --git a/crates/aft/src/commands/configure.rs b/crates/aft/src/commands/configure.rs index 7c52db446..f619be9b3 100644 --- a/crates/aft/src/commands/configure.rs +++ b/crates/aft/src/commands/configure.rs @@ -1683,7 +1683,7 @@ fn delay_symbol_prewarm_for_debug() { thread::sleep(Duration::from_millis(delay_ms)); } -fn walk_semantic_project_files_bounded( +pub(crate) fn walk_semantic_project_files_bounded( root: &Path, max_files: usize, ) -> Result, usize> { diff --git a/crates/aft/src/commands/semantic_search.rs b/crates/aft/src/commands/semantic_search.rs index b43034f6b..0ece028a8 100644 --- a/crates/aft/src/commands/semantic_search.rs +++ b/crates/aft/src/commands/semantic_search.rs @@ -4628,7 +4628,6 @@ mod tests { )) }); - assert_eq!(response["interpreted_as"], "lexical"); assert!(response["results"] .as_array() diff --git a/crates/aft/src/config.rs b/crates/aft/src/config.rs index 284efdaa9..cb932e7fa 100644 --- a/crates/aft/src/config.rs +++ b/crates/aft/src/config.rs @@ -53,6 +53,33 @@ impl IndexKind { } } } +/// Host resource policy for standing index maintenance. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IndexResourcePolicy { + /// Pause new background slices when authoritative host signals report pressure. + #[default] + Balanced, + /// Ignore host pressure admission while preserving concurrency and correctness bounds. + Performance, +} + +impl IndexResourcePolicy { + pub const fn as_str(self) -> &'static str { + match self { + Self::Balanced => "balanced", + Self::Performance => "performance", + } + } + + pub fn from_name(name: &str) -> Option { + match name { + "balanced" => Some(Self::Balanced), + "performance" => Some(Self::Performance), + _ => None, + } + } +} /// One user-configured root whose literal path spelling is its durable identity. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -67,6 +94,7 @@ pub struct IndexRootConfig { #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct IndexConfig { + pub resource_policy: IndexResourcePolicy, pub roots: Vec, } diff --git a/crates/aft/src/config_resolve.rs b/crates/aft/src/config_resolve.rs index 2b7c80c3e..e5e6f34e4 100644 --- a/crates/aft/src/config_resolve.rs +++ b/crates/aft/src/config_resolve.rs @@ -14,10 +14,11 @@ use serde_json::{Map, Value}; use crate::config::{ expand_index_root_path, normalize_git_co_author, BackupConfig, Config, GhShimConfig, GitConfig, - IndexConfig, IndexKind, IndexRootConfig, InspectConfig, SandboxConfig, SemanticBackend, - SemanticBackendConfig, UserServerDef, WorktreeConfig, DEFAULT_INSPECT_DIAGNOSTICS_TIMEOUT_MS, - MAX_INSPECT_DIAGNOSTICS_TIMEOUT_MS, MAX_SEMANTIC_QUERY_TIMEOUT_MS, - MIN_INSPECT_DIAGNOSTICS_TIMEOUT_MS, MIN_SEMANTIC_QUERY_TIMEOUT_MS, + IndexConfig, IndexKind, IndexResourcePolicy, IndexRootConfig, InspectConfig, SandboxConfig, + SemanticBackend, SemanticBackendConfig, UserServerDef, WorktreeConfig, + DEFAULT_INSPECT_DIAGNOSTICS_TIMEOUT_MS, MAX_INSPECT_DIAGNOSTICS_TIMEOUT_MS, + MAX_SEMANTIC_QUERY_TIMEOUT_MS, MIN_INSPECT_DIAGNOSTICS_TIMEOUT_MS, + MIN_SEMANTIC_QUERY_TIMEOUT_MS, }; use crate::harness::Harness; use crate::jsonc::strip_jsonc; @@ -519,6 +520,7 @@ pub struct RawSandbox { #[serde(default)] pub struct RawIndex { pub roots: Option>, + pub resource_policy: Option, } #[derive(Debug, Clone, Default, Deserialize, PartialEq)] @@ -1237,6 +1239,14 @@ fn record_project_drops(raw: &RawAftConfig, tier: &str, dropped: &mut Vec, warnings: &mut Vec IndexResourcePolicy::Balanced, + Some(name) => IndexResourcePolicy::from_name(name).unwrap_or_else(|| { + warnings.push(ConfigWarning { + code: "invalid_index_resource_policy", + key: "index.resource_policy", + tier: "user".to_string(), + value: name.to_string(), + message: format!( + "Invalid index.resource_policy {name:?}; valid values: balanced, performance" + ), + }); + IndexResourcePolicy::Balanced + }), + }; + let Some(roots) = raw.roots.as_ref() else { - return IndexConfig::default(); + return IndexConfig { + resource_policy, + ..IndexConfig::default() + }; }; let home = std::env::var_os("HOME") @@ -1448,12 +1478,16 @@ fn resolve_index_config(raw: Option<&RawIndex>, warnings: &mut Vec AdmissionDecision { + if policy == IndexResourcePolicy::Performance { + self.paused = false; + self.healthy_samples = 0; + return AdmissionDecision::Admit; + } + + if let Some(reason) = pause_reason(snapshot) { + self.paused = true; + self.healthy_samples = 0; + return AdmissionDecision::Paused(reason); + } + + if !self.paused { + return AdmissionDecision::Admit; + } + + self.healthy_samples = self.healthy_samples.saturating_add(1); + if self.healthy_samples >= HEALTHY_SAMPLES_TO_RESUME { + self.paused = false; + self.healthy_samples = 0; + AdmissionDecision::Admit + } else { + AdmissionDecision::Paused(PauseReason::Recovering) + } + } +} + +fn pause_reason(snapshot: ResourceSnapshot) -> Option { + match snapshot.power { + PowerState::BatterySaving => return Some(PauseReason::BatterySaving), + PowerState::Unknown => return Some(PauseReason::UnknownPower), + PowerState::External | PowerState::Battery | PowerState::NoBattery => {} + } + match snapshot.memory_pressure { + SignalState::High => return Some(PauseReason::MemoryPressure), + SignalState::Unknown => return Some(PauseReason::UnknownMemoryPressure), + SignalState::Healthy => {} + } + match snapshot.io_pressure { + SignalState::High => return Some(PauseReason::IoPressure), + SignalState::Unknown => return Some(PauseReason::UnknownIoPressure), + SignalState::Healthy => {} + } + match snapshot.cpu_pressure { + SignalState::High => Some(PauseReason::CpuPressure), + SignalState::Unknown => Some(PauseReason::UnknownCpuPressure), + SignalState::Healthy => None, + } +} + +#[cfg(target_os = "linux")] +fn parse_linux_psi(input: &str, kind: PressureKind) -> SignalState { + let prefix = match kind { + PressureKind::Cpu => "some ", + PressureKind::Stall => "full ", + }; + let Some(line) = input.lines().find(|line| line.starts_with(prefix)) else { + return SignalState::Unknown; + }; + let Some(avg10) = line + .split_ascii_whitespace() + .find_map(|field| field.strip_prefix("avg10=")) + .and_then(|value| value.parse::().ok()) + else { + return SignalState::Unknown; + }; + if avg10 > 0.0 { + SignalState::High + } else { + SignalState::Healthy + } +} +#[cfg(target_os = "linux")] +fn sample_linux_power_at(root: &std::path::Path) -> PowerState { + let Ok(entries) = std::fs::read_dir(root) else { + return PowerState::Unknown; + }; + let mut battery_capacity = None; + let mut found_battery = false; + for entry in entries.flatten() { + let path = entry.path(); + let kind = std::fs::read_to_string(path.join("type")) + .ok() + .map(|value| value.trim().to_owned()); + match kind.as_deref() { + Some("Mains" | "USB" | "USB_C" | "USB_PD") => { + if std::fs::read_to_string(path.join("online")) + .ok() + .is_some_and(|value| value.trim() == "1") + { + return PowerState::External; + } + } + Some("Battery") => { + found_battery = true; + if let Ok(value) = std::fs::read_to_string(path.join("capacity")) { + battery_capacity = value.trim().parse::().ok().or(battery_capacity); + } + } + _ => {} + } + } + if !found_battery { + PowerState::NoBattery + } else if battery_capacity.is_some_and(|capacity| capacity <= 10) { + PowerState::BatterySaving + } else { + PowerState::Battery + } +} + +#[cfg(target_os = "linux")] +#[derive(Debug, Clone, Copy)] +enum PressureKind { + Cpu, + Stall, +} + +pub fn sample_resources() -> ResourceSnapshot { + platform::sample() +} + +#[cfg(target_os = "linux")] +mod platform { + use super::*; + + pub(super) fn sample() -> ResourceSnapshot { + let pressure = |path: &str, kind| { + std::fs::read_to_string(path) + .ok() + .map_or(SignalState::Unknown, |value| parse_linux_psi(&value, kind)) + }; + ResourceSnapshot { + power: sample_linux_power_at(std::path::Path::new("/sys/class/power_supply")), + cpu_pressure: pressure("/proc/pressure/cpu", PressureKind::Cpu), + memory_pressure: pressure("/proc/pressure/memory", PressureKind::Stall), + io_pressure: pressure("/proc/pressure/io", PressureKind::Stall), + } + } +} + +#[cfg(not(target_os = "linux"))] +mod platform { + use super::*; + + pub(super) fn sample() -> ResourceSnapshot { + ResourceSnapshot { + power: PowerState::Unknown, + cpu_pressure: SignalState::Unknown, + memory_pressure: SignalState::Unknown, + io_pressure: SignalState::Unknown, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn healthy() -> ResourceSnapshot { + ResourceSnapshot { + power: PowerState::External, + cpu_pressure: SignalState::Healthy, + memory_pressure: SignalState::Healthy, + io_pressure: SignalState::Healthy, + } + } + + #[test] + fn balanced_pauses_on_battery_saving_and_pressure() { + let mut gate = ResourceAdmissionGate::default(); + let mut battery = healthy(); + battery.power = PowerState::BatterySaving; + assert_eq!( + gate.observe(IndexResourcePolicy::Balanced, battery), + AdmissionDecision::Paused(PauseReason::BatterySaving) + ); + + let mut pressured = healthy(); + pressured.memory_pressure = SignalState::High; + assert_eq!( + gate.observe(IndexResourcePolicy::Balanced, pressured), + AdmissionDecision::Paused(PauseReason::MemoryPressure) + ); + } + + #[test] + fn balanced_reports_unknown_portable_pressure_conservatively() { + let mut gate = ResourceAdmissionGate::default(); + let mut unknown = healthy(); + unknown.io_pressure = SignalState::Unknown; + assert_eq!( + gate.observe(IndexResourcePolicy::Balanced, unknown), + AdmissionDecision::Paused(PauseReason::UnknownIoPressure) + ); + } + + #[test] + fn desktop_without_battery_is_not_treated_as_battery_powered() { + let mut gate = ResourceAdmissionGate::default(); + let mut desktop = healthy(); + desktop.power = PowerState::NoBattery; + for _ in 0..HEALTHY_SAMPLES_TO_RESUME { + gate.observe(IndexResourcePolicy::Balanced, desktop); + } + assert_eq!( + gate.observe(IndexResourcePolicy::Balanced, desktop), + AdmissionDecision::Admit + ); + } + + #[test] + fn balanced_requires_consecutive_healthy_samples_after_pause() { + let mut gate = ResourceAdmissionGate::default(); + let mut pressured = healthy(); + pressured.io_pressure = SignalState::High; + assert!(matches!( + gate.observe(IndexResourcePolicy::Balanced, pressured), + AdmissionDecision::Paused(PauseReason::IoPressure) + )); + + for _ in 1..HEALTHY_SAMPLES_TO_RESUME { + assert_eq!( + gate.observe(IndexResourcePolicy::Balanced, healthy()), + AdmissionDecision::Paused(PauseReason::Recovering) + ); + } + assert_eq!( + gate.observe(IndexResourcePolicy::Balanced, healthy()), + AdmissionDecision::Admit + ); + } + + #[test] + fn unhealthy_sample_resets_resume_hysteresis() { + let mut gate = ResourceAdmissionGate::default(); + let mut pressured = healthy(); + pressured.cpu_pressure = SignalState::High; + gate.observe(IndexResourcePolicy::Balanced, pressured); + gate.observe(IndexResourcePolicy::Balanced, healthy()); + gate.observe(IndexResourcePolicy::Balanced, pressured); + + for _ in 1..HEALTHY_SAMPLES_TO_RESUME { + assert_eq!( + gate.observe(IndexResourcePolicy::Balanced, healthy()), + AdmissionDecision::Paused(PauseReason::Recovering) + ); + } + } + + #[test] + fn performance_bypasses_resource_admission_only() { + let mut gate = ResourceAdmissionGate::default(); + let snapshot = ResourceSnapshot { + power: PowerState::BatterySaving, + cpu_pressure: SignalState::High, + memory_pressure: SignalState::High, + io_pressure: SignalState::High, + }; + assert_eq!( + gate.observe(IndexResourcePolicy::Performance, snapshot), + AdmissionDecision::Admit + ); + } + #[cfg(target_os = "linux")] + #[test] + fn linux_psi_uses_cpu_some_and_full_stall_pressure() { + let healthy = "some avg10=0.00 avg60=1.00 avg300=2.00 total=10\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=0\n"; + let cpu_high = "some avg10=0.01 avg60=0.00 avg300=0.00 total=10\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=0\n"; + let stall_high = "some avg10=2.00 avg60=1.00 avg300=2.00 total=10\nfull avg10=0.01 avg60=0.00 avg300=0.00 total=1\n"; + assert_eq!( + parse_linux_psi(healthy, PressureKind::Cpu), + SignalState::Healthy + ); + assert_eq!( + parse_linux_psi(cpu_high, PressureKind::Cpu), + SignalState::High + ); + assert_eq!( + parse_linux_psi(healthy, PressureKind::Stall), + SignalState::Healthy + ); + assert_eq!( + parse_linux_psi(stall_high, PressureKind::Stall), + SignalState::High + ); + assert_eq!( + parse_linux_psi("garbled", PressureKind::Cpu), + SignalState::Unknown + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_power_sampler_distinguishes_ac_battery_saver_and_desktop() { + let dir = tempfile::tempdir().unwrap(); + assert_eq!(sample_linux_power_at(dir.path()), PowerState::NoBattery); + + let ac = dir.path().join("AC"); + std::fs::create_dir(&ac).unwrap(); + std::fs::write(ac.join("type"), "Mains\n").unwrap(); + std::fs::write(ac.join("online"), "1\n").unwrap(); + assert_eq!(sample_linux_power_at(dir.path()), PowerState::External); + + std::fs::write(ac.join("online"), "0\n").unwrap(); + let battery = dir.path().join("BAT0"); + std::fs::create_dir(&battery).unwrap(); + std::fs::write(battery.join("type"), "Battery\n").unwrap(); + std::fs::write(battery.join("capacity"), "80\n").unwrap(); + assert_eq!(sample_linux_power_at(dir.path()), PowerState::Battery); + + std::fs::write(battery.join("capacity"), "5\n").unwrap(); + assert_eq!(sample_linux_power_at(dir.path()), PowerState::BatterySaving); + } +} diff --git a/crates/aft/src/search_index.rs b/crates/aft/src/search_index.rs index 1636484a5..50346a617 100644 --- a/crates/aft/src/search_index.rs +++ b/crates/aft/src/search_index.rs @@ -59,6 +59,40 @@ static TRANSIENT_SEARCH_CACHE_SWEEP_CURSORS: OnceLock>>>> = OnceLock::new(); +const SEARCH_STAGING_VERSION: u32 = 1; +const SEARCH_STAGING_MANIFEST: &str = "search-staging-v1.json"; +const SEARCH_STAGING_DIR: &str = "search-staging-v1"; +const SEARCH_SLICE_FILES: usize = 32; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SearchBuildSliceOutcome { + Yielded, + Complete, +} + +#[derive(Debug, Deserialize, Serialize)] +struct SearchStagingManifest { + version: u32, + corpus_fingerprint: String, + canonical_root: PathBuf, + ignore_fingerprint: String, + max_file_size: u64, + paths: Vec, + cursor: usize, + spill_seq: usize, + files: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +struct SearchStagingFile { + path: PathBuf, + size: u64, + modified_nanos: u128, + content_hash: [u8; 32], + indexed: bool, + included: bool, + trigram_count: u32, +} #[cfg(debug_assertions)] thread_local! { @@ -900,6 +934,156 @@ impl SearchIndex { } } } + pub(crate) fn resume_cold_build_slice( + root: &Path, + max_file_size: u64, + cache_dir: &Path, + ) -> std::io::Result { + fs::create_dir_all(cache_dir)?; + let staging_dir = cache_dir.join(SEARCH_STAGING_DIR); + let manifest_path = staging_dir.join(SEARCH_STAGING_MANIFEST); + let canonical_root = fs::canonicalize(root).unwrap_or_else(|_| root.to_path_buf()); + let ignore_fingerprint = ignore_rules_fingerprint(&canonical_root); + let filters = PathFilters::default(); + let paths = walk_project_files(&canonical_root, &filters); + let corpus_fingerprint = + search_corpus_fingerprint(&canonical_root, &ignore_fingerprint, max_file_size, &paths); + let mut manifest = load_search_staging_manifest(&manifest_path) + .filter(|manifest| { + manifest.version == SEARCH_STAGING_VERSION + && manifest.corpus_fingerprint == corpus_fingerprint + && manifest.canonical_root == canonical_root + && manifest.ignore_fingerprint == ignore_fingerprint + && manifest.max_file_size == max_file_size + && manifest.paths == paths + && manifest.cursor <= manifest.paths.len() + && manifest.files.len() == manifest.cursor + }) + .unwrap_or_else(|| { + let _ = fs::remove_dir_all(&staging_dir); + SearchStagingManifest { + version: SEARCH_STAGING_VERSION, + corpus_fingerprint: corpus_fingerprint.clone(), + canonical_root: canonical_root.clone(), + ignore_fingerprint: ignore_fingerprint.clone(), + max_file_size, + paths: paths.clone(), + cursor: 0, + spill_seq: 0, + files: Vec::new(), + } + }); + fs::create_dir_all(&staging_dir)?; + + if manifest.cursor < manifest.paths.len() { + let end = (manifest.cursor + SEARCH_SLICE_FILES).min(manifest.paths.len()); + let mut block = Vec::new(); + for path in &manifest.paths[manifest.cursor..end] { + let file_id = u32::try_from(manifest.files.len()) + .map_err(|_| std::io::Error::other("too many files to index"))?; + match prepare_search_path(path, max_file_size) { + PreparedSearchPath::Indexed(file) => { + let trigram_count = + u32::try_from(file.trigram_map.len()).unwrap_or(u32::MAX); + for (trigram, filter) in file.trigram_map { + block.push(SpillRecord { + trigram, + file_id, + next_mask: filter.next_mask, + loc_mask: filter.loc_mask, + }); + } + manifest.files.push(search_staging_file( + path, + file.metadata, + file.content_hash, + true, + true, + trigram_count, + )); + } + PreparedSearchPath::Unindexed(metadata) => { + manifest.files.push(search_staging_file( + path, + metadata, + cache_freshness::zero_hash(), + false, + true, + 0, + )) + } + PreparedSearchPath::Skipped => manifest.files.push(search_staging_file( + path, + SearchFileMetadata { + size: 0, + modified: UNIX_EPOCH, + }, + cache_freshness::zero_hash(), + false, + false, + 0, + )), + } + } + if !block.is_empty() { + flush_spill_segment(&staging_dir, manifest.spill_seq, &mut block)?; + manifest.spill_seq += 1; + } + manifest.cursor = end; + write_search_staging_manifest(&manifest_path, &manifest)?; + return Ok(SearchBuildSliceOutcome::Yielded); + } + + let mut files = Vec::with_capacity(manifest.files.len()); + let mut path_to_id = HashMap::with_capacity(manifest.files.len()); + let mut unindexed_files = HashSet::new(); + let mut file_trigram_count = Vec::with_capacity(manifest.files.len()); + for staged in manifest.files.iter().filter(|staged| staged.included) { + let file_id = u32::try_from(files.len()) + .map_err(|_| std::io::Error::other("too many files to index"))?; + let seconds = u64::try_from(staged.modified_nanos / 1_000_000_000).unwrap_or(u64::MAX); + let nanos = u32::try_from(staged.modified_nanos % 1_000_000_000).unwrap_or(0); + files.push(FileEntry { + path: staged.path.clone(), + size: staged.size, + modified: UNIX_EPOCH + Duration::new(seconds, nanos), + content_hash: blake3::Hash::from_bytes(staged.content_hash), + }); + path_to_id.insert(staged.path.clone(), file_id); + if !staged.indexed { + unindexed_files.insert(file_id); + } + file_trigram_count.push(staged.trigram_count); + } + let plan = CacheWritePlan { + project_root: canonical_root.clone(), + git_head: current_git_head(&canonical_root), + ignore_fingerprint, + max_file_size, + files: files.clone(), + path_to_id: path_to_id.clone(), + unindexed_files: unindexed_files.clone(), + file_trigram_count: file_trigram_count.clone(), + id_map: Arc::new( + (0..files.len()) + .filter_map(|id| { + let id = u32::try_from(id).ok()?; + Some((id, id)) + }) + .collect(), + ), + }; + let mut sources: Vec> = (0..manifest.spill_seq) + .map(|seq| SpillSegmentSource::open(&staging_dir.join(format!("segment.{seq:06}.bin")))) + .collect::>>()? + .into_iter() + .map(|source| Box::new(source) as Box) + .collect(); + let base = write_cache_file_from_sources(cache_dir, &plan, &mut sources)?; + drop(base); + fs::remove_dir_all(&staging_dir)?; + Ok(SearchBuildSliceOutcome::Complete) + } fn build_in_memory(root: &Path, max_file_size: u64, started: Instant) -> Self { let project_root = fs::canonicalize(root).unwrap_or_else(|_| root.to_path_buf()); @@ -951,6 +1135,11 @@ impl SearchIndex { .num_threads(pool_size) .thread_name(|index| format!("aft-search-build-{index}")) .stack_size(8 * 1024 * 1024) + .start_handler(|_| { + // Search builds are background maintenance. Keep transport and + // interactive reader threads ahead in the OS CPU and I/O schedulers. + crate::thread_priority::demote_background(); + }) .build() { Ok(pool) => Some(pool), @@ -2883,6 +3072,11 @@ fn build_streaming_index( .num_threads(pool_size) .thread_name(|index| format!("aft-search-build-{index}")) .stack_size(8 * 1024 * 1024) + .start_handler(|_| { + // One large root can keep every search worker busy for seconds. + // Demote each worker so concurrent roots cannot starve SubC control traffic. + crate::thread_priority::demote_background(); + }) .build() .ok(); @@ -3245,9 +3439,75 @@ fn build_lookup_section_bytes(lookup_entries: &[LookupEntry]) -> std::io::Result .map_err(|error| std::io::Error::other(error.to_string()))? .into_inner(); let checksum = crc32fast::hash(&lookup_blob); + lookup_blob.extend_from_slice(&checksum.to_le_bytes()); Ok(lookup_blob) } +fn search_staging_file( + path: &Path, + metadata: SearchFileMetadata, + content_hash: blake3::Hash, + indexed: bool, + included: bool, + trigram_count: u32, +) -> SearchStagingFile { + let modified_nanos = metadata + .modified + .duration_since(UNIX_EPOCH) + .unwrap_or(Duration::ZERO) + .as_nanos(); + SearchStagingFile { + path: path.to_path_buf(), + size: metadata.size, + modified_nanos, + content_hash: *content_hash.as_bytes(), + indexed, + included, + trigram_count, + } +} + +fn search_corpus_fingerprint( + root: &Path, + ignore_fingerprint: &str, + max_file_size: u64, + paths: &[PathBuf], +) -> String { + let mut hasher = blake3::Hasher::new(); + hasher.update(root.to_string_lossy().as_bytes()); + hasher.update(ignore_fingerprint.as_bytes()); + hasher.update(&max_file_size.to_le_bytes()); + for path in paths { + hasher.update(path.to_string_lossy().as_bytes()); + if let Ok(metadata) = fs::metadata(path) { + hasher.update(&metadata.len().to_le_bytes()); + let modified = metadata + .modified() + .ok() + .and_then(|time| time.duration_since(UNIX_EPOCH).ok()) + .map_or(0, |duration| duration.as_nanos()); + hasher.update(&modified.to_le_bytes()); + } + } + hasher.finalize().to_hex().to_string() +} + +fn load_search_staging_manifest(path: &Path) -> Option { + serde_json::from_slice(&fs::read(path).ok()?).ok() +} + +fn write_search_staging_manifest( + path: &Path, + manifest: &SearchStagingManifest, +) -> std::io::Result<()> { + let bytes = serde_json::to_vec(manifest).map_err(std::io::Error::other)?; + let temporary = path.with_extension("json.tmp"); + fs::write(&temporary, bytes)?; + File::open(&temporary)?.sync_all()?; + fs::rename(&temporary, path)?; + sync_parent_dir(path); + Ok(()) +} fn build_file_trigram_count_extension(counts: &[u32]) -> std::io::Result> { let mut writer = BufWriter::new(Cursor::new(Vec::new())); @@ -8266,6 +8526,68 @@ mod tests { ); } + #[test] + fn resumable_search_build_yields_then_matches_monolithic_results() { + let dir = tempfile::tempdir().expect("create temp dir"); + let project = dir.path().join("project"); + let cache = dir.path().join("cache"); + fs::create_dir_all(&project).expect("create project"); + for index in 0..70 { + fs::write( + project.join(format!("file_{index:03}.rs")), + format!("pub fn marker_{index}() {{ println!(\"resume_marker_{index}\"); }}\n"), + ) + .expect("write source"); + } + let expected = SearchIndex::build_with_limit(&project, DEFAULT_MAX_FILE_SIZE); + let first = SearchIndex::resume_cold_build_slice(&project, DEFAULT_MAX_FILE_SIZE, &cache) + .expect("first slice"); + assert_eq!(first, SearchBuildSliceOutcome::Yielded); + assert!(!cache.join("cache.bin").exists()); + + let mut slices = 1; + while SearchIndex::resume_cold_build_slice(&project, DEFAULT_MAX_FILE_SIZE, &cache) + .expect("resume slice") + == SearchBuildSliceOutcome::Yielded + { + slices += 1; + } + assert!(slices >= 2); + let actual = SearchIndex::read_from_disk(&cache, &project).expect("published index"); + let expected_result = expected.grep("resume_marker_37", true, &[], &[], &project, 10); + let actual_result = actual.grep("resume_marker_37", true, &[], &[], &project, 10); + assert_eq!(expected_result.matches, actual_result.matches); + assert_eq!(expected_result.total_matches, actual_result.total_matches); + } + + #[test] + fn resumable_search_rejects_corrupt_and_changed_staging() { + let dir = tempfile::tempdir().expect("create temp dir"); + let project = dir.path().join("project"); + let cache = dir.path().join("cache"); + fs::create_dir_all(&project).expect("create project"); + for index in 0..40 { + fs::write(project.join(format!("file_{index:03}.rs")), "fn old() {}\n") + .expect("write source"); + } + assert_eq!( + SearchIndex::resume_cold_build_slice(&project, DEFAULT_MAX_FILE_SIZE, &cache) + .expect("first slice"), + SearchBuildSliceOutcome::Yielded + ); + let manifest = cache.join(SEARCH_STAGING_DIR).join(SEARCH_STAGING_MANIFEST); + fs::write(&manifest, b"not-json").expect("corrupt manifest"); + fs::write(project.join("file_000.rs"), "fn changed() {}\n").expect("change corpus"); + assert_eq!( + SearchIndex::resume_cold_build_slice(&project, DEFAULT_MAX_FILE_SIZE, &cache) + .expect("restart slice"), + SearchBuildSliceOutcome::Yielded + ); + let restarted = load_search_staging_manifest(&manifest).expect("replacement manifest"); + assert_eq!(restarted.cursor, SEARCH_SLICE_FILES); + assert_eq!(restarted.files.len(), SEARCH_SLICE_FILES); + } + #[test] fn ignore_rule_discovery_respects_gitignore() { let _git_env = crate::test_env::hermetic_git_env_guard(); diff --git a/crates/aft/src/semantic_index.rs b/crates/aft/src/semantic_index.rs index 400ae9f14..fd5e84862 100644 --- a/crates/aft/src/semantic_index.rs +++ b/crates/aft/src/semantic_index.rs @@ -23,7 +23,7 @@ use std::io::{self, BufReader, BufWriter, Cursor, Read, Write}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, OnceLock, Weak}; -use std::time::{Duration, Instant, SystemTime}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use url::Url; const DEFAULT_DIMENSION: usize = 384; @@ -121,6 +121,38 @@ impl EmbeddingRequestPolicy { } } +const SEMANTIC_STAGING_VERSION: u32 = 1; +const SEMANTIC_STAGING_FILE: &str = "semantic-staging-v1.json"; +const SEMANTIC_COLLECT_SLICE_FILES: usize = 32; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SemanticBuildSliceOutcome { + Yielded, + Complete, +} + +#[derive(Debug, Serialize, Deserialize)] +struct SemanticStagingManifest { + version: u32, + canonical_root: PathBuf, + fingerprint: SemanticIndexFingerprint, + files: Vec, + corpus_fingerprint: String, + collect_cursor: usize, + embed_cursor: usize, + chunks: Vec, + metadata: Vec, + vectors: Vec>, +} + +#[derive(Debug, Serialize, Deserialize)] +struct SemanticStagingMetadata { + path: PathBuf, + modified_nanos: u128, + size: u64, + content_hash: [u8; 32], +} + pub struct SemanticIndexLock { _guard: Option, } @@ -1822,7 +1854,7 @@ pub fn format_embedding_init_error(error: impl Display) -> String { } /// A chunk of code ready for embedding — derived from a Symbol with context enrichment -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct SemanticChunk { /// Absolute file path pub file: PathBuf, @@ -2020,6 +2052,42 @@ fn borrowed_artifact_identity(data_path: &Path) -> Result<(String, blake3::Hash) let fingerprint = String::from_utf8(fingerprint).map_err(|error| error.to_string())?; Ok((fingerprint, artifact_content_hash)) } +fn semantic_corpus_fingerprint( + root: &Path, + files: &[PathBuf], + fingerprint: &SemanticIndexFingerprint, +) -> String { + let mut hasher = blake3::Hasher::new(); + hasher.update(root.to_string_lossy().as_bytes()); + hasher.update(&serde_json::to_vec(fingerprint).unwrap_or_default()); + for path in files { + hasher.update(path.to_string_lossy().as_bytes()); + if let Ok(metadata) = fs::metadata(path) { + hasher.update(&metadata.len().to_le_bytes()); + let modified = metadata + .modified() + .ok() + .and_then(|time| time.duration_since(UNIX_EPOCH).ok()) + .map_or(0, |duration| duration.as_nanos()); + hasher.update(&modified.to_le_bytes()); + } + } + hasher.finalize().to_hex().to_string() +} + +fn load_semantic_staging(path: &Path) -> Option { + serde_json::from_slice(&fs::read(path).ok()?).ok() +} + +fn write_semantic_staging(path: &Path, manifest: &SemanticStagingManifest) -> Result<(), String> { + let temporary = path.with_extension("json.tmp"); + let bytes = serde_json::to_vec(manifest).map_err(|error| error.to_string())?; + fs::write(&temporary, bytes).map_err(|error| error.to_string())?; + fs::File::open(&temporary) + .and_then(|file| file.sync_all()) + .map_err(|error| error.to_string())?; + crate::fs_lock::rename_over(&temporary, path).map_err(|error| error.to_string()) +} /// The semantic index — stores embeddings for all symbols in a project. /// Borrow-only roots retain only a root path plus an Arc to immutable relative data. @@ -2778,6 +2846,149 @@ impl SemanticIndex { &mut should_continue, ) } + pub(crate) fn resume_cold_build_slice( + project_root: &Path, + files: &[PathBuf], + model: &mut SemanticEmbeddingModel, + config: &SemanticBackendConfig, + storage_dir: &Path, + project_key: &str, + ) -> Result { + let canonical_root = + fs::canonicalize(project_root).unwrap_or_else(|_| project_root.to_path_buf()); + let fingerprint = model.fingerprint(config)?; + let corpus_fingerprint = semantic_corpus_fingerprint(&canonical_root, files, &fingerprint); + let dir = storage_dir.join("semantic").join(project_key); + let staging_path = dir.join(SEMANTIC_STAGING_FILE); + fs::create_dir_all(&dir).map_err(|error| error.to_string())?; + let mut manifest = load_semantic_staging(&staging_path) + .filter(|manifest| { + manifest.version == SEMANTIC_STAGING_VERSION + && manifest.canonical_root == canonical_root + && manifest.corpus_fingerprint == corpus_fingerprint + && manifest.files == files + && manifest.collect_cursor <= files.len() + && manifest.embed_cursor <= manifest.chunks.len() + && manifest.vectors.len() == manifest.embed_cursor + && manifest + .vectors + .iter() + .all(|vector| vector.len() == fingerprint.dimension) + }) + .unwrap_or(SemanticStagingManifest { + version: SEMANTIC_STAGING_VERSION, + canonical_root: canonical_root.clone(), + fingerprint: fingerprint.clone(), + files: files.to_vec(), + corpus_fingerprint, + collect_cursor: 0, + embed_cursor: 0, + chunks: Vec::new(), + metadata: Vec::new(), + vectors: Vec::new(), + }); + + if manifest.collect_cursor < files.len() { + let end = (manifest.collect_cursor + SEMANTIC_COLLECT_SLICE_FILES).min(files.len()); + let (chunks, metadata) = + Self::collect_chunks(&canonical_root, &files[manifest.collect_cursor..end]); + manifest.chunks.extend(chunks); + manifest + .metadata + .extend(metadata.into_iter().map(|(path, metadata)| { + SemanticStagingMetadata { + path, + modified_nanos: metadata + .mtime + .duration_since(UNIX_EPOCH) + .unwrap_or(Duration::ZERO) + .as_nanos(), + size: metadata.size, + content_hash: *metadata.content_hash.as_bytes(), + } + })); + manifest.collect_cursor = end; + write_semantic_staging(&staging_path, &manifest)?; + return Ok(SemanticBuildSliceOutcome::Yielded); + } + + if manifest.embed_cursor < manifest.chunks.len() { + let end = + (manifest.embed_cursor + model.max_batch_size().max(1)).min(manifest.chunks.len()); + let texts = manifest.chunks[manifest.embed_cursor..end] + .iter() + .map(|chunk| chunk.embed_text.clone()) + .collect(); + let vectors = model.embed(texts)?; + validate_embedding_batch(&vectors, end - manifest.embed_cursor, "embedding backend")?; + if vectors + .iter() + .any(|vector| vector.len() != fingerprint.dimension) + { + let _ = fs::remove_file(&staging_path); + return Err( + "embedding dimension changed during resumable semantic build".to_string(), + ); + } + manifest.vectors.extend(vectors); + manifest.embed_cursor = end; + write_semantic_staging(&staging_path, &manifest)?; + return Ok(SemanticBuildSliceOutcome::Yielded); + } + + let file_metadata = manifest + .metadata + .iter() + .map(|metadata| { + let seconds = + u64::try_from(metadata.modified_nanos / 1_000_000_000).unwrap_or(u64::MAX); + let nanos = u32::try_from(metadata.modified_nanos % 1_000_000_000).unwrap_or(0); + ( + metadata.path.clone(), + IndexedFileMetadata { + mtime: UNIX_EPOCH + Duration::new(seconds, nanos), + size: metadata.size, + content_hash: blake3::Hash::from_bytes(metadata.content_hash), + }, + ) + }) + .collect::>(); + let entries = manifest + .chunks + .into_iter() + .zip(manifest.vectors) + .map(|(chunk, vector)| EmbeddingEntry::new(chunk, vector)) + .collect::>(); + let mut index = Self { + entries, + file_mtimes: file_metadata + .iter() + .map(|(path, metadata)| (path.clone(), metadata.mtime)) + .collect(), + file_sizes: file_metadata + .iter() + .map(|(path, metadata)| (path.clone(), metadata.size)) + .collect(), + any_missing_sizes: false, + file_hashes: file_metadata + .into_iter() + .map(|(path, metadata)| (path, metadata.content_hash)) + .collect(), + dimension: fingerprint.dimension, + fingerprint: Some(fingerprint), + project_root: canonical_root, + deferred_files: HashSet::new(), + shared_base: None, + #[cfg(test)] + removal_retain_passes: 0, + }; + index.materialize_shared_base(); + if !index.write_to_disk(storage_dir, project_key) { + return Err("failed to publish resumable semantic index".to_string()); + } + fs::remove_file(&staging_path).map_err(|error| error.to_string())?; + Ok(SemanticBuildSliceOutcome::Complete) + } /// Build the semantic index and report embedding progress using entry counts. pub fn build_with_progress( @@ -5579,6 +5790,60 @@ mod tests { (format!("http://{}", addr), handle) } + fn start_resumable_embedding_server( + expected_requests: usize, + ) -> (String, thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind resumable server"); + let addr = listener.local_addr().expect("local addr"); + let handle = thread::spawn(move || { + for _ in 0..expected_requests { + let (mut stream, _) = listener.accept().expect("accept embedding request"); + let mut bytes = Vec::new(); + let mut buffer = [0u8; 4096]; + let (header_end, content_length) = loop { + let count = stream.read(&mut buffer).expect("read embedding request"); + bytes.extend_from_slice(&buffer[..count]); + if let Some(position) = + bytes.windows(4).position(|window| window == b"\r\n\r\n") + { + let headers = String::from_utf8_lossy(&bytes[..position + 4]); + let length = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok()) + .flatten() + }) + .unwrap_or(0); + if bytes.len() >= position + 4 + length { + break (position + 4, length); + } + } + }; + let request: serde_json::Value = + serde_json::from_slice(&bytes[header_end..header_end + content_length]) + .expect("embedding request JSON"); + let input_count = request + .get("input") + .and_then(serde_json::Value::as_array) + .map_or(1, Vec::len); + let data = (0..input_count).map(|index| { + serde_json::json!({"embedding": [1.0, index as f32, 0.25], "index": index}) + }).collect::>(); + let body = serde_json::json!({"data": data}).to_string(); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), body + ); + stream + .write_all(response.as_bytes()) + .expect("write embedding response"); + } + }); + (format!("http://{addr}"), handle) + } + fn start_truncated_body_server(attempts: usize) -> (String, thread::JoinHandle<()>) { let listener = TcpListener::bind("127.0.0.1:0").expect("bind truncated test server"); listener @@ -7340,6 +7605,125 @@ public class Greeter { .any(|entry| entry.chunk.name == "old_symbol")); } + #[test] + fn resumable_semantic_build_yields_rejects_stale_state_and_matches_monolithic_corpus() { + let dir = tempfile::tempdir().expect("temp dir"); + let project = dir.path().join("project"); + fs::create_dir_all(&project).expect("project dir"); + for index in 0..35 { + fs::write( + project.join(format!("file_{index:03}.rs")), + format!("pub fn marker_{index}() {{ println!(\"semantic_resume_{index}\"); }}\n"), + ) + .expect("write fixture"); + } + let files = (0..35) + .map(|index| project.join(format!("file_{index:03}.rs"))) + .collect::>(); + let (base_url, server) = start_resumable_embedding_server(2); + let config = SemanticBackendConfig { + backend: SemanticBackend::OpenAiCompatible, + model: "resume-test".to_string(), + base_url: Some(base_url), + max_batch_size: 256, + ..Default::default() + }; + let mut model = SemanticEmbeddingModel::from_config(&config).expect("model"); + assert_eq!( + SemanticIndex::resume_cold_build_slice( + &project, + &files, + &mut model, + &config, + dir.path(), + "resume" + ) + .expect("collect slice"), + SemanticBuildSliceOutcome::Yielded, + ); + let staging = dir + .path() + .join("semantic/resume") + .join(SEMANTIC_STAGING_FILE); + assert!(staging.exists()); + fs::write(&staging, b"corrupt").expect("corrupt staging"); + assert_eq!( + SemanticIndex::resume_cold_build_slice( + &project, + &files, + &mut model, + &config, + dir.path(), + "resume" + ) + .expect("replacement collect slice"), + SemanticBuildSliceOutcome::Yielded, + ); + assert_eq!( + SemanticIndex::resume_cold_build_slice( + &project, + &files, + &mut model, + &config, + dir.path(), + "resume" + ) + .expect("final collect slice"), + SemanticBuildSliceOutcome::Yielded, + ); + assert_eq!( + SemanticIndex::resume_cold_build_slice( + &project, + &files, + &mut model, + &config, + dir.path(), + "resume" + ) + .expect("embedding slice"), + SemanticBuildSliceOutcome::Yielded, + ); + assert_eq!( + SemanticIndex::resume_cold_build_slice( + &project, + &files, + &mut model, + &config, + dir.path(), + "resume" + ) + .expect("publish slice"), + SemanticBuildSliceOutcome::Complete, + ); + server.join().expect("embedding server"); + let fingerprint = model.fingerprint(&config).expect("fingerprint").as_string(); + let actual = SemanticIndex::read_from_disk( + dir.path(), + "resume", + &project, + false, + Some(&fingerprint), + ) + .expect("published semantic index"); + let mut embedder = RecordingEmbedder::default(); + let expected = + SemanticIndex::build(&project, &files, &mut |texts| embedder.embed(texts), 256) + .expect("monolithic index"); + assert_eq!(actual.len(), expected.len()); + let actual_names = actual + .entries + .iter() + .map(|entry| (&entry.chunk.file, &entry.chunk.name)) + .collect::>(); + let expected_names = expected + .entries + .iter() + .map(|entry| (&entry.chunk.file, &entry.chunk.name)) + .collect::>(); + assert_eq!(actual_names, expected_names); + assert!(!staging.exists()); + } + #[test] fn refresh_all_clean_reports_zero_counts_and_no_embedding_work() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/aft/src/standing_roots.rs b/crates/aft/src/standing_roots.rs index e1b04f777..c93e871ca 100644 --- a/crates/aft/src/standing_roots.rs +++ b/crates/aft/src/standing_roots.rs @@ -721,7 +721,10 @@ mod tests { fn config(storage: &Path, roots: Vec) -> Config { Config { storage_dir: Some(storage.to_path_buf()), - index: IndexConfig { roots }, + index: IndexConfig { + roots, + ..IndexConfig::default() + }, ..Config::default() } } diff --git a/crates/aft/src/standing_scheduler.rs b/crates/aft/src/standing_scheduler.rs new file mode 100644 index 000000000..b41f67b1a --- /dev/null +++ b/crates/aft/src/standing_scheduler.rs @@ -0,0 +1,180 @@ +use std::collections::{HashMap, HashSet, VecDeque}; +use std::hash::Hash; + +#[derive(Debug)] +pub struct DeficitRoundRobin { + quantum: u64, + queue: VecDeque, + deficits: HashMap, + in_flight: HashSet, +} + +impl DeficitRoundRobin +where + K: Clone + Eq + Hash, +{ + pub fn new(quantum: u64) -> Self { + assert!(quantum > 0, "DRR quantum must be positive"); + Self { + quantum, + queue: VecDeque::new(), + deficits: HashMap::new(), + in_flight: HashSet::new(), + } + } + + pub fn reconcile(&mut self, keys: I) + where + I: IntoIterator, + { + let keys = keys.into_iter().collect::>(); + self.queue.retain(|key| keys.contains(key)); + self.deficits.retain(|key, _| keys.contains(key)); + self.in_flight.retain(|key| keys.contains(key)); + for key in keys { + if !self.deficits.contains_key(&key) { + self.deficits.insert(key.clone(), 0); + self.queue.push_back(key); + } + } + } + + pub fn next(&mut self) -> Option { + if self.queue.is_empty() { + return None; + } + let rounds = self.queue.len(); + for _ in 0..rounds { + let key = self.queue.pop_front()?; + let deficit = self.deficits.get_mut(&key)?; + *deficit += i128::from(self.quantum); + if *deficit >= 0 { + self.in_flight.insert(key.clone()); + return Some(key); + } + self.queue.push_back(key); + } + None + } + + pub fn complete(&mut self, key: K, cost: u64, has_more: bool) { + if !self.in_flight.remove(&key) { + return; + } + if let Some(deficit) = self.deficits.get_mut(&key) { + *deficit -= i128::from(cost); + } + if has_more { + self.queue.push_back(key); + } else { + self.deficits.remove(&key); + } + } + + pub fn len(&self) -> usize { + self.deficits.len() + } + + pub fn is_empty(&self) -> bool { + self.deficits.is_empty() + } +} +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)] +pub struct StandingSchedulerTelemetry { + pub queued_roots: usize, + pub running_slices: usize, + pub completed_slices: u64, + pub yielded_slices: u64, + pub pause_reason: Option, + pub resource_policy: String, +} + +static TELEMETRY: std::sync::LazyLock> = + std::sync::LazyLock::new(|| parking_lot::RwLock::new(StandingSchedulerTelemetry::default())); + +pub fn publish_telemetry(snapshot: StandingSchedulerTelemetry) { + *TELEMETRY.write() = snapshot; +} + +pub fn telemetry() -> StandingSchedulerTelemetry { + TELEMETRY.read().clone() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn active_roots_rotate_without_starvation() { + let mut scheduler = DeficitRoundRobin::new(10); + scheduler.reconcile(["large", "small", "medium"]); + let mut order = Vec::new(); + for cost in [10, 10, 10, 10, 10, 10] { + let key = scheduler.next().unwrap(); + order.push(key); + scheduler.complete(key, cost, true); + } + assert_eq!( + order, + ["large", "small", "medium", "large", "small", "medium"] + ); + } + + #[test] + fn expensive_root_pays_debt_before_its_next_slice() { + let mut scheduler = DeficitRoundRobin::new(10); + scheduler.reconcile(["large", "small"]); + let large = scheduler.next().unwrap(); + scheduler.complete(large, 30, true); + let small = scheduler.next().unwrap(); + scheduler.complete(small, 5, true); + assert_eq!(scheduler.next(), Some("small")); + } + + #[test] + fn reconcile_removes_stale_and_appends_new_roots_deterministically() { + let mut scheduler = DeficitRoundRobin::new(10); + scheduler.reconcile(["a", "b"]); + let a = scheduler.next().unwrap(); + scheduler.complete(a, 10, true); + scheduler.reconcile(["b", "c"]); + assert_eq!(scheduler.next(), Some("b")); + scheduler.complete("b", 10, true); + assert_eq!(scheduler.next(), Some("c")); + } + + #[test] + fn completed_root_leaves_the_queue() { + let mut scheduler = DeficitRoundRobin::new(10); + scheduler.reconcile(["a", "b"]); + let a = scheduler.next().unwrap(); + scheduler.complete(a, 3, false); + assert_eq!(scheduler.len(), 1); + assert_eq!(scheduler.next(), Some("b")); + } + + #[test] + fn up_to_two_distinct_roots_can_be_in_flight() { + let mut scheduler = DeficitRoundRobin::new(10); + scheduler.reconcile(["a", "b", "c"]); + assert_eq!(scheduler.next(), Some("a")); + assert_eq!(scheduler.next(), Some("b")); + scheduler.complete("a", 10, true); + scheduler.complete("b", 10, true); + assert_eq!(scheduler.next(), Some("c")); + } + + #[test] + fn scheduler_telemetry_round_trips_health_fields() { + let expected = StandingSchedulerTelemetry { + queued_roots: 8, + running_slices: 2, + completed_slices: 21, + yielded_slices: 3, + pause_reason: Some("io_pressure".to_string()), + resource_policy: "balanced".to_string(), + }; + publish_telemetry(expected.clone()); + assert_eq!(telemetry(), expected); + } +} diff --git a/crates/aft/src/subc/health.rs b/crates/aft/src/subc/health.rs index 1a4aa6e04..28286b3b9 100644 --- a/crates/aft/src/subc/health.rs +++ b/crates/aft/src/subc/health.rs @@ -1029,6 +1029,11 @@ pub(super) fn build_health_report( "dispatch_liveness".to_string(), dispatch_liveness_metrics(executor), ); + metrics.insert( + "standing_scheduler".to_string(), + serde_json::to_value(crate::standing_scheduler::telemetry()) + .unwrap_or_else(|_| json!({ "unavailable": true })), + ); let mut dispatch_path = dispatch_path_metrics.snapshot(pending_binds); if let Some(dispatch_path) = dispatch_path.as_object_mut() { dispatch_path.insert("mutating_lanes".to_string(), mutating_lanes); @@ -1825,6 +1830,7 @@ mod tests { path: root.display().to_string(), indexes: vec![crate::config::IndexKind::Search], }], + ..crate::config::IndexConfig::default() }, ..crate::config::Config::default() } diff --git a/crates/aft/src/subc/standing.rs b/crates/aft/src/subc/standing.rs index 6d250da01..478c8c6c0 100644 --- a/crates/aft/src/subc/standing.rs +++ b/crates/aft/src/subc/standing.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; use std::sync::Arc; +use std::time::Instant; use parking_lot::Mutex; @@ -13,8 +14,10 @@ use crate::config::{Config, IndexKind}; use crate::context::{App, AppContext}; use crate::executor::{Executor, Lane, MaintenanceCoalesceKey}; use crate::path_identity::ProjectRootId; +use crate::resource_policy::{sample_resources, AdmissionDecision, ResourceAdmissionGate}; use crate::root_cache; use crate::standing_roots::{StandingRootEntry, StandingRoots}; +use crate::standing_scheduler::DeficitRoundRobin; /// The standing cadence is intentionally the same arm cadence that already /// drives `due_maintenance_jobs`; no standing timer or scheduler is created. @@ -23,6 +26,40 @@ pub(super) const STANDING_MAINTENANCE_INTERVAL: std::time::Duration = super::DRA #[cfg(test)] static LAST_STANDING_VERIFY_STRATEGY: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); +const STANDING_SERVICE_QUANTUM_MS: u64 = 250; + +struct PendingStandingSlice { + receiver: tokio::sync::oneshot::Receiver, + started_at: Instant, +} + +struct StandingScheduleState { + queue: DeficitRoundRobin, + entries: HashMap, + next_kind: HashMap, + pending: HashMap, + resource_gate: ResourceAdmissionGate, + completed_slices: u64, + yielded_slices: u64, + pause_reason: Option, + resource_policy: String, +} + +impl Default for StandingScheduleState { + fn default() -> Self { + Self { + queue: DeficitRoundRobin::new(STANDING_SERVICE_QUANTUM_MS), + entries: HashMap::new(), + next_kind: HashMap::new(), + pending: HashMap::new(), + resource_gate: ResourceAdmissionGate::default(), + completed_slices: 0, + yielded_slices: 0, + pause_reason: None, + resource_policy: "balanced".to_string(), + } + } +} pub(super) struct StandingActor { app: Arc, @@ -32,6 +69,7 @@ pub(super) struct StandingActor { /// Root ids registered solely to host unbound standing work. Session actors /// are never removed by this owner. owned_actors: Mutex>, + schedule: Mutex, } impl StandingActor { @@ -42,6 +80,7 @@ impl StandingActor { roots: StandingRoots::default(), observed_config: Mutex::new(Config::default()), owned_actors: Mutex::new(HashMap::new()), + schedule: Mutex::new(StandingScheduleState::default()), } } @@ -114,10 +153,7 @@ impl StandingActor { } } } - - /// Reconcile the observed snapshot and enqueue one coalesced pass per root. - /// Entry order and `search`, `semantic`, `callgraph` kind order are retained - /// by `StandingRoots::entries` and `IndexKind::ALL` respectively. + /// Reconcile configured roots, collect completed slices, and fill available slots. pub(super) fn tick(&self) { self.observe_config_snapshot(); let snapshot = self.observed_config.lock().clone(); @@ -128,11 +164,139 @@ impl StandingActor { return; } }; - self.retire_removed_actors(&report.removed); self.resume_entries_without_bound_session(&report.active_entries); - for entry in report.active_entries { - self.submit_entry_pass(entry, &snapshot); + self.reconcile_schedule(report.active_entries); + self.drain_completed_slices(); + if matches!( + self.schedule + .lock() + .resource_gate + .observe(snapshot.index.resource_policy, sample_resources(),), + AdmissionDecision::Admit + ) { + self.dispatch_ready_slices(&snapshot); + } + } + + fn reconcile_schedule(&self, entries: Vec) { + let mut schedule = self.schedule.lock(); + let keys = entries + .iter() + .map(|entry| entry.literal_path.clone()) + .collect::>(); + schedule.queue.reconcile(keys.iter().cloned()); + schedule.entries = entries + .into_iter() + .map(|entry| (entry.literal_path.clone(), entry)) + .collect(); + schedule.next_kind.retain(|key, _| keys.contains(key)); + for key in keys { + schedule.next_kind.entry(key).or_insert(0); + } + Self::publish_schedule_telemetry(&schedule); + } + + fn publish_schedule_telemetry(schedule: &StandingScheduleState) { + crate::standing_scheduler::publish_telemetry( + crate::standing_scheduler::StandingSchedulerTelemetry { + queued_roots: schedule.queue.len().saturating_sub(schedule.pending.len()), + running_slices: schedule.pending.len(), + completed_slices: schedule.completed_slices, + yielded_slices: schedule.yielded_slices, + pause_reason: schedule.pause_reason.clone(), + resource_policy: schedule.resource_policy.clone(), + }, + ); + } + + fn drain_completed_slices(&self) { + let mut schedule = self.schedule.lock(); + let completed = schedule + .pending + .iter_mut() + .filter_map(|(key, pending)| match pending.receiver.try_recv() { + Ok(response) => Some((key.clone(), response, pending.started_at.elapsed())), + Err(tokio::sync::oneshot::error::TryRecvError::Closed) => Some(( + key.clone(), + crate::protocol::Response::error( + "standing", + "standing_slice_closed", + "standing slice response channel closed", + ), + pending.started_at.elapsed(), + )), + Err(tokio::sync::oneshot::error::TryRecvError::Empty) => None, + }) + .collect::>(); + for (key, response, elapsed) in completed { + schedule.pending.remove(&key); + let has_more = response + .data + .get("has_more") + .and_then(serde_json::Value::as_bool) + .unwrap_or(true); + let kind_complete = response + .data + .get("kind_complete") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + if kind_complete { + let next = schedule.next_kind.get(&key).copied().unwrap_or(0) + 1; + schedule.next_kind.insert(key.clone(), next); + } + if !has_more { + schedule.next_kind.insert(key.clone(), 0); + } + let cost = u64::try_from(elapsed.as_millis()) + .unwrap_or(u64::MAX) + .max(1); + schedule.completed_slices = schedule.completed_slices.saturating_add(1); + if response + .data + .get("yielded") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) + { + schedule.yielded_slices = schedule.yielded_slices.saturating_add(1); + } + Self::publish_schedule_telemetry(&schedule); + schedule.queue.complete(key, cost, has_more); + } + } + + fn dispatch_ready_slices(&self, snapshot: &Config) { + loop { + let entry = { + let mut schedule = self.schedule.lock(); + if schedule.pending.len() >= crate::cold_build_limiter::limit() { + return; + } + let Some(key) = schedule.queue.next() else { + return; + }; + let Some(entry) = schedule.entries.get(&key).cloned() else { + schedule.queue.complete(key, 1, false); + continue; + }; + entry + }; + let Some(receiver) = self.submit_entry_slice(entry.clone(), snapshot) else { + self.schedule + .lock() + .queue + .complete(entry.literal_path, 1, true); + continue; + }; + let mut schedule = self.schedule.lock(); + schedule.pending.insert( + entry.literal_path, + PendingStandingSlice { + receiver, + started_at: Instant::now(), + }, + ); + Self::publish_schedule_telemetry(&schedule); } } @@ -184,28 +348,39 @@ impl StandingActor { } } - fn submit_entry_pass(&self, entry: StandingRootEntry, snapshot: &Config) { - let Some(root_id) = self.ensure_actor(&entry, snapshot) else { - return; + fn submit_entry_slice( + &self, + entry: StandingRootEntry, + snapshot: &Config, + ) -> Option> { + let root_id = self.ensure_actor(&entry, snapshot)?; + let kind_index = *self + .schedule + .lock() + .next_kind + .get(&entry.literal_path) + .unwrap_or(&0); + let selected = IndexKind::ALL + .iter() + .copied() + .enumerate() + .skip(kind_index) + .find(|(_, kind)| entry.indexes.contains(kind)); + let Some((kind_index, kind)) = selected else { + return None; }; + let roots = self.roots.clone(); let literal_path = entry.literal_path.clone(); - let executor_request_id = format!("subc-standing-pass-{}", entry.literal_path); + let executor_request_id = format!("subc-standing-slice-{}-{}", literal_path, kind.as_str()); let response_request_id = executor_request_id.clone(); let job = Box::new(move |ctx: &AppContext| { let Some(admission) = roots.admit_build(&literal_path) else { return crate::protocol::Response::success( response_request_id, - serde_json::json!({"standing": true, "entry": literal_path, "admitted": false}), + serde_json::json!({"standing": true, "entry": literal_path, "admitted": false, "has_more": true}), ); }; - // Nonblocking standing admission: lifecycle admission stays inside - // this serialized maintenance job, but the cold-build acquire is - // immediate. When no slot is available the pass YIELDS — returning - // without advancing publication state — and the 250ms standing - // tick resubmits the coalesced pass. A yielded pass therefore never - // occupies a maintenance worker waiting for a cold slot, so heavy - // standing indexing cannot consume interactive reader capacity. let Some(permit) = crate::cold_build_limiter::try_acquire_standing_with_limiter( &ctx.cold_build_limiter(), format!("standing:{}", literal_path), @@ -213,56 +388,66 @@ impl StandingActor { ) else { return crate::protocol::Response::success( response_request_id, - serde_json::json!({"standing": true, "entry": literal_path, "admitted": false, "yielded": true}), + serde_json::json!({"standing": true, "entry": literal_path, "admitted": false, "yielded": true, "has_more": true}), ); }; debug_assert_eq!( permit.admission_epoch, admission.publication.admission_epoch ); - for kind in IndexKind::ALL { - if !entry.indexes.contains(&kind) { - continue; - } - if crate::executor::current_job_cancelled() { - break; - } - // A strict plan is selected unconditionally at a standing pass - // boundary. The artifact-specific loader/build code consumes the - // plan when a resident or disk artifact is present; a failed or - // interrupted attempt intentionally leaves the durable flag set. - let verified = strict_verify_current_state(ctx, &entry, kind) - || (kind == IndexKind::Search - && build_missing_search_after_strict_check( - ctx, - &roots, - &entry, - &admission, - permit.admission_epoch, - )); - if verified { - if let Err(error) = roots.record_strict_verification(&literal_path, kind) { - log::warn!( - "standing strict verification outcome could not commit for {} {}: {}", - literal_path, - kind.as_str(), - error - ); - } + let (kind_complete, yielded) = if crate::executor::current_job_cancelled() { + (false, true) + } else if strict_verify_current_state(ctx, &entry, kind) { + (true, false) + } else if kind == IndexKind::Search { + build_missing_search_after_strict_check( + ctx, + &roots, + &entry, + &admission, + permit.admission_epoch, + ) + } else if kind == IndexKind::Semantic { + build_missing_semantic_after_strict_check(ctx, &entry) + } else { + (false, true) + }; + if kind_complete { + if let Err(error) = roots.record_strict_verification(&literal_path, kind) { + log::warn!( + "standing strict verification outcome could not commit for {} {}: {}", + literal_path, + kind.as_str(), + error + ); } } + let has_later_kind = entry.indexes.iter().any(|candidate| { + IndexKind::ALL + .iter() + .position(|kind| kind == candidate) + .is_some_and(|index| index > kind_index) + }); + let has_more = !kind_complete || has_later_kind; crate::protocol::Response::success( response_request_id, - serde_json::json!({"standing": true, "entry": literal_path}), + serde_json::json!({ + "standing": true, + "entry": literal_path, + "kind": kind.as_str(), + "kind_complete": kind_complete, + "yielded": yielded, + "has_more": has_more, + }), ) }); - let _ = self.executor.submit_coalescable_maintenance_async( + Some(self.executor.submit_coalescable_maintenance_async( root_id, Lane::MaintenanceCommit, executor_request_id, MaintenanceCoalesceKey::StandingPass, job, - ); + )) } fn ensure_actor(&self, entry: &StandingRootEntry, snapshot: &Config) -> Option { @@ -370,9 +555,9 @@ fn build_missing_search_after_strict_check( entry: &StandingRootEntry, admission: &crate::standing_roots::StandingBuildAdmission, permit_epoch: u64, -) -> bool { +) -> (bool, bool) { if admission.cancellation_requested() || crate::executor::current_job_cancelled() { - return false; + return (false, true); } let config = ctx.config(); let cache_dir = crate::search_index::resolve_cache_dir_with_key( @@ -381,13 +566,7 @@ fn build_missing_search_after_strict_check( ); let max_file_size = config.search_index_max_file_size; drop(config); - let before_fingerprint = root_fingerprint(&entry.resolved_target); let configure_generation = ctx.configure_generation(); - let mut index = crate::search_index::SearchIndex::build_with_limit_to_cache_dir( - &entry.resolved_target, - max_file_size, - &cache_dir, - ); let lease = match crate::root_cache::WriterLease::acquire_shared( crate::root_cache::RootCacheDomain::Index, &cache_dir, @@ -395,39 +574,77 @@ fn build_missing_search_after_strict_check( &entry.resolved_target, ) { Ok(Some(lease)) => lease, - Ok(None) | Err(_) => return false, + Ok(None) | Err(_) => return (false, true), }; - roots + let outcome = roots .publish_if_current( &entry.literal_path, admission.publication, &lease, - || root_fingerprint(&entry.resolved_target) == before_fingerprint, + || true, || { permit_epoch == admission.publication.admission_epoch && ctx.configure_generation() == configure_generation && !admission.cancellation_requested() }, || { - index.write_to_disk( + crate::search_index::SearchIndex::resume_cold_build_slice( + &entry.resolved_target, + max_file_size, &cache_dir, - crate::search_index::current_git_head(&entry.resolved_target).as_deref(), ) + .ok() }, ) .ok() .flatten() - .unwrap_or(false) + .flatten(); + match outcome { + Some(crate::search_index::SearchBuildSliceOutcome::Complete) => (true, false), + Some(crate::search_index::SearchBuildSliceOutcome::Yielded) | None => (false, true), + } } -fn root_fingerprint(root: &std::path::Path) -> Option<(u64, Option)> { - let metadata = std::fs::metadata(root).ok()?; - let modified = metadata - .modified() - .ok() - .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) - .map(|time| time.as_nanos()); - Some((metadata.len(), modified)) +fn build_missing_semantic_after_strict_check( + ctx: &AppContext, + entry: &StandingRootEntry, +) -> (bool, bool) { + let config = ctx.config(); + let semantic_config = config.semantic.clone(); + let storage_dir = config.storage_dir.clone(); + drop(config); + let Some(storage_dir) = storage_dir else { + return (false, true); + }; + let files = match crate::commands::configure::walk_semantic_project_files_bounded( + &entry.resolved_target, + semantic_config.max_files, + ) { + Ok(files) => files, + Err(_) => return (false, true), + }; + let mut model = match crate::semantic_index::EmbeddingModel::from_config(&semantic_config) { + Ok(model) => model, + Err(error) => { + log::warn!("standing semantic model initialization failed: {}", error); + return (false, true); + } + }; + match crate::semantic_index::SemanticIndex::resume_cold_build_slice( + &entry.resolved_target, + &files, + &mut model, + &semantic_config, + &storage_dir, + &entry.artifact_key, + ) { + Ok(crate::semantic_index::SemanticBuildSliceOutcome::Complete) => (true, false), + Ok(crate::semantic_index::SemanticBuildSliceOutcome::Yielded) => (false, true), + Err(error) => { + log::warn!("standing semantic slice failed: {}", error); + (false, true) + } + } } #[cfg(test)] diff --git a/crates/aft/tests/integration/inspect_engine_test.rs b/crates/aft/tests/integration/inspect_engine_test.rs index b8b0884fb..6fbe2f709 100644 --- a/crates/aft/tests/integration/inspect_engine_test.rs +++ b/crates/aft/tests/integration/inspect_engine_test.rs @@ -102,7 +102,10 @@ fn interleaving_worker( let is_large = job.project_root == large_root; if is_large { large_started.store(true, Ordering::SeqCst); - thread::sleep(Duration::from_millis(800)); + let deadline = Instant::now() + Duration::from_secs(5); + while !small_finished.load(Ordering::SeqCst) && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } large_finished.store(true, Ordering::SeqCst); } else { small_interleaved.store(!large_finished.load(Ordering::SeqCst), Ordering::SeqCst); diff --git a/crates/aft/tests/integration/lsp_rename_test.rs b/crates/aft/tests/integration/lsp_rename_test.rs index 84b5b5082..0534c5453 100644 --- a/crates/aft/tests/integration/lsp_rename_test.rs +++ b/crates/aft/tests/integration/lsp_rename_test.rs @@ -49,11 +49,17 @@ fn rust_workspace_with_file() -> (tempfile::TempDir, PathBuf) { (temp_dir, main_rs) } -fn app_context_with_fake_lsp() -> AppContext { - let ctx = AppContext::new(Box::new(TreeSitterProvider::new()), Config::default()); +fn app_context_with_fake_lsp() -> (AppContext, tempfile::TempDir) { + let storage = tempdir().expect("checkpoint storage tempdir"); + let mut config = Config::default(); + config.storage_dir = Some(storage.path().to_path_buf()); + let ctx = AppContext::new(Box::new(TreeSitterProvider::new()), config); ctx.lsp() .override_binary(ServerKind::Rust, fake_server_path()); - ctx + ctx.checkpoint() + .lock() + .set_storage_dir_for_harness(storage.path().to_path_buf(), aft::harness::Harness::Pi); + (ctx, storage) } fn collect_event(ctx: &AppContext, predicate: F) -> Option @@ -103,7 +109,7 @@ fn file_uri(path: &Path) -> String { #[test] fn test_prepare_rename_success() { let (_temp_dir, main_rs) = rust_workspace_with_file(); - let ctx = app_context_with_fake_lsp(); + let (ctx, _storage) = app_context_with_fake_lsp(); let req: RawRequest = serde_json::from_value(serde_json::json!({ "id": "prepare-1", @@ -129,7 +135,7 @@ fn test_prepare_rename_success() { #[test] fn test_rename_applies_changes() { let (_temp_dir, main_rs) = rust_workspace_with_file(); - let ctx = app_context_with_fake_lsp(); + let (ctx, _storage) = app_context_with_fake_lsp(); let req: RawRequest = serde_json::from_value(serde_json::json!({ "id": "rename-1", @@ -161,7 +167,7 @@ fn test_rename_applies_changes() { #[test] fn test_rename_rollback_on_failure() { let (_temp_dir, main_rs) = rust_workspace_with_file(); - let ctx = app_context_with_fake_lsp(); + let (ctx, _storage) = app_context_with_fake_lsp(); let original = fs::read_to_string(&main_rs).expect("read original file"); let req: RawRequest = serde_json::from_value(serde_json::json!({ @@ -194,7 +200,7 @@ fn test_rename_rollback_on_failure() { #[test] fn test_rename_rollback_on_failure_when_backups_disabled() { let (_temp_dir, main_rs) = rust_workspace_with_file(); - let ctx = app_context_with_fake_lsp(); + let (ctx, _storage) = app_context_with_fake_lsp(); ctx.backup().lock().set_policy(BackupPolicy { enabled: false, ..BackupPolicy::default() @@ -227,7 +233,7 @@ fn test_rename_rollback_on_failure_when_backups_disabled() { #[test] fn test_rename_notifies_lsp() { let (_temp_dir, main_rs) = rust_workspace_with_file(); - let ctx = app_context_with_fake_lsp(); + let (ctx, _storage) = app_context_with_fake_lsp(); let expected_uri = file_uri(&main_rs); let req: RawRequest = serde_json::from_value(serde_json::json!({ diff --git a/crates/aft/tests/standing_roots_acceptance_test.rs b/crates/aft/tests/standing_roots_acceptance_test.rs index c9719eb81..cf2753633 100644 --- a/crates/aft/tests/standing_roots_acceptance_test.rs +++ b/crates/aft/tests/standing_roots_acceptance_test.rs @@ -47,7 +47,10 @@ use serde_json::Value; fn config(storage: &Path, roots: Vec) -> Config { Config { storage_dir: Some(storage.to_path_buf()), - index: IndexConfig { roots }, + index: IndexConfig { + roots, + ..IndexConfig::default() + }, ..Config::default() } } diff --git a/docs/config.md b/docs/config.md index 2ad3689ac..76edf5154 100644 --- a/docs/config.md +++ b/docs/config.md @@ -133,6 +133,15 @@ The backup store treats its on-disk tree as authoritative across processes; dele // Default: false "search_index": false, + // Background index admission policy. Default: "balanced". + // "balanced" pauses new standing-root slices on battery saving, memory or I/O + // pressure, and resumes only after consecutive healthy samples. "performance" + // ignores battery and pressure admission while retaining bounded concurrency, + // fair root rotation, slice checkpoints, and background OS thread priority. + "index": { + "resource_policy": "balanced" // "balanced" | "performance" + }, + // Linked-worktree RAM overlay for the trigram index. Default: false. // When true, a borrow-only worktree applies its own file-watcher events to // the in-RAM delta of the borrowed search index (and invalidates the symbol diff --git a/packages/opencode-plugin/src/__tests__/config.test.ts b/packages/opencode-plugin/src/__tests__/config.test.ts index 47b75e04b..4b1bc2ecf 100644 --- a/packages/opencode-plugin/src/__tests__/config.test.ts +++ b/packages/opencode-plugin/src/__tests__/config.test.ts @@ -915,6 +915,34 @@ describe("loadAftConfig", () => { } }); + test("index resource policy defaults, validates, and remains user-only", () => { + expect(AftConfigSchema.parse({}).index?.resource_policy ?? "balanced").toBe("balanced"); + expect( + AftConfigSchema.parse({ index: { resource_policy: "balanced" } }).index?.resource_policy, + ).toBe("balanced"); + expect( + AftConfigSchema.parse({ index: { resource_policy: "performance" } }).index?.resource_policy, + ).toBe("performance"); + expect(AftConfigSchema.safeParse({ index: { resource_policy: "unlimited" } }).success).toBe( + false, + ); + + const fixture = createConfigFixture(); + writeFileSync( + fixture.userConfigPath, + JSON.stringify({ index: { resource_policy: "performance" } }), + ); + writeFileSync( + fixture.projectConfigPath, + JSON.stringify({ index: { resource_policy: "balanced" } }), + ); + const result = runConfigLoader(fixture.projectDirectory, { + HOME: join(fixture.root, "home"), + XDG_CONFIG_HOME: fixture.xdgConfigHome, + }); + expect(JSON.parse(result.stdout).index.resource_policy).toBe("performance"); + }); + test("strict schema still rejects keys outside both harnesses", () => { expect(AftConfigSchema.safeParse({ genuinely_unknown_key: true }).success).toBe(false); }); diff --git a/packages/opencode-plugin/src/config.ts b/packages/opencode-plugin/src/config.ts index b333a0761..7540dfd95 100644 --- a/packages/opencode-plugin/src/config.ts +++ b/packages/opencode-plugin/src/config.ts @@ -117,6 +117,7 @@ const IndexRootSchema = z })); const IndexConfigSchema = z.object({ + resource_policy: z.enum(["balanced", "performance"]).optional(), roots: z.array(IndexRootSchema).optional(), }); diff --git a/packages/pi-plugin/src/__tests__/config.test.ts b/packages/pi-plugin/src/__tests__/config.test.ts index 4ae4319eb..1d36100b4 100644 --- a/packages/pi-plugin/src/__tests__/config.test.ts +++ b/packages/pi-plugin/src/__tests__/config.test.ts @@ -66,6 +66,34 @@ afterEach(() => { tempRoots.clear(); }); + test("index resource policy defaults, validates, and remains user-only", () => { + expect(AftConfigSchema.parse({}).index?.resource_policy ?? "balanced").toBe("balanced"); + expect( + AftConfigSchema.parse({ index: { resource_policy: "balanced" } }).index?.resource_policy, + ).toBe("balanced"); + expect( + AftConfigSchema.parse({ index: { resource_policy: "performance" } }).index?.resource_policy, + ).toBe("performance"); + expect(AftConfigSchema.safeParse({ index: { resource_policy: "unlimited" } }).success).toBe( + false, + ); + + const fixture = createConfigFixture(); + writeFileSync( + fixture.userConfigPath, + JSON.stringify({ index: { resource_policy: "performance" } }), + ); + writeFileSync( + fixture.projectConfigPath, + JSON.stringify({ index: { resource_policy: "balanced" } }), + ); + const result = runConfigLoader(fixture.projectDirectory, { + HOME: fixture.home, + XDG_CONFIG_HOME: fixture.xdgConfigHome, + }); + expect(JSON.parse(result.stdout).index.resource_policy).toBe("performance"); + }); + describe("loadAftConfig", () => { test("gh_read honors only the user tier and warns for project overrides", () => { const fixture = createConfigFixture(); diff --git a/packages/pi-plugin/src/config.ts b/packages/pi-plugin/src/config.ts index e89a16e73..626f0d9e4 100644 --- a/packages/pi-plugin/src/config.ts +++ b/packages/pi-plugin/src/config.ts @@ -124,6 +124,7 @@ export interface IndexRootConfig { } export interface IndexConfig { + resource_policy?: "balanced" | "performance"; roots?: IndexRootConfig[]; } @@ -521,6 +522,7 @@ const IndexRootSchema = z })); const IndexConfigSchema = z.object({ + resource_policy: z.enum(["balanced", "performance"]).optional(), roots: z.array(IndexRootSchema).optional(), }); diff --git a/specs/standing-index-resource-policy/plan.md b/specs/standing-index-resource-policy/plan.md new file mode 100644 index 000000000..02c4e6dc7 --- /dev/null +++ b/specs/standing-index-resource-policy/plan.md @@ -0,0 +1,220 @@ +# Implementation Plan: Pressure-Aware Standing Index Scheduler + +## Goal + +Keep configured standing indexes progressing across many roots without making the AFT daemon unresponsive or consuming laptop power continuously. Replace the current submit-every-root loop with a process-wide fair scheduler. Make the safe resource policy the default. Add an explicit user-only performance policy for operators who accept unrestricted background power use. + +## Configuration Contract + +Add this user-tier configuration: + +```jsonc +{ + "index": { + "resource_policy": "balanced", + "roots": [] + } +} +``` + +`resource_policy` accepts: + +| Value | Behavior | +|---|---| +| `balanced` | Default. Admit bounded background slices only while the host has adequate resources. Pause new slices on battery-saving or high-pressure signals. Keep interactive readers independent. | +| `performance` | Ignore battery and host-pressure admission signals. Keep queue bounds, cancellation, publication fences, and background thread priority demotion. | + +The field remains user-only with `index.roots`. Project configuration cannot weaken the machine owner policy. Unknown values fail config validation. Existing configurations resolve to `balanced`. + +## Architecture + +### Process-Wide Fair Scheduler + +Replace the loop in `StandingActor::tick` that submits every active root. Add scheduler state owned by `StandingActor`: + +- A stable ring ordered by the normalized standing-root entry order. +- A cursor that advances after each admitted slice. +- Per-root artifact-kind progress in fixed `search`, `semantic`, `callgraph` order. +- At most the available cold-build slots worth of submitted standing slices. +- One coalesced executor job per selected root. + +Use deficit round robin with one unit per bounded artifact slice. A root that yields because the cold limiter or resource policy denies admission retains its position without accumulating unbounded credit. A root that completes a slice advances behind the other runnable roots. A removed root loses its scheduler state. A paused session-owned root leaves the runnable ring and resumes at its prior artifact cursor after unbind. + +The scheduler never waits in an executor worker. It first checks resource admission and then uses the existing immediate standing cold-build acquire. The 250 ms tick only performs cheap admission and scheduling checks. It does not write artifact state. A denied root is reconsidered on a later tick. + +### Resource Admission + +Add `crates/aft/src/resource_policy.rs` as a platform adapter with a small pure decision core. + +`balanced` admits a new slice only when all available authoritative signals permit it: + +- Linux: use `/sys/class/power_supply/*/type` plus `online` or `status` to detect external power. Use `/proc/pressure/cpu`, `/proc/pressure/memory`, and `/proc/pressure/io` for pressure stall information when available. Use `sysinfo`-independent standard-library reads. +- macOS: use IOKit power-source state and `getloadavg` or host statistics through existing platform FFI patterns. Do not execute subprocesses. +- Windows: use `GetSystemPowerStatus` and system load or memory status through direct Win32 FFI. +- Unsupported or unreadable signals: fail conservatively for a portable host-pressure signal, but do not classify a desktop with no battery as battery-powered. Record a named unknown signal in telemetry. + +The decision core applies hysteresis. It requires consecutive healthy samples before resuming and pauses immediately on a hard battery-saving or memory-pressure signal. Sampling occurs on the standing tick and is cached for a bounded interval. It never runs on an interactive request path. + +`performance` bypasses this admission decision only. It does not bypass the cold-build concurrency limit, executor caps, cancellation, writer leases, publication epochs, or `thread_priority` demotion. + +Do not expose numeric pressure thresholds in configuration in this change. Keep one supported safe policy and one explicit bypass. Thresholds must be based on platform semantics and measured acceptance tests, not arbitrary user knobs. + +## Resumable Artifact Slices + +The slices solve two separate problems. First, they bound how long one root owns a scarce cold-build slot, which lets other roots make progress. Second, they preserve completed expensive work across rotation, cancellation, daemon restart, or supersession. A slice is a substantial unit of artifact work, not one scheduler tick. + +Persist only after a slice performs real work and reaches an existing safe commit boundary. Do not write while idle, denied, or waiting. Coalesce cursor metadata with the slice output and rate-limit metadata-only checkpoints. The 250 ms scheduler cadence must never become a 250 ms disk-write cadence. + +### Callgraph + +Reuse the existing durable staging database and corpus fingerprint in `crates/aft/src/callgraph_store/mod.rs`. + +Refactor the internal cold-build stage loop into `resume_cold_build_slice` with a bounded work budget. Return `Progress`, `Complete`, `Superseded`, or `Failed`. Stop at existing durable boundaries: + +- File extraction inventory batches. +- Extraction batches capped by the existing file and byte limits. +- Resolution windows capped by the existing reference limit. +- Dispatch and publication barriers. + +Commit the stage cursor in the same transaction as completed stage work before returning `Progress`. Preserve existing corpus-change restart and same-corpus adoption behavior. Do not issue a cursor-only commit when no stage work completed. + +### Search + +Add a durable search staging manifest under the existing transient build directory. Key it by the artifact cache key, corpus fingerprint, search format version, ignore-rule fingerprint, and max-file-size policy. + +Split `build_streaming_index` into resumable phases: + +1. Stable file inventory and metadata snapshot. +2. Bounded file collection and trigram spill-segment generation. +3. Bounded merge runs into staged postings and lookup sections. +4. Header, checksum, fsync, and atomic publication. + +Persist the next inventory index with completed spill or merge output after each work slice. Do not persist on scheduler ticks or denied admission. A matching successor adopts the staging manifest. A changed fingerprint discards the staging generation. Published readers continue to use the previous complete generation until the final atomic swap. + +### Semantic + +Add a semantic staging file under the existing semantic cache root. Key it by corpus fingerprint plus `SemanticIndexFingerprint`, chunking version, and model table epoch. + +Split build work into: + +1. Bounded source collection to stable chunk records. +2. Bounded embedding batches using the configured backend batch limit. +3. Append-only persisted embedding records with per-batch checksum. +4. Final deterministic assembly and atomic semantic cache publication. + +Resume only when every fingerprint component matches. Persist an embedding checkpoint only after a completed backend batch, and combine its cursor with the appended embedding records. Do not write on scheduler ticks. Truncate an incomplete final record after a crash. Never expose partial semantic results. Preserve query cache isolation and existing cancellation checks. + +## Code Changes + +### Configuration + +- `crates/aft/src/config.rs`: add `IndexResourcePolicy` and `IndexConfig.resource_policy`, defaulting to `Balanced`. +- `crates/aft/src/config_resolve.rs`: add `RawIndex.resource_policy`, enforce user-only ownership, resolve the default, and report invalid values. +- `packages/opencode-plugin/src/config.ts`: add the duplicated Zod enum and default-preserving index schema field. +- `packages/pi-plugin/src/config.ts`: add the same schema contract. +- `assets/aft.schema.json`: regenerate the public schema through the existing schema build path. + +### Scheduling and Admission + +- `crates/aft/src/subc/standing.rs`: replace submit-all ticking with fair runnable-root selection, artifact cursors, bounded slice dispatch, and resource admission. +- `crates/aft/src/resource_policy.rs`: add platform sampling, cached snapshots, hysteresis, pure admission decisions, and telemetry types. +- `crates/aft/src/lib.rs`: register the new module. +- `crates/aft/src/cold_build_limiter.rs`: expose the current available standing capacity or a non-consuming admission query if the scheduler needs it. Keep the immediate permit API authoritative inside the serialized job. +- `crates/aft/src/subc/health.rs`: expose policy, power state, pressure state, pause reason, runnable-root count, scheduler cursor, slice completions, yields, and resumes. +- `crates/aft/src/logging.rs`: add the same compact standing-scheduler fields to busy executor diagnostics. + +### Artifact Slices + +- `crates/aft/src/callgraph_store/mod.rs`: expose one durable bounded cold-build slice. +- `crates/aft/src/search_index.rs`: add the staging manifest, resumable spill/merge phases, and atomic finalization. +- `crates/aft/src/semantic_index.rs`: add persisted chunk/embedding batches and deterministic finalization. +- `crates/aft/src/context.rs` and `crates/aft/src/subc/standing.rs`: route each selected artifact slice through the matching resume API and commit standing verification only after complete publication. + +### Documentation + +- `docs/config.md`: document standing roots, `balanced`, `performance`, the user-only boundary, and the fact that performance still respects safety and correctness bounds. +- `ARCHITECTURE.md`: document fair slice scheduling, resource admission, durable resume, and publication visibility. +- `STRUCTURE.md`: list the resource-policy module and staging responsibilities. + +## TDD Task List + +### Phase 1: Configuration and Decision Core + +- [ ] Add failing Rust config tests for omitted, balanced, performance, invalid, and project-tier stripping. +- [ ] Add failing OpenCode and Pi config tests for the same contract. +- [ ] Implement `IndexResourcePolicy` through all config surfaces. +- [ ] Add failing pure decision tests for AC power, battery saving, pressure, unknown signals, hysteresis, and performance bypass. +- [ ] Implement the platform-neutral resource decision core and platform samplers. + +### Phase 2: Fair Scheduler + +- [ ] Add failing standing unit tests proving deterministic rotation, no root starvation, removal, session pause/resume, denied-admission retry, and bounded submissions. +- [ ] Implement the runnable ring, cursor, per-kind state, and slice completion feedback. +- [ ] Add failing telemetry tests for policy and pause reasons. +- [ ] Implement health and logging projection. + +### Phase 3: Callgraph Slices + +- [ ] Add a failing test that stops after one durable callgraph slice and resumes in a new store instance. +- [ ] Add failing same-corpus adoption and changed-corpus restart tests at each stage boundary. +- [ ] Refactor the existing stage loop into the bounded resume API. + +### Phase 4: Search Slices + +- [ ] Add failing crash/resume tests for inventory, spill generation, merge, and pre-publication boundaries. +- [ ] Add failing changed-corpus and corrupt-manifest rejection tests. +- [ ] Implement the staged search format and bounded resume API. +- [ ] Prove byte-equivalent logical query results against the existing monolithic builder. + +### Phase 5: Semantic Slices + +- [ ] Add failing resume tests across chunk collection, embedding batches, and finalization. +- [ ] Add failing fingerprint, table-epoch, partial-record, and cancellation tests. +- [ ] Implement semantic staging and bounded resume. +- [ ] Prove result equivalence and that no partial result is visible. + +### Phase 6: End-to-End Load Contract + +- [ ] Extend `crates/aft/tests/standing_roots_acceptance_test.rs` with many roots and all artifact kinds. +- [ ] Extend `crates/aft/tests/integration/subc_storm_test.rs` to prove reader and health latency while roots rotate and pause. +- [ ] Add a performance-policy case that ignores simulated battery and pressure signals while preserving queue and cold-build bounds. +- [ ] Run the release-calibrated storm gate. +- [ ] Run the complete Rust and bridge regression suites. +- [ ] Update the configuration and architecture documentation. + +## Acceptance Criteria + +- Given more runnable roots than cold-build slots, each root completes bounded slices in deterministic rotation without starvation. +- Given `resource_policy: balanced` and a battery-saving or high-pressure signal, no new standing slice starts. Interactive reads and health checks remain responsive. +- Given recovery to a healthy state, hysteresis prevents rapid pause/resume oscillation and standing work resumes automatically. +- Given `resource_policy: performance`, standing work ignores battery and pressure admission while all correctness and concurrency bounds remain active. +- Given a daemon restart or superseded builder, matching search, semantic, and callgraph staging resumes from the last committed boundary. +- Given a corpus or model fingerprint change, incompatible staging is rejected and rebuilt. +- Given an incomplete staging artifact, readers continue to use the prior complete generation. +- Given static-root load, the release storm meets its existing retry-free latency contracts. + +## Verification + +Run these gates after the focused RED/GREEN cycles: + +```bash +cargo test -p agent-file-tools standing_roots +cargo test -p agent-file-tools callgraph_store +cargo test -p agent-file-tools search_index +cargo test -p agent-file-tools semantic_index +cargo test -p agent-file-tools --test integration subc_storm_test +bun test packages/opencode-plugin/src/__tests__/config.test.ts packages/pi-plugin/src/__tests__/config.test.ts +AFT_GATE_PHASES=storm scripts/rust-test-gate.sh +cargo test -p agent-file-tools +bun test packages/aft-bridge packages/opencode-plugin packages/pi-plugin +``` + +The load test must record per-root slice counts, maximum reader latency, health latency, pause duration, and process CPU time. Compare `balanced` and `performance` with the same root corpus. Treat these as acceptance evidence rather than permanent fixed thresholds unless the existing release storm already defines a limit. + +## Risks + +- Risk: A monolithic search or semantic build defeats root fairness. Implement true intra-kind resume before claiming fairness complete. +- Risk: A partial staging format corrupts published readers. Keep staging generation-specific and publish only through the existing atomic generation swap. +- Risk: Platform signals differ or disappear. Keep the decision core explicit about unknown data and expose the reason in health telemetry. +- Risk: A performance bypass disables correctness controls. Limit the bypass to resource admission only. +- Risk: Frequent durable checkpoints increase write amplification. Measure staging writes in the acceptance test and use existing natural batch boundaries. From 6376984f1100ec37c7c09ab4aac25e1bc9557b5f Mon Sep 17 00:00:00 2001 From: Naadir Jeewa Date: Mon, 31 Aug 2026 15:47:52 +0100 Subject: [PATCH 04/14] fix(subc): preserve responsiveness under load Bound Pi-facing requests below the host deadline, move allocator scans off the transport thread, and avoid repeated standing-root reconciliation. Signed-off-by: Naadir Jeewa --- crates/aft/src/main.rs | 11 ++- crates/aft/src/memory.rs | 89 +++++++++---------- crates/aft/src/subc/mod.rs | 89 +++++++++++++++---- crates/aft/src/subc/standing.rs | 82 ++++++++++++++--- .../src/__tests__/bridge-transport.test.ts | 29 +++++- packages/aft-bridge/src/bridge.ts | 6 ++ packages/aft-bridge/src/subc-transport.ts | 20 +++-- .../pi-plugin/src/__tests__/_shared.test.ts | 34 ++++++- packages/pi-plugin/src/__tests__/bash.test.ts | 18 ++-- .../pi-plugin/src/__tests__/e2e/bash.test.ts | 10 +-- .../pi-plugin/src/__tests__/inspect.test.ts | 10 ++- packages/pi-plugin/src/tools/_shared.ts | 64 +++++++++---- packages/pi-plugin/src/tools/ast.ts | 8 +- packages/pi-plugin/src/tools/bash.ts | 35 ++++---- packages/pi-plugin/src/tools/conflicts.ts | 4 +- packages/pi-plugin/src/tools/fs.ts | 34 +++---- packages/pi-plugin/src/tools/hoisted.ts | 24 ++--- packages/pi-plugin/src/tools/imports.ts | 4 +- packages/pi-plugin/src/tools/inspect.ts | 28 ++++-- packages/pi-plugin/src/tools/navigate.ts | 4 +- packages/pi-plugin/src/tools/reading.ts | 10 +-- packages/pi-plugin/src/tools/refactor.ts | 4 +- packages/pi-plugin/src/tools/safety.ts | 4 +- 23 files changed, 413 insertions(+), 208 deletions(-) diff --git a/crates/aft/src/main.rs b/crates/aft/src/main.rs index fc3d3ec11..605cfc367 100644 --- a/crates/aft/src/main.rs +++ b/crates/aft/src/main.rs @@ -212,11 +212,10 @@ fn main() { const DRAIN_INTERVAL: Duration = Duration::from_millis(250); const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100); let mut pending = PendingResponses::default(); - // Opportunistic allocator relief: rate-limit stamp for the slack check that - // runs on the periodic drain wake (threshold + spacing live in memory.rs so - // subc and standalone share one policy). + // Rate-limit stamp for detached allocator slack scans. The stdin loop + // performs only a cheap cadence comparison on each periodic drain wake. #[cfg(any(target_os = "macos", target_os = "linux"))] - let mut last_slack_relief: Option = None; + let mut last_slack_scan: Option = None; let (line_tx, line_rx) = mpsc::channel::>(); let mut graceful_stdin_shutdown = false; thread::spawn(move || { @@ -252,8 +251,8 @@ fn main() { #[cfg(any(target_os = "macos", target_os = "linux"))] { let now = std::time::Instant::now(); - if aft::memory::spawn_allocator_slack_relief_if_due(last_slack_relief, now) { - last_slack_relief = Some(now); + if aft::memory::spawn_allocator_slack_scan_if_due(last_slack_scan, now) { + last_slack_scan = Some(now); } } if shutdown_requested.load(Ordering::SeqCst) { diff --git a/crates/aft/src/memory.rs b/crates/aft/src/memory.rs index 6b3a460dd..227be74cc 100644 --- a/crates/aft/src/memory.rs +++ b/crates/aft/src/memory.rs @@ -703,54 +703,49 @@ unsafe extern "C" { /// pressure relief is worth the zone-lock contention it briefly causes. pub const ALLOCATOR_SLACK_RELIEF_THRESHOLD_BYTES: u64 = 1024 * 1024 * 1024; -/// Minimum spacing between opportunistic relief passes so a workload that -/// legitimately cycles through large allocations does not thrash the allocator. -pub const ALLOCATOR_SLACK_RELIEF_MIN_INTERVAL: std::time::Duration = +/// Minimum spacing between allocator slack scans. +/// +/// Linux `mallinfo2()` walks every glibc arena under allocator locks. Keep that +/// work off the transport thread and do not repeat it on each maintenance tick. +pub const ALLOCATOR_SLACK_SCAN_MIN_INTERVAL: std::time::Duration = std::time::Duration::from_secs(300); -/// Decide whether an opportunistic allocator relief pass is due. -/// -/// Pure so the policy is unit-testable: fires only when the allocator reports -/// at least the threshold of retained slack AND the previous pass is old -/// enough. Callers own actually measuring the snapshot and running the pass. -pub fn allocator_slack_relief_due( - retained_slack_bytes: Option, - last_relief: Option, +/// Decide whether an allocator slack scan is due. +pub fn allocator_slack_scan_due( + last_scan: Option, now: std::time::Instant, ) -> bool { - let Some(slack) = retained_slack_bytes else { - return false; - }; - if slack < ALLOCATOR_SLACK_RELIEF_THRESHOLD_BYTES { - return false; - } - match last_relief { + match last_scan { None => true, - Some(at) => now.duration_since(at) >= ALLOCATOR_SLACK_RELIEF_MIN_INTERVAL, + Some(at) => now.duration_since(at) >= ALLOCATOR_SLACK_SCAN_MIN_INTERVAL, } } -/// Opportunistically return unused allocator pages when slack is large, even -/// while sessions are active. The whole-process idle sweep only fires when -/// every root has been quiet, so one long-lived chatty session used to block -/// reclamation for the process lifetime (observed: 5.1 GB RSS over ~600 MB of -/// live data). Runs the relief on a detached thread because allocator trimming -/// walks allocator state under its lock and must not stall the dispatch loop or -/// health probes. +/// Decide whether an opportunistic allocator relief pass is due for a measured +/// slack value. +pub fn allocator_slack_relief_due(retained_slack_bytes: Option) -> bool { + retained_slack_bytes.is_some_and(|slack| slack >= ALLOCATOR_SLACK_RELIEF_THRESHOLD_BYTES) +} + +/// Measure allocator slack and return unused pages from a detached thread. /// -/// Returns true when a pass was spawned (caller records the timestamp). +/// Returns true when a scan was spawned. The caller records that time so its +/// frequent transport or stdin tick performs only a cheap cadence comparison. #[cfg(any(target_os = "macos", target_os = "linux"))] -pub fn spawn_allocator_slack_relief_if_due( - last_relief: Option, +pub fn spawn_allocator_slack_scan_if_due( + last_scan: Option, now: std::time::Instant, ) -> bool { - let slack = allocator_memory_snapshot().retained_slack_bytes; - if !allocator_slack_relief_due(slack, last_relief, now) { + if !allocator_slack_scan_due(last_scan, now) { return false; } std::thread::Builder::new() .name("aft-mem-relief".to_string()) .spawn(|| { + let slack = allocator_memory_snapshot().retained_slack_bytes; + if !allocator_slack_relief_due(slack) { + return; + } let relief = relieve_allocator_pressure(); log::info!( "allocator slack relief: released={} allocator_slack_bytes_before={:?} allocator_slack_bytes_after={:?} rss_bytes_before={:?} rss_bytes_after={:?}", @@ -878,26 +873,26 @@ mod tests { } #[test] - fn slack_relief_fires_on_large_slack_and_respects_spacing() { + fn slack_relief_requires_large_measured_slack() { + let threshold = ALLOCATOR_SLACK_RELIEF_THRESHOLD_BYTES; + assert!(!allocator_slack_relief_due(None)); + assert!(!allocator_slack_relief_due(Some(threshold - 1))); + assert!(allocator_slack_relief_due(Some(threshold))); + } + + #[test] + fn slack_scan_runs_once_per_interval() { use std::time::{Duration, Instant}; let now = Instant::now(); - let big = Some(ALLOCATOR_SLACK_RELIEF_THRESHOLD_BYTES); - // Unknown slack (allocator stats unavailable) never fires. - assert!(!allocator_slack_relief_due(None, None, now)); - // Below threshold never fires. - assert!(!allocator_slack_relief_due( - Some(ALLOCATOR_SLACK_RELIEF_THRESHOLD_BYTES - 1), - None, + assert!(allocator_slack_scan_due(None, now)); + assert!(!allocator_slack_scan_due( + Some(now - Duration::from_secs(10)), + now + )); + assert!(allocator_slack_scan_due( + Some(now - ALLOCATOR_SLACK_SCAN_MIN_INTERVAL), now )); - // At threshold with no prior pass fires. - assert!(allocator_slack_relief_due(big, None, now)); - // A recent pass suppresses the next one... - let recent = now - Duration::from_secs(10); - assert!(!allocator_slack_relief_due(big, Some(recent), now)); - // ...until the minimum spacing has elapsed. - let stale = now - ALLOCATOR_SLACK_RELIEF_MIN_INTERVAL; - assert!(allocator_slack_relief_due(big, Some(stale), now)); } #[test] diff --git a/crates/aft/src/subc/mod.rs b/crates/aft/src/subc/mod.rs index 58023f317..034abd6f3 100644 --- a/crates/aft/src/subc/mod.rs +++ b/crates/aft/src/subc/mod.rs @@ -2920,8 +2920,11 @@ where // stream (the next read would parse a body byte as a frame header). A // dedicated reader task owns the socket, reads whole frames sequentially, and // forwards them over a channel; the loop selects on the cancel-safe `recv()`. - let (reader_tx, mut reader_rx) = mpsc::channel::>(256); - let reader_task = spawn_reader_task(read, reader_tx); + let (control_reader_tx, mut control_reader_rx) = + mpsc::channel::>(32); + let (data_reader_tx, mut data_reader_rx) = + mpsc::channel::>(256); + let reader_task = spawn_reader_task(read, control_reader_tx, data_reader_tx); let shutdown = Arc::new(Notify::new()); // Drain-tick deadline is tracked manually and checked at the TOP of every // loop turn rather than as an Interval select arm: the select below is @@ -2939,10 +2942,10 @@ where // this existing maintenance timer arm and never create a standing timer. standing_actor.reconcile_at_startup(); let mut next_standing_pass_at = tokio::time::Instant::now(); - // Rate-limit stamp for opportunistic allocator slack relief (checked on the - // maintenance tick; policy shared with standalone via memory.rs). + // Rate-limit stamp for detached allocator slack scans. The maintenance + // tick performs only a cheap cadence comparison on the transport thread. #[cfg(any(target_os = "macos", target_os = "linux"))] - let mut last_slack_relief: Option = None; + let mut last_slack_scan: Option = None; let (maintenance_tx, mut maintenance_rx) = mpsc::channel::(256); let (bash_deferred_tx, mut bash_deferred_rx) = mpsc::channel::(256); @@ -3169,7 +3172,7 @@ where log::warn!("subc attach: fatal executor response requested teardown"); break Ok(ModuleLoopExit::SkipSearchFlush); } - maybe_frame = reader_rx.recv() => { + maybe_frame = recv_prioritized_frame(&mut control_reader_rx, &mut data_reader_rx) => { let frame = match maybe_frame { None => { log::info!("subc attach: daemon closed connection"); @@ -3599,20 +3602,17 @@ where next_standing_pass_at = tokio::time::Instant::now() + standing::STANDING_MAINTENANCE_INTERVAL; } - // Opportunistic allocator relief, independent of the idle - // sweep: the sweep's whole-process idle gate never opens while - // any session stays active, which let freed warm-up arenas sit - // resident for the process lifetime (5.1 GB RSS over ~600 MB - // live). Slack threshold + spacing live in memory.rs; the pass - // itself runs on a detached thread. + // Scan and trim allocator arenas on a detached thread. On glibc, + // mallinfo2() walks every arena under allocator locks, so the + // transport thread must only evaluate the scan cadence here. #[cfg(any(target_os = "macos", target_os = "linux"))] { let now_std = std::time::Instant::now(); - if crate::memory::spawn_allocator_slack_relief_if_due( - last_slack_relief, + if crate::memory::spawn_allocator_slack_scan_if_due( + last_slack_scan, now_std, ) { - last_slack_relief = Some(now_std); + last_slack_scan = Some(now_std); } } next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD; @@ -3798,7 +3798,8 @@ where fn spawn_reader_task( mut read: R, - tx: mpsc::Sender>, + control_tx: mpsc::Sender>, + data_tx: mpsc::Sender>, ) -> JoinHandle<()> where R: AsyncRead + Unpin + Send + 'static, @@ -3807,16 +3808,18 @@ where loop { match read_frame(&mut read).await { Ok(Some(frame)) => { + let is_control = frame.header.channel == 0; let decoded = DecodedFrame { frame, phase_trace: PhaseTrace::new(Instant::now()), }; + let tx = if is_control { &control_tx } else { &data_tx }; if tx.send(Ok(decoded)).await.is_err() { return; } } Ok(None) => { - // EOF: let the loop observe channel close as "daemon closed". + // EOF: let the loop observe both channel closures as "daemon closed". return; } Err(error) => { @@ -3838,7 +3841,7 @@ where return; } } - let _ = tx.send(Err(SubcError::FrameIo(error))).await; + let _ = control_tx.send(Err(SubcError::FrameIo(error))).await; return; } } @@ -3846,6 +3849,17 @@ where }) } +async fn recv_prioritized_frame( + control_rx: &mut mpsc::Receiver>, + data_rx: &mut mpsc::Receiver>, +) -> Option> { + tokio::select! { + biased; + frame = control_rx.recv() => frame, + frame = data_rx.recv() => frame, + } +} + async fn finish_writer_task( mut writer_task: JoinHandle>, ) -> Result<(), SubcError> { @@ -6707,6 +6721,45 @@ mod tests { } } + #[tokio::test] + async fn reader_routes_control_frames_around_buffered_data_frames() { + let (mut daemon, module) = tokio::io::duplex(16 * 1024); + let (priority_tx, mut priority_rx) = mpsc::channel(4); + let (data_tx, mut data_rx) = mpsc::channel(4); + let reader = spawn_reader_task(module, priority_tx, data_tx); + + let data = Frame::build( + FrameType::Request, + control_flags(), + 7, + 1, + 1, + br#"{}"#.to_vec(), + ) + .unwrap(); + let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 2, Vec::new()).unwrap(); + write_frame(&mut daemon, &data).await.unwrap(); + write_frame(&mut daemon, &ping).await.unwrap(); + + tokio::time::sleep(Duration::from_millis(10)).await; + let priority = tokio::time::timeout( + Duration::from_secs(1), + recv_prioritized_frame(&mut priority_rx, &mut data_rx), + ) + .await + .expect("priority frame timeout") + .expect("priority ingress closed") + .expect("priority ingress error"); + assert_eq!(priority.frame.header.ty, FrameType::Ping); + + let data = recv_prioritized_frame(&mut priority_rx, &mut data_rx) + .await + .unwrap() + .unwrap(); + assert_eq!(data.frame.header.ty, FrameType::Request); + reader.abort(); + } + #[test] fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() { let transient_errors = vec![ diff --git a/crates/aft/src/subc/standing.rs b/crates/aft/src/subc/standing.rs index 478c8c6c0..cc9bf7703 100644 --- a/crates/aft/src/subc/standing.rs +++ b/crates/aft/src/subc/standing.rs @@ -28,6 +28,25 @@ static LAST_STANDING_VERIFY_STRATEGY: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); const STANDING_SERVICE_QUANTUM_MS: u64 = 250; +#[derive(Clone, Debug, PartialEq, Eq)] +struct StandingReconcileKey { + storage_dir: Option, + roots: Vec, +} + +impl StandingReconcileKey { + fn from_config(config: &Config) -> Self { + Self { + storage_dir: config.storage_dir.clone(), + roots: config.index.roots.clone(), + } + } + + fn requires_reconcile(&self, config: &Config) -> bool { + self != &Self::from_config(config) + } +} + struct PendingStandingSlice { receiver: tokio::sync::oneshot::Receiver, started_at: Instant, @@ -66,6 +85,7 @@ pub(super) struct StandingActor { executor: Arc, roots: StandingRoots, observed_config: Mutex, + reconciled_config: Mutex>, /// Root ids registered solely to host unbound standing work. Session actors /// are never removed by this owner. owned_actors: Mutex>, @@ -79,6 +99,7 @@ impl StandingActor { executor, roots: StandingRoots::default(), observed_config: Mutex::new(Config::default()), + reconciled_config: Mutex::new(None), owned_actors: Mutex::new(HashMap::new()), schedule: Mutex::new(StandingScheduleState::default()), } @@ -87,8 +108,12 @@ impl StandingActor { /// Startup reconciliation is intentionally direct and empty until subc has /// observed a user-tier configuration snapshot from a successful RouteBind. pub(super) fn reconcile_at_startup(&self) { - if let Err(error) = self.roots.reconcile(&Config::default()) { - log::warn!("standing roots startup reconciliation failed: {error}"); + let config = Config::default(); + match self.roots.reconcile(&config) { + Ok(_) => { + *self.reconciled_config.lock() = Some(StandingReconcileKey::from_config(&config)) + } + Err(error) => log::warn!("standing roots startup reconciliation failed: {error}"), } } @@ -128,6 +153,7 @@ impl StandingActor { log::warn!("standing roots bind reconciliation refused: {error}"); return; } + *self.reconciled_config.lock() = Some(StandingReconcileKey::from_config(&snapshot)); let Some(session_root) = ctx .canonical_cache_root_opt() .or_else(|| snapshot.project_root.clone()) @@ -157,16 +183,28 @@ impl StandingActor { pub(super) fn tick(&self) { self.observe_config_snapshot(); let snapshot = self.observed_config.lock().clone(); - let report = match self.roots.reconcile(&snapshot) { - Ok(report) => report, - Err(error) => { - log::warn!("standing roots reconciliation refused: {error}"); - return; - } + let reconcile_key = StandingReconcileKey::from_config(&snapshot); + let entries = if self + .reconciled_config + .lock() + .as_ref() + .is_none_or(|previous| previous.requires_reconcile(&snapshot)) + { + let report = match self.roots.reconcile(&snapshot) { + Ok(report) => report, + Err(error) => { + log::warn!("standing roots reconciliation refused: {error}"); + return; + } + }; + self.retire_removed_actors(&report.removed); + *self.reconciled_config.lock() = Some(reconcile_key); + report.active_entries + } else { + self.roots.entries() }; - self.retire_removed_actors(&report.removed); - self.resume_entries_without_bound_session(&report.active_entries); - self.reconcile_schedule(report.active_entries); + self.resume_entries_without_bound_session(&entries); + self.reconcile_schedule(entries); self.drain_completed_slices(); if matches!( self.schedule @@ -659,6 +697,28 @@ mod tests { ); } + #[test] + fn unchanged_standing_config_does_not_require_reconciliation() { + let mut config = Config::default(); + config.storage_dir = Some(std::path::PathBuf::from("/tmp/aft-standing-test")); + config.index.roots.push(crate::config::IndexRootConfig { + path: "/tmp/root".to_string(), + indexes: vec![IndexKind::Search], + }); + + let key = StandingReconcileKey::from_config(&config); + assert!(!key.requires_reconcile(&config)); + + config.index.resource_policy = crate::config::IndexResourcePolicy::Performance; + assert!(!key.requires_reconcile(&config)); + + config.index.roots.push(crate::config::IndexRootConfig { + path: "/tmp/root-two".to_string(), + indexes: vec![IndexKind::Search], + }); + assert!(key.requires_reconcile(&config)); + } + #[test] fn strict_search_verification_accepts_metadata_only_drift() { let storage = tempfile::tempdir().unwrap(); diff --git a/packages/aft-bridge/src/__tests__/bridge-transport.test.ts b/packages/aft-bridge/src/__tests__/bridge-transport.test.ts index 8574222e6..6758fd843 100644 --- a/packages/aft-bridge/src/__tests__/bridge-transport.test.ts +++ b/packages/aft-bridge/src/__tests__/bridge-transport.test.ts @@ -149,8 +149,20 @@ process.stdin.on("data", (chunk) => { try { await Promise.all([ - pool.toolCall(workDir, { sessionID: "session-a" }, "read", { path: "sample.ts" }), - pool.toolCall(workDir, { sessionID: "session-b" }, "read", { path: "sample.ts" }), + pool.toolCall( + workDir, + { sessionID: "session-a" }, + "read", + { path: "sample.ts" }, + { transportTimeoutMs: 24_000 }, + ), + pool.toolCall( + workDir, + { sessionID: "session-b" }, + "read", + { path: "sample.ts" }, + { transportTimeoutMs: 24_000 }, + ), ]); const requests = readFileSync(requestsPath, "utf8") @@ -166,11 +178,20 @@ process.stdin.on("data", (chunk) => { .map((request) => ({ session_id: request.session_id, edit_slot_survives: request.edit_slot_survives, + deadline_ms_remaining: request.deadline_ms_remaining, })) .sort((left, right) => String(left.session_id).localeCompare(String(right.session_id))), ).toEqual([ - { session_id: "session-a", edit_slot_survives: true }, - { session_id: "session-b", edit_slot_survives: true }, + { + session_id: "session-a", + edit_slot_survives: true, + deadline_ms_remaining: 24_000, + }, + { + session_id: "session-b", + edit_slot_survives: true, + deadline_ms_remaining: 24_000, + }, ]); const carrierLogs = logs.filter(({ message }) => diff --git a/packages/aft-bridge/src/bridge.ts b/packages/aft-bridge/src/bridge.ts index cffb0a234..9be1ea0e7 100644 --- a/packages/aft-bridge/src/bridge.ts +++ b/packages/aft-bridge/src/bridge.ts @@ -366,6 +366,8 @@ export interface BridgeRequestOptions { abortSignal?: AbortSignal; /** Per-call transport timeout in milliseconds. Defaults to the bridge-wide timeout. */ transportTimeoutMs?: number; + /** Optional server execution budget stamped on tool_call request metadata. */ + executionDeadlineMs?: number; /** * Skip bridge-hang escalation for this request. * @@ -771,6 +773,10 @@ export class BinaryBridge implements AftProjectTransport { } const { preview, ...sendOptions } = options ?? {}; if (preview === true) params.preview = true; + const requestBudgetMs = sendOptions.executionDeadlineMs ?? sendOptions.transportTimeoutMs; + if (requestBudgetMs !== undefined && Number.isFinite(requestBudgetMs)) { + params.deadline_ms_remaining = Math.max(0, Math.floor(requestBudgetMs)); + } return (await this.send( "tool_call", params, diff --git a/packages/aft-bridge/src/subc-transport.ts b/packages/aft-bridge/src/subc-transport.ts index bdcdda24c..5aa0a6893 100644 --- a/packages/aft-bridge/src/subc-transport.ts +++ b/packages/aft-bridge/src/subc-transport.ts @@ -811,11 +811,12 @@ class SubcTransport implements AftProjectTransport { options?: ToolCallOptions, ): Promise { this.assertCurrent(); - const { preview, timeoutMs, onProgress } = this.splitOptions(options); + const { preview, timeoutMs, executionDeadlineMs, onProgress } = this.splitOptions(options); const body: Record = { name, arguments: rawArgs }; const editSlotSurvives = this.pool.getEditSlotSurvives(); if (editSlotSurvives !== undefined) body.edit_slot_survives = editSlotSurvives; if (preview === true) body.preview = true; + if (executionDeadlineMs !== undefined) body.deadline_ms_remaining = executionDeadlineMs; const reply = await this.pool.routeRequest( this.identityFor(sessionId), body, @@ -863,6 +864,7 @@ class SubcTransport implements AftProjectTransport { private splitOptions(options?: ToolCallOptions): { preview?: boolean; timeoutMs?: number; + executionDeadlineMs?: number; onProgress?: RequestOptions["onProgress"]; } { if (!options) return {}; @@ -874,8 +876,11 @@ class SubcTransport implements AftProjectTransport { // The pool default budget applies when the caller supplied neither. const timeoutMs = options.transportTimeoutMs ?? options.timeoutMs ?? this.pool.poolDefaultTimeoutMs; - const onProgress = (options as { onProgress?: RequestOptions["onProgress"] }).onProgress; - return { preview, timeoutMs, onProgress }; + const executionDeadlineMs = options.executionDeadlineMs; + const onProgress = options.onProgress + ? (body: Uint8Array) => options.onProgress?.({ kind: "stdout", text: new TextDecoder().decode(body) }) + : undefined; + return { preview, timeoutMs, executionDeadlineMs, onProgress }; } } @@ -1682,10 +1687,15 @@ export class SubcTransportPool implements AftTransportPool { } const requestTimeoutMs = remaining !== undefined ? Math.max(1, Math.floor(remaining)) : timeoutMs; + const requestedExecutionDeadline = body.deadline_ms_remaining; + const serverDeadline = + typeof requestedExecutionDeadline === "number" && Number.isFinite(requestedExecutionDeadline) + ? Math.min(remaining ?? requestedExecutionDeadline, requestedExecutionDeadline) + : remaining; const deadlineBody = - remaining === undefined || !Number.isFinite(remaining) + serverDeadline === undefined || !Number.isFinite(serverDeadline) ? body - : { ...body, deadline_ms_remaining: Math.max(0, Math.floor(remaining)) }; + : { ...body, deadline_ms_remaining: Math.max(0, Math.floor(serverDeadline)) }; const reply = await client.request(route, deadlineBody, { timeoutMs: requestTimeoutMs, onProgress, diff --git a/packages/pi-plugin/src/__tests__/_shared.test.ts b/packages/pi-plugin/src/__tests__/_shared.test.ts index a5a730079..8a4262de2 100644 --- a/packages/pi-plugin/src/__tests__/_shared.test.ts +++ b/packages/pi-plugin/src/__tests__/_shared.test.ts @@ -48,7 +48,7 @@ describe("tool shared helpers", () => { expect(requested).toEqual([projectRoot]); }); - test("callBridge propagates session id, warning client, and long-command timeout", async () => { + test("callBridge caps every synchronous transport request below Pi's hard deadline", async () => { const { bridge, calls } = makeMockBridge((_command, params) => ({ success: true, params })); const extCtx = makeExtContext(projectRoot, "pi-session-123"); @@ -58,11 +58,13 @@ describe("tool shared helpers", () => { expect(calls).toHaveLength(1); expect(calls[0].command).toBe("grep"); expect(calls[0].params).toEqual({ pattern: "needle", session_id: "pi-session-123" }); - expect(calls[0].options?.timeoutMs).toBe(60_000); + expect(calls[0].options?.timeoutMs).toBe(25_000); + expect(calls[0].options?.transportTimeoutMs).toBe(25_000); + expect(calls[0].options?.executionDeadlineMs).toBe(24_000); expect(calls[0].options?.configureWarningClient).toBe(extCtx); }); - test("callBridge keeps explicit transport options while preserving default timeout", async () => { + test("callBridge caps explicit transport options at the Pi deadline", async () => { const { bridge, calls } = makeMockBridge(() => ({ success: true })); await callBridge(bridge, "bash", { command: "sleep 60" }, makeExtContext(), { @@ -70,7 +72,8 @@ describe("tool shared helpers", () => { keepBridgeOnTimeout: true, }); - expect(calls[0].options?.transportTimeoutMs).toBe(70_000); + expect(calls[0].options?.transportTimeoutMs).toBe(25_000); + expect(calls[0].options?.executionDeadlineMs).toBe(24_000); expect(calls[0].options?.keepBridgeOnTimeout).toBe(true); expect(calls[0].options?.configureWarningClient).toBeDefined(); }); @@ -101,6 +104,29 @@ describe("tool shared helpers", () => { preview: true, }); expect(calls[0].options?.configureWarningClient).toBe(extCtx); + expect(calls[0].options?.executionDeadlineMs).toBe(24_000); + }); + + test("callToolCall emits visible progress while a tool remains pending", async () => { + let release!: () => void; + const pending = new Promise(resolve => { + release = resolve; + }); + const updates: unknown[] = []; + const { bridge } = makeMockBridge(async () => { + await pending; + return { success: true, text: "ok" }; + }); + + const call = callToolCall(bridge, "inspect", {}, makeExtContext(), { + onUpdate: update => updates.push(update), + progressIntervalMs: 5, + }); + await Bun.sleep(12); + release(); + await call; + + expect(updates.length).toBeGreaterThan(0); }); test("callBridge throws Rust error messages instead of exposing failure payloads", async () => { diff --git a/packages/pi-plugin/src/__tests__/bash.test.ts b/packages/pi-plugin/src/__tests__/bash.test.ts index ea4b50be6..b27677539 100644 --- a/packages/pi-plugin/src/__tests__/bash.test.ts +++ b/packages/pi-plugin/src/__tests__/bash.test.ts @@ -481,7 +481,7 @@ describe("bash tool adapter", () => { expect(bashCall[2].transportTimeoutMs).toBe(25_000); }); - test("wait true forwards foreground wait mode and scales transport timeout", async () => { + test("wait true remains bounded and promotes unfinished work", async () => { const tools = new Map(); const api = makeMockApi(tools); const calls: unknown[] = []; @@ -521,13 +521,13 @@ describe("bash tool adapter", () => { expect(calls.map((call) => (call as [string])[0])).toEqual(["bash"]); const bashCall = calls[0] as [string, Record, Record]; expect(bashCall[1]).toMatchObject({ - wait: true, - block_to_completion: true, + wait: false, + block_to_completion: false, timeout: 250, background: false, notify_on_completion: false, }); - expect(bashCall[2].transportTimeoutMs).toBe(10_250); + expect(bashCall[2].transportTimeoutMs).toBe(25_000); }); test("wait true rejects background and pty contradictions", async () => { @@ -656,7 +656,7 @@ describe("bash tool adapter", () => { expect(bashCall[2].transportTimeoutMs).toBe(10_050); }); - test("background disabled foreground command is block-to-completion on the server", async () => { + test("background-disabled foreground command still promotes before Pi's deadline", async () => { const tools = new Map(); const api = makeMockApi(tools); const calls: unknown[] = []; @@ -701,8 +701,8 @@ describe("bash tool adapter", () => { expect(bashParams.notify_on_completion).toBe(false); expect(bashParams.pty).toBe(false); expect(bashParams.timeout).toBe(25); - expect(bashParams.block_to_completion).toBe(true); - expect(bashCall[2].transportTimeoutMs).toBe(10_025); + expect(bashParams.block_to_completion).toBe(false); + expect(bashCall[2].transportTimeoutMs).toBe(25_000); }); test("async bash_watch registration does not add synthetic outstanding task", async () => { @@ -1177,7 +1177,7 @@ describe("bash tool adapter", () => { ]); for (const call of calls as Array<[string, Record, Record]>) { expect(call[2].keepBridgeOnTimeout).toBe(true); - expect(call[2].transportTimeoutMs).toBe(30_000); + expect(call[2].transportTimeoutMs).toBe(25_000); } }); @@ -1208,7 +1208,7 @@ describe("bash tool adapter", () => { expect(calls.some((call) => (call as [string])[0] === "bash_regex_match")).toBe(false); const callArgs = calls[0] as [string, Record, Record]; expect(callArgs[2].keepBridgeOnTimeout).toBe(true); - expect(callArgs[2].transportTimeoutMs).toBe(30_000); + expect(callArgs[2].transportTimeoutMs).toBe(25_000); } finally { await rm(join(outputPath, ".."), { recursive: true, force: true }); } diff --git a/packages/pi-plugin/src/__tests__/e2e/bash.test.ts b/packages/pi-plugin/src/__tests__/e2e/bash.test.ts index f6ccff44d..58628be19 100644 --- a/packages/pi-plugin/src/__tests__/e2e/bash.test.ts +++ b/packages/pi-plugin/src/__tests__/e2e/bash.test.ts @@ -245,7 +245,7 @@ maybeDescribe("e2e bash command (Pi adapter + bridge + Rust)", () => { expect(nonConfigureCommands(bridgeCalls)).toEqual(["bash"]); }); - test("wait true returns a long foreground command directly", async () => { + test("wait true promotes unfinished work before Pi's deadline", async () => { const { h, bash, bridgeCalls } = await pluginHarness({ experimental_bash_background: true }); const result = await withEnv({ AFT_TEST_FOREGROUND_WAIT_MS: "25" }, async () => @@ -256,12 +256,12 @@ maybeDescribe("e2e bash command (Pi adapter + bridge + Rust)", () => { }), ); - expect(result.output).toContain("waited\n"); - expect(result.output).not.toContain("promoted to background"); + expect(result.output).toContain("promoted to background"); + expect(result.details.task_id).toMatch(/^bash-[a-f0-9]{16}$/); expect(nonConfigureCommands(bridgeCalls)).toEqual(["bash"]); expect(bridgeCalls[0].params).toMatchObject({ - wait: true, - block_to_completion: true, + wait: false, + block_to_completion: false, timeout: 5_000, }); }, 30_000); diff --git a/packages/pi-plugin/src/__tests__/inspect.test.ts b/packages/pi-plugin/src/__tests__/inspect.test.ts index 2760cd04e..64ade79c3 100644 --- a/packages/pi-plugin/src/__tests__/inspect.test.ts +++ b/packages/pi-plugin/src/__tests__/inspect.test.ts @@ -225,17 +225,18 @@ describe("Pi aft_inspect surface", () => { expect(calls[0]?.command).toBe("tool_call"); }); - test("uses the default diagnostics deadline plus transport headroom", async () => { + test("caps the default diagnostics and transport deadlines below Pi's hard limit", async () => { const { api, tools } = makeMockApi(); const { bridge, calls } = makeMockBridge(() => freshTerminal()); registerInspectTool(api, makePluginContext(bridge)); await executeTool(tools.get("aft_inspect")!, {}, makeExtContext(projectRoot, "pi-session")); - expect(calls[0]?.options).toMatchObject({ transportTimeoutMs: 150_000 }); + expect(calls[0]?.params.arguments).toMatchObject({ diagnostics_timeout_ms: 24_000 }); + expect(calls[0]?.options).toMatchObject({ transportTimeoutMs: 25_000 }); }); - test("sends explicit inspect arguments with the configured diagnostics budget", async () => { + test("caps configured inspect diagnostics budgets below Pi's hard limit", async () => { const { api, tools } = makeMockApi(); const { bridge, calls } = makeMockBridge(() => freshTerminal()); registerInspectTool( @@ -253,8 +254,9 @@ describe("Pi aft_inspect surface", () => { sections: "todos", scope: ["src", "tests"], topK: 9, + diagnostics_timeout_ms: 24_000, }); - expect(calls[0]?.options).toMatchObject({ transportTimeoutMs: 210_000 }); + expect(calls[0]?.options).toMatchObject({ transportTimeoutMs: 25_000 }); expect(calls[0]?.options).not.toHaveProperty("keepBridgeOnTimeout"); }); diff --git a/packages/pi-plugin/src/tools/_shared.ts b/packages/pi-plugin/src/tools/_shared.ts index 31c15eecc..f8461c75f 100644 --- a/packages/pi-plugin/src/tools/_shared.ts +++ b/packages/pi-plugin/src/tools/_shared.ts @@ -15,7 +15,7 @@ import { isBashTransportDeadError, prepareCanonicalEditArguments, prepareCanonicalPathArguments, - timeoutForCommand, + timeoutForCommand as bridgeTimeoutForCommand, } from "@cortexkit/aft-bridge"; import type { AgentToolResult, @@ -30,6 +30,31 @@ type TextContent = { type: "text"; text: string; textSignature?: string }; type ImageContent = { type: "image"; data: string; mimeType: string }; type ContentBlock = TextContent | ImageContent; +export const PI_TOOL_TRANSPORT_TIMEOUT_MS = 25_000; +export const PI_TOOL_EXECUTION_TIMEOUT_MS = 24_000; +const DEFAULT_PROGRESS_INTERVAL_MS = 5_000; + +export interface PiToolCallOptions> extends ToolCallOptions { + onUpdate?: (update: AgentToolResult) => void; + progressIntervalMs?: number; +} + +function piTransportOptions( + command: string, + options: BridgeRequestOptions = {}, +): BridgeRequestOptions { + const requested = options.transportTimeoutMs ?? bridgeTimeoutForCommand(command); + const transportTimeoutMs = Math.min(requested ?? PI_TOOL_TRANSPORT_TIMEOUT_MS, PI_TOOL_TRANSPORT_TIMEOUT_MS); + return { + ...options, + transportTimeoutMs, + executionDeadlineMs: Math.min( + options.executionDeadlineMs ?? PI_TOOL_EXECUTION_TIMEOUT_MS, + PI_TOOL_EXECUTION_TIMEOUT_MS, + ), + }; +} + /** * Optional integer field schema for Pi tool parameters. * @@ -140,16 +165,14 @@ export async function callBridge( extCtx?: ExtensionContext, options?: BridgeRequestOptions, ): Promise> { - const timeoutMs = timeoutForCommand(command); const merged: Record = { ...params }; const sessionId = extCtx ? resolveSessionId(extCtx) : undefined; if (sessionId) { merged.session_id = sessionId; } const sendOptions = { - ...(timeoutMs !== undefined ? { timeoutMs } : {}), + ...piTransportOptions(command, options), configureWarningClient: extCtx, - ...options, }; let response: Record; try { @@ -180,12 +203,12 @@ export async function callBridge( * timeout, forwards warnings, gathers any follow-up data, and returns the raw * response plus the text summary the model will receive. */ -export async function callToolCall( +export async function callToolCall>( bridge: AftProjectTransport, name: string, rawArgs: Record = {}, extCtx?: ExtensionContext, - options?: ToolCallOptions, + options?: PiToolCallOptions, ): Promise { return callToolCallForSession( bridge, @@ -203,30 +226,37 @@ export async function callToolCall( * session manager again after an await, because another active session can * become current between preflight, preview, and apply. */ -export async function callToolCallForSession( +export async function callToolCallForSession>( bridge: AftProjectTransport, name: string, rawArgs: Record, sessionId: string | undefined, extCtx?: ExtensionContext, - options?: ToolCallOptions, + options?: PiToolCallOptions, ): Promise { - const timeoutMs = timeoutForCommand(name); const sendOptions = { - ...(timeoutMs !== undefined ? { timeoutMs } : {}), + ...piTransportOptions(name, options), configureWarningClient: extCtx, - ...options, }; + const startedAt = Date.now(); + const progressTimer = options?.onUpdate + ? setInterval(() => { + const elapsedMs = Date.now() - startedAt; + options.onUpdate?.( + textResult(`${name} is still running (${Math.max(1, Math.floor(elapsedMs / 1000))}s)`, { + tool: name, + elapsed_ms: elapsedMs, + }) as AgentToolResult, + ); + }, options.progressIntervalMs ?? DEFAULT_PROGRESS_INTERVAL_MS) + : undefined; let response: ToolCallResult; try { - response = await bridge.toolCall( - sessionId, - name, - rawArgs, - Object.keys(sendOptions).length > 0 ? sendOptions : undefined, - ); + response = await bridge.toolCall(sessionId, name, rawArgs, sendOptions); } catch (error) { throw adaptToolError(name, error); + } finally { + if (progressTimer) clearInterval(progressTimer); } ingestBgCompletions(sessionId, response.bg_completions); return response; diff --git a/packages/pi-plugin/src/tools/ast.ts b/packages/pi-plugin/src/tools/ast.ts index 27eebef7c..57104d8f6 100644 --- a/packages/pi-plugin/src/tools/ast.ts +++ b/packages/pi-plugin/src/tools/ast.ts @@ -308,7 +308,7 @@ export function registerAstTools(pi: ExtensionAPI, ctx: PluginContext, surface: _toolCallId: string, params: Static, _signal, - _onUpdate, + onUpdate, extCtx, ) { const paths = await resolveAstPaths(extCtx, params.paths); @@ -325,7 +325,7 @@ export function registerAstTools(pi: ExtensionAPI, ctx: PluginContext, surface: if (!isEmptyParam(paths)) rawArgs.paths = paths; if (!isEmptyParam(params.globs)) rawArgs.globs = params.globs; if (params.contextLines !== undefined) rawArgs.contextLines = params.contextLines; - const response = await callToolCall(bridge, "ast_search", rawArgs, extCtx); + const response = await callToolCall(bridge, "ast_search", rawArgs, extCtx, { onUpdate }); if (response.success === false) { throw new Error(response.text || response.message || "ast_search failed"); } @@ -351,7 +351,7 @@ export function registerAstTools(pi: ExtensionAPI, ctx: PluginContext, surface: _toolCallId: string, params: Static, _signal, - _onUpdate, + onUpdate, extCtx, ) { const paths = await resolveAstPaths(extCtx, params.paths); @@ -368,7 +368,7 @@ export function registerAstTools(pi: ExtensionAPI, ctx: PluginContext, surface: if (!isEmptyParam(params.globs)) rawArgs.globs = params.globs; // Coerce at the boundary: dryRun "true" must stay preview-only (coerceBoolean). rawArgs.dryRun = coerceBoolean(params.dryRun); - const response = await callToolCall(bridge, "ast_replace", rawArgs, extCtx); + const response = await callToolCall(bridge, "ast_replace", rawArgs, extCtx, { onUpdate }); if (response.success === false) { throw new Error(response.text || response.message || "ast_replace failed"); } diff --git a/packages/pi-plugin/src/tools/bash.ts b/packages/pi-plugin/src/tools/bash.ts index a5bf966fb..3495db51f 100644 --- a/packages/pi-plugin/src/tools/bash.ts +++ b/packages/pi-plugin/src/tools/bash.ts @@ -62,24 +62,18 @@ function resolveForegroundWaitMs(configured: number): number { } return configured; } -// Baseline bridge transport budget for bash-family control calls. The main -// orchestrated bash tool overrides this per request because Rust may hold the -// final response until the foreground wait window or hard-kill cap elapses. -const BASH_TRANSPORT_TIMEOUT_MS = 30_000; -const DEFAULT_HARD_TIMEOUT_MS = 30 * 60 * 1000; -// The margin gives Rust time to promote or finalize the task and deliver the -// final response after the server's foreground wait window or hard kill timeout. -const BASH_TRANSPORT_MARGIN_MS = 10_000; +// Pi hard-fails a tool call at 30 seconds. Keep every synchronous bash request +// below that boundary. Rust promotes unfinished foreground work to background. +const BASH_TRANSPORT_TIMEOUT_MS = 25_000; +const PI_BASH_FOREGROUND_LIMIT_MS = 24_000; function orchestratedTransportTimeoutMs( - blockToCompletion: boolean, - wait: boolean, - effectiveTimeout: number | undefined, + _blockToCompletion: boolean, + _wait: boolean, + _effectiveTimeout: number | undefined, foregroundWaitMs: number, ): number { - const waitBudget = - blockToCompletion || wait ? (effectiveTimeout ?? DEFAULT_HARD_TIMEOUT_MS) : foregroundWaitMs; - return waitBudget + BASH_TRANSPORT_MARGIN_MS; + return Math.min(foregroundWaitMs + 10_000, BASH_TRANSPORT_TIMEOUT_MS); } // Background task completion metadata shape (from Track D) @@ -526,10 +520,13 @@ export function registerBashTool( if (requestedWait && rawRequestedBackground) { throw new Error("wait:true cannot be used with background:true."); } - // Coerce at the boundary: stringified pty/background flags (coerceBoolean). const requestedPty = !backgroundDisabled && rawRequestedPty; const effectiveBackground = !backgroundDisabled && (rawRequestedBackground || requestedPty); - const blockToCompletion = backgroundDisabled || requestedWait; + // Pi cannot safely attach to a tool for 30 seconds. Preserve explicit + // background behavior, but promote every unfinished foreground command. + const blockToCompletion = false; + const serverWait = false; + const boundedForegroundWaitMs = Math.min(foregroundWaitMs, PI_BASH_FOREGROUND_LIMIT_MS); // Hard-kill timeout sent to the bridge. For an EXPLICIT background task a // small `timeout` is a legitimate kill cap, so honor it verbatim. For the // FOREGROUND auto-promote path a `timeout` below the foreground wait @@ -583,7 +580,7 @@ export function registerBashTool( pty_cols: ptyCols, foreground_orchestrate: true, block_to_completion: blockToCompletion, - wait: requestedWait, + wait: serverWait, sandbox: params.sandbox, ...(isPowerShell ? { shell: "powershell" } : {}), }, @@ -591,9 +588,9 @@ export function registerBashTool( { transportTimeoutMs: orchestratedTransportTimeoutMs( blockToCompletion, - requestedWait, + serverWait, effectiveTimeout, - foregroundWaitMs, + boundedForegroundWaitMs, ), onProgress: ({ text }) => { streamed += text; diff --git a/packages/pi-plugin/src/tools/conflicts.ts b/packages/pi-plugin/src/tools/conflicts.ts index 6454de146..3a7e4467e 100644 --- a/packages/pi-plugin/src/tools/conflicts.ts +++ b/packages/pi-plugin/src/tools/conflicts.ts @@ -89,7 +89,7 @@ export function registerConflictsTool(pi: ExtensionAPI, ctx: PluginContext): voi description: "Show all git merge conflicts across the repository — returns line-numbered conflict regions with context for every conflicted file in a single call.", parameters: ConflictsParams, - async execute(_toolCallId: string, params, _signal, _onUpdate, extCtx) { + async execute(_toolCallId: string, params, _signal, onUpdate, extCtx) { const bridge = bridgeFor(ctx, extCtx.cwd); const reqParams: Record = {}; const path = (params as { path?: unknown })?.path; @@ -102,7 +102,7 @@ export function registerConflictsTool(pi: ExtensionAPI, ctx: PluginContext): voi }); reqParams.path = await resolvePathArg(extCtx.cwd, path); } - const response = await callToolCall(bridge, "conflicts", reqParams, extCtx); + const response = await callToolCall(bridge, "conflicts", reqParams, extCtx, { onUpdate }); if (response.success === false) { throw new Error(response.text || response.message || "conflicts failed"); } diff --git a/packages/pi-plugin/src/tools/fs.ts b/packages/pi-plugin/src/tools/fs.ts index 2530f11e2..035c25483 100644 --- a/packages/pi-plugin/src/tools/fs.ts +++ b/packages/pi-plugin/src/tools/fs.ts @@ -157,7 +157,7 @@ export function registerFsTools(pi: ExtensionAPI, ctx: PluginContext, surface: F _toolCallId: string, params: Static, _signal, - _onUpdate, + onUpdate, extCtx, ) { // Coerce at the boundary: some hosts deliver `files` as a bare string @@ -180,17 +180,12 @@ export function registerFsTools(pi: ExtensionAPI, ctx: PluginContext, surface: F const bridge = bridgeFor(ctx, extCtx.cwd); // Single batched call so every file shares one op_id; one // `aft_safety undo` then restores the whole delete atomically. - const response = await callToolCall( - bridge, - "delete", - { - files, - // Coerce at the boundary, like `files`: a stringified "true" from the - // model must not silently drop the flag (see coerceBoolean). - recursive: coerceBoolean(params.recursive), - }, - extCtx, - ); + const response = await callToolCall(bridge, "delete", { + files, + // Coerce at the boundary, like `files`: a stringified "true" from the + // model must not silently drop the flag (see coerceBoolean). + recursive: coerceBoolean(params.recursive), + }, extCtx, { onUpdate }); if (response.success === false) { throw new Error(response.text || response.message || "delete failed"); } @@ -234,7 +229,7 @@ export function registerFsTools(pi: ExtensionAPI, ctx: PluginContext, surface: F _toolCallId: string, params: Static, _signal, - _onUpdate, + onUpdate, extCtx, ) { const filePath = await resolvePathArg(extCtx.cwd, params.path as string); @@ -247,15 +242,10 @@ export function registerFsTools(pi: ExtensionAPI, ctx: PluginContext, surface: F } const bridge = bridgeFor(ctx, extCtx.cwd); - const response = await callToolCall( - bridge, - "move", - { - filePath: params.path, - destination: params.destination, - }, - extCtx, - ); + const response = await callToolCall(bridge, "move", { + filePath: params.path, + destination: params.destination, + }, extCtx, { onUpdate }); if (response.success === false) { throw new Error(response.text || response.message || "move failed"); } diff --git a/packages/pi-plugin/src/tools/hoisted.ts b/packages/pi-plugin/src/tools/hoisted.ts index e8b0ce277..5ba9e8fef 100644 --- a/packages/pi-plugin/src/tools/hoisted.ts +++ b/packages/pi-plugin/src/tools/hoisted.ts @@ -540,7 +540,7 @@ export function registerHoistedTools( _toolCallId: string, params: Static, _signal, - _onUpdate, + onUpdate, extCtx, ) { const bridge = bridgeFor(ctx, extCtx.cwd); @@ -572,7 +572,7 @@ export function registerHoistedTools( if (limit !== undefined) rawArgs.limit = limit; const visionCapability = visionCapabilityForPiModel(extCtx); if (visionCapability !== undefined) rawArgs.vision_capability = visionCapability; - const response = await callToolCall(bridge, "read", rawArgs, extCtx); + const response = await callToolCall(bridge, "read", rawArgs, extCtx, { onUpdate }); if (response.success === false) { throw new Error(response.text || response.message || "read failed"); } @@ -628,7 +628,7 @@ export function registerHoistedTools( _toolCallId: string, params: Static, _signal, - _onUpdate, + onUpdate, extCtx, ) { const filePathArg = mutationFilePathArg(params); @@ -647,7 +647,7 @@ export function registerHoistedTools( filePath: filePathArg, content: params.content, }; - const response = await callToolCall(bridge, "write", rawArgs, extCtx); + const response = await callToolCall(bridge, "write", rawArgs, extCtx, { onUpdate }); if (response.success === false) { throw toolErrorFromResponse("write", response); } @@ -678,7 +678,7 @@ export function registerHoistedTools( _toolCallId: string, params: Static, _signal, - _onUpdate, + onUpdate, extCtx, ) { const bridge = bridgeFor(ctx, extCtx.cwd); @@ -690,6 +690,7 @@ export function registerHoistedTools( rawArgs, sessionId, extCtx, + { onUpdate }, ); if (preflight.success === false) throw toolErrorFromResponse("edit", preflight); for (const target of [ @@ -706,9 +707,12 @@ export function registerHoistedTools( } const preview = await callToolCallForSession(bridge, "edit", rawArgs, sessionId, extCtx, { preview: true, + onUpdate, }); if (preview.success === false) throw toolErrorFromResponse("edit", preview); - const response = await callToolCallForSession(bridge, "edit", rawArgs, sessionId, extCtx); + const response = await callToolCallForSession(bridge, "edit", rawArgs, sessionId, extCtx, { + onUpdate, + }); if (response.success === false) throw toolErrorFromResponse("edit", response); return buildMutationResult(response); }, @@ -735,7 +739,7 @@ export function registerHoistedTools( _toolCallId: string, params: Static, _signal, - _onUpdate, + onUpdate, extCtx, ) { const argsRecord = params as Record; @@ -766,7 +770,7 @@ export function registerHoistedTools( if (argsRecord[key] !== undefined) rawArgs[key] = argsRecord[key]; } - const response = await callToolCall(bridge, "edit", rawArgs, extCtx); + const response = await callToolCall(bridge, "edit", rawArgs, extCtx, { onUpdate }); if (response.success === false) { throw toolErrorFromResponse("edit", response); } @@ -796,7 +800,7 @@ export function registerHoistedTools( _toolCallId: string, params: Static, _signal, - _onUpdate, + onUpdate, extCtx, ) { const bridge = bridgeFor(ctx, extCtx.cwd); @@ -817,7 +821,7 @@ export function registerHoistedTools( } if (params.include) req.include = params.include; - const response = await callToolCall(bridge, "grep", req, extCtx); + const response = await callToolCall(bridge, "grep", req, extCtx, { onUpdate }); if (response.success === false) { throw new Error(response.text || response.message || "grep failed"); } diff --git a/packages/pi-plugin/src/tools/imports.ts b/packages/pi-plugin/src/tools/imports.ts index 19ccbbec3..5ad2b37af 100644 --- a/packages/pi-plugin/src/tools/imports.ts +++ b/packages/pi-plugin/src/tools/imports.ts @@ -180,7 +180,7 @@ export function registerImportTools(pi: ExtensionAPI, ctx: PluginContext): void _toolCallId: string, params: Static, _signal, - _onUpdate, + onUpdate, extCtx, ) { if ((params.op === "add" || params.op === "remove") && isEmptyParam(params.module)) { @@ -203,7 +203,7 @@ export function registerImportTools(pi: ExtensionAPI, ctx: PluginContext): void if (params.typeOnly !== undefined) rawArgs.typeOnly = params.typeOnly; if (params.validate !== undefined) rawArgs.validate = params.validate; - const response = await callToolCall(bridge, "import", rawArgs, extCtx); + const response = await callToolCall(bridge, "import", rawArgs, extCtx, { onUpdate }); if (response.success === false) { throw new Error(response.text || response.message || `${params.op} failed`); } diff --git a/packages/pi-plugin/src/tools/inspect.ts b/packages/pi-plugin/src/tools/inspect.ts index 70f2712cf..93fd9bd41 100644 --- a/packages/pi-plugin/src/tools/inspect.ts +++ b/packages/pi-plugin/src/tools/inspect.ts @@ -11,7 +11,14 @@ import type { import { type Static, Type } from "typebox"; import { resolveInspectDiagnosticsTimeoutMs } from "../config.js"; import type { PluginContext } from "../types.js"; -import { bridgeFor, callToolCall, isEmptyParam, textResult } from "./_shared.js"; +import { + bridgeFor, + callToolCall, + isEmptyParam, + PI_TOOL_EXECUTION_TIMEOUT_MS, + PI_TOOL_TRANSPORT_TIMEOUT_MS, + textResult, +} from "./_shared.js"; import { assertExternalDirectoryPermission, resolvePathArg } from "./hoisted.js"; import { asNumber, @@ -27,9 +34,9 @@ import { renderToolCall, } from "./render-helpers.js"; -// The Rust diagnostics phase may block until its configured deadline. Keep the -// transport alive long enough to receive that terminal response. -const INSPECT_TRANSPORT_HEADROOM_MS = 30_000; +// Keep Rust's phase deadline below the Pi transport deadline so inspect can +// return an honest terminal with completed phases instead of a host timeout. +const INSPECT_DIAGNOSTICS_TIMEOUT_MS = PI_TOOL_EXECUTION_TIMEOUT_MS; const InspectParams = Type.Object({ sections: Type.Optional( @@ -464,18 +471,23 @@ export function registerInspectTool(pi: ExtensionAPI, ctx: PluginContext): void "Use when: starting work on unfamiliar code, after multi-edit batches to check diagnostics, before a refactor, before review, or to verify cleanup completeness.\n\n" + "Treat `dead_code` as a hint, not proof: reachability is call-based, so symbols reached only via method dispatch or referenced only in type position may be false positives — verify before deleting.", parameters: InspectParams, - async execute(_toolCallId, params: Static, _signal, _onUpdate, extCtx) { + async execute(_toolCallId, params: Static, _signal, onUpdate, extCtx) { const bridge = bridgeFor(ctx, extCtx.cwd); const sections = normalizeStringOrArray(params.sections); const scope = await resolveAndGateScope(extCtx, ctx, normalizeStringOrArray(params.scope)); const topK = validateOptionalTopK(params.topK); - const rawArgs: Record = {}; + const rawArgs: Record = { + diagnostics_timeout_ms: Math.min( + resolveInspectDiagnosticsTimeoutMs(ctx.config), + INSPECT_DIAGNOSTICS_TIMEOUT_MS, + ), + }; if (sections !== undefined) rawArgs.sections = sections; if (scope !== undefined) rawArgs.scope = scope; if (topK !== undefined) rawArgs.topK = topK; const response = await callToolCall(bridge, "inspect", rawArgs, extCtx, { - transportTimeoutMs: - resolveInspectDiagnosticsTimeoutMs(ctx.config) + INSPECT_TRANSPORT_HEADROOM_MS, + transportTimeoutMs: PI_TOOL_TRANSPORT_TIMEOUT_MS, + onUpdate, }); const terminal = parseInspectTerminal(response); if (terminal) return textResult(renderInspectTerminal(terminal, response.text), response); diff --git a/packages/pi-plugin/src/tools/navigate.ts b/packages/pi-plugin/src/tools/navigate.ts index a51975c0b..71e992429 100644 --- a/packages/pi-plugin/src/tools/navigate.ts +++ b/packages/pi-plugin/src/tools/navigate.ts @@ -137,7 +137,7 @@ export function registerNavigateTool(pi: ExtensionAPI, ctx: PluginContext): void description: "Answer code-relationship questions from a real call graph — instead of grep + read chains. Reach for this whenever the question is about how symbols connect. Use aft_zoom with `callgraph:true` for one-level forward calls-out while reading source; use aft_callgraph only for reverse callers or multi-level traces so you do not double-fetch the same relationships. All ops require both `path` and `symbol`. Use `callers` for call sites (before renaming/signature changes), `impact` for blast radius (what breaks if a symbol changes), `call_tree` for what a function calls, `trace_to` for how execution reaches a symbol from entry points, `trace_to_symbol` for the shortest path from one symbol to another (requires `toSymbol`; if ambiguous, the error returns candidate files — retry with `toPath`), `trace_data` to follow a value across assignments/params. Markers: ~ = edge resolved by name only (may point at the wrong same-named symbol); [unresolved] = callee not resolved to a definition, so the location shown is the call site. Unmarked edges are resolved exactly. By default, unresolved external/stdlib leaf calls in call_tree are collapsed into one summary per parent; pass includeUnresolved=true to show every unresolved edge individually.", parameters: navigateParamsSchema(), - async execute(_toolCallId: string, params: NavigateArgs, _signal, _onUpdate, extCtx) { + async execute(_toolCallId: string, params: NavigateArgs, _signal, onUpdate, extCtx) { if (isEmptyParam(params.path)) { throw new Error(`op='${params.op}' requires a \`path\``); } @@ -178,7 +178,7 @@ export function registerNavigateTool(pi: ExtensionAPI, ctx: PluginContext): void rawArgs.includeTests = coerceBoolean(params.includeTests); if (!isEmptyParam(params.includeUnresolved)) rawArgs.includeUnresolved = coerceBoolean(params.includeUnresolved); - const response = await callToolCall(bridge, "callgraph", rawArgs, extCtx); + const response = await callToolCall(bridge, "callgraph", rawArgs, extCtx, { onUpdate }); if (response.success === false) { const code = typeof response.code === "string" ? response.code : ""; const text = response.text || formatBridgeErrorMessage(params.op, response, rawArgs); diff --git a/packages/pi-plugin/src/tools/reading.ts b/packages/pi-plugin/src/tools/reading.ts index d589872de..4836ca8c3 100644 --- a/packages/pi-plugin/src/tools/reading.ts +++ b/packages/pi-plugin/src/tools/reading.ts @@ -356,7 +356,7 @@ export function registerReadingTools( _toolCallId: string, params: Static, _signal, - _onUpdate, + onUpdate, extCtx, ) { const bridge = bridgeFor(ctx, extCtx.cwd); @@ -392,7 +392,7 @@ export function registerReadingTools( } } - const response = await callToolCall(bridge, "outline", rawArgs, extCtx); + const response = await callToolCall(bridge, "outline", rawArgs, extCtx, { onUpdate }); if (response.success === false) { throw new Error(response.text || response.message || "outline failed"); } @@ -432,7 +432,7 @@ export function registerReadingTools( _toolCallId: string, params: Static, _signal, - _onUpdate, + onUpdate, extCtx, ) { const bridge = bridgeFor(ctx, extCtx.cwd); @@ -506,7 +506,7 @@ export function registerReadingTools( if (contextLines !== undefined) rawArgs.contextLines = contextLines; if (wantCallgraph) rawArgs.callgraph = true; - const response = await callToolCall(bridge, "zoom", rawArgs, extCtx); + const response = await callToolCall(bridge, "zoom", rawArgs, extCtx, { onUpdate }); if (response.success === false) { throw new Error(response.text || response.message || "zoom failed"); } @@ -535,7 +535,7 @@ export function registerReadingTools( if (contextLines !== undefined) rawArgs.contextLines = contextLines; if (wantCallgraph) rawArgs.callgraph = true; - const response = await callToolCall(bridge, "zoom", rawArgs, extCtx); + const response = await callToolCall(bridge, "zoom", rawArgs, extCtx, { onUpdate }); if (response.success === false) { throw new Error(response.text || response.message || "zoom failed"); } diff --git a/packages/pi-plugin/src/tools/refactor.ts b/packages/pi-plugin/src/tools/refactor.ts index c0b8094a7..928b1850f 100644 --- a/packages/pi-plugin/src/tools/refactor.ts +++ b/packages/pi-plugin/src/tools/refactor.ts @@ -133,7 +133,7 @@ export function registerRefactorTool(pi: ExtensionAPI, ctx: PluginContext): void _toolCallId: string, params: Static, _signal, - _onUpdate, + onUpdate, extCtx, ) { // Per-op required-field validation using isEmptyParam so empty strings @@ -195,7 +195,7 @@ export function registerRefactorTool(pi: ExtensionAPI, ctx: PluginContext): void if (startLine !== undefined) rawArgs.startLine = startLine; if (endLine !== undefined) rawArgs.endLine = endLine; if (callSiteLine !== undefined) rawArgs.callSiteLine = callSiteLine; - const response = await callToolCall(bridge, "refactor", rawArgs, extCtx); + const response = await callToolCall(bridge, "refactor", rawArgs, extCtx, { onUpdate }); if (response.success === false) { throw new Error(response.text || response.message || `${params.op} failed`); } diff --git a/packages/pi-plugin/src/tools/safety.ts b/packages/pi-plugin/src/tools/safety.ts index 3d1c7f056..3918aa3e8 100644 --- a/packages/pi-plugin/src/tools/safety.ts +++ b/packages/pi-plugin/src/tools/safety.ts @@ -191,7 +191,7 @@ export function registerSafetyTool(pi: ExtensionAPI, ctx: PluginContext): void { _toolCallId: string, params: Static, _signal, - _onUpdate, + onUpdate, extCtx, ) { if (params.op === "history" && !params.path) { @@ -253,7 +253,7 @@ export function registerSafetyTool(pi: ExtensionAPI, ctx: PluginContext): void { if (filePath) rawArgs.filePath = filePath; if (params.name) rawArgs.name = params.name; if (files) rawArgs.files = files; - const response = await callToolCall(bridge, "safety", rawArgs, extCtx); + const response = await callToolCall(bridge, "safety", rawArgs, extCtx, { onUpdate }); if (response.success === false) { throw new Error(response.text || response.message || `${params.op} failed`); } From 0ba32cf9d6ce5e62f06679152e1c9517edbce275 Mon Sep 17 00:00:00 2001 From: Naadir Jeewa Date: Mon, 31 Aug 2026 16:37:58 +0100 Subject: [PATCH 05/14] docs: map branch architecture Signed-off-by: Naadir Jeewa Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- .gitignore | 1 + ARCHITECTURE.md | 27 ++- README.md | 1 + STRUCTURE.md | 9 +- docs/architecture-for-contributors.md | 272 ++++++++++++++++++++++++++ 5 files changed, 304 insertions(+), 6 deletions(-) create mode 100644 docs/architecture-for-contributors.md diff --git a/.gitignore b/.gitignore index 3305c89c4..4fe67fc77 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ vendor/ coverage/ .cache/ tmp/ +.gradle/ # Added by cargo diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 9fccc2801..9083d67fb 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -87,6 +87,14 @@ - For subc mode (when `subc.connection_file` is set): send `{name, arguments}` as a data-plane request over a tool-provider route channel opened and cached per session identity (`BindIdentity`) -- `packages/aft-bridge/src/subc-transport.ts` 3. Dispatch the request to the target command or executor. Under standalone mode, dispatch through the Rust stdin NDJSON loop. Under subc mode, process frames via the TCP loopback client loop. Local `configure` commands are satisfied locally on bind. Native plumbing tools (`bash_drain_completions`, `bash_ack_completions`, `bash_regex_match`) bypass the tool manifest check but reinject the BIND session ID to keep sessions isolated. The execution outcome is processed through the server-side text formatter (`crates/aft/src/subc_format.rs`) and a pending response finalizer seam (`crates/aft/src/response_finalize.rs`). Subc response frames contain `structuredContent` for first-party binds to re-lift the full flat response shape into `ToolCallResult` at the transport boundary, maintaining parity with standalone mode. For untrusted (MCP) binds, the server returns text-only replies (omitting `structuredContent` entirely) to prevent models like Claude Code from consuming raw JSON dumps and to save token costs. Monotonic phase traces (`PhaseTrace` and `ToolCallPhaseDurations`) track the timing/performance of subc tool calls across multiple phases (queuing, translation, execution, formatting, finalization, and egress) for slow-call diagnostics. Under subc mode, the initial attach loop retries transient connection and authentication failures (using an exponential backoff with jitter up to a 60-second budget) to recover from temporary daemon unavailability. Retry request dispatch once when a route is proven absent (receiving daemon `unknown_channel` or client `StaleRouteHandleError` before write). A cancelled route bind (e.g. Goodbye or deadline expiry) signals the configure job's cooperative `JobCancellation` handle; the running configure command checks this at phase boundaries (`configure_cancelled` and `root_commit_probe_cancelled`) to abort early and avoid building indexes or running git root commit probes for a dead route. +**Deadline and executor flow:** + +1. Bound Pi-facing synchronous calls below the host's hard 30-second limit. The Pi adapter assigns a 25-second transport budget and emits progress updates every 5 seconds without treating them as keepalives -- `packages/pi-plugin/src/tools/_shared.ts`. +2. Carry one absolute request budget through bridge route opening and request dispatch. A route-open retry does not reset the caller's budget, and caller-scoped `not_sent` expiry does not invalidate the shared client -- `packages/aft-bridge/src/bridge.ts`, `packages/aft-bridge/src/subc-transport.ts`. +3. Normalize the remaining wire budget into a local absolute deadline and submit interactive work with at most 24 seconds of Rust execution time -- `crates/aft/src/subc/mod.rs`, `crates/aft/src/executor/mod.rs`. +4. Admit jobs into bounded process-wide and per-actor queues. Interactive and maintenance jobs use separate capacity accounting. Interactive admission prefers readers, promotes deadline-pressured writers, and prunes expired jobs before dispatch. The executor rotates active actors with deficit round-robin scheduling and reserves capacity for maintenance progress -- `crates/aft/src/executor/mod.rs`. +5. Promote unfinished synchronous bash waits to background tasks before the host deadline. Return the task identity so a later call can observe completion -- `packages/pi-plugin/src/tools/bash.ts`, `crates/aft/src/commands/bash_orchestrate.rs`. + **Edit pipeline:** 1. Validate path and verify symlink safety (recursively follow components up to 40 hops to reject escaping paths), resolving relative paths against the bound project root via `AppContext::resolve_relative_path` before validation and safety keying -- `crates/aft/src/context.rs` @@ -108,8 +116,9 @@ 1. Index project files using a disk-backed, pread-based trigram search index that keeps memory overhead bounded -- `crates/aft/src/search_index.rs`. To prevent redundant disk hashing and index re-verification loops during configure bind/warmup sequences, a verification memo with a 10-minute TTL manages cache freshness checks, utilizing metadata stat checks (`VerifyStrategy::StatFirst`) when possible rather than strict content hashing. For grafted history roots, canonicalize the sorted, deduplicated set of root commits before hashing artifact keys to prevent Git traversal-order changes from triggering redundant index rebuilds. 2. Optionally index with dense embeddings (fastembed, OpenAI-compatible, Ollama, or Synapse over SubC) -- `crates/aft/src/semantic_index.rs`, `crates/aft/src/synapse_embed.rs`. Serialize cold semantic warmups by gating callgraph store building and Tier 2 diagnostics refreshes behind active cold semantic index seeds. Coalesce watcher-driven semantic re-embeds under a 15-second quiet window (`SEMANTIC_REFRESH_QUIET_WINDOW_MS`) to bundle edit bursts into a single collection pass, while masking changed files from search results until indexed to preserve query correctness. Reconfiguring semantic settings or project roots cancels superseded semantic builders while adopting matching live builders. In tests, override this quiet window via the `AFT_SEMANTIC_QUIET_WINDOW_MS` environment variable. Limit process-wide semantic refresh concurrency using the `ColdBuildLimiter` (sharing the slot budget with other heavy maintenance operations) to prevent concurrent background refreshes from overloading remote or local embedding backends -- `crates/aft/src/cold_build_limiter.rs`, `crates/aft/src/commands/configure.rs`. 3. Schedule standing-root search, semantic, and callgraph construction through the process-wide pressure-aware deficit round-robin scheduler -- `crates/aft/src/standing_scheduler.rs`, `crates/aft/src/resource_policy.rs`, `crates/aft/src/subc/standing.rs`. The scheduler admits at most the configured cold-build concurrency, rotates unfinished roots after each durable slice, and charges measured elapsed work against each root's deficit. Search, semantic, and callgraph builders persist versioned staging state and publish atomically only after the complete corpus is ready. The default `index.resource_policy = "balanced"` pauses new slices under battery saving or CPU, memory, and I/O pressure and resumes with hysteresis. `"performance"` bypasses resource admission for users who accept the power cost, but retains bounded concurrency, fair rotation, resumable checkpoints, and OS background thread priority. -4. Classify query shape (prose vs code) using the query shape parser -- `crates/aft/src/query_shape.rs`. Identify "type-concept identifier queries" (TitleCase PascalCase types combined with lowercase concepts) to trigger definition semantic priors. -5. Serve `grep` (trigram, full-text) and `aft_search` (semantic + hybrid) queries, delegating to `GrepExecutor` for accelerated path evaluation and enforcing execution safety limits (like `MAX_FALLBACK_WALK_FILES` and `FALLBACK_WALK_BUDGET`) during fallback walks when indexes are building or unavailable -- `crates/aft/src/grep_executor.rs`, `crates/aft/src/commands/grep.rs`, `crates/aft/src/commands/semantic_search.rs`. Under standalone bridge mode, interactive semantic searches support cancellable deferred polling in the main event loop. Borrow-only lexical and semantic snapshot opens bypass the cold-build limiter to prevent fresh-worktree search starvation while first searches wait cancellation-aware for a bounded loading window (2.5s). Interactive query embeddings and search artifact waits are bounded by dedicated budgets (`QueryBudget` and bounded interactive search artifact wait timeouts; `query_timeout_ms` clamped to 500..15000ms, defaulting to 3000ms) to keep interactive requests fast without affecting background build/refresh timeouts, falling back to lexical search if query embedding fails or times out. Downrank generated documentation artifacts (e.g. minified CSS/JS, maps, SVGs) in lexical and hybrid search results. For external search requests, resolve and cache external git roots, querying cached read-only search and semantic indexes from the `borrowed_index_cache` (capped at 4 concurrent entries) to avoid redundant git probes and disk parsing. +4. Keep standing-root reconciliation off the steady-state transport hot path -- `crates/aft/src/subc/standing.rs`. The standing actor caches the effective `storage_dir` and `index.roots`; an unchanged key skips SQLite access and root resolution on the 250 ms maintenance tick. Resource-policy-only changes do not trigger reconciliation. +5. Classify query shape (prose vs code) using the query shape parser -- `crates/aft/src/query_shape.rs`. Identify "type-concept identifier queries" (TitleCase PascalCase types combined with lowercase concepts) to trigger definition semantic priors. +6. Serve `grep` (trigram, full-text) and `aft_search` (semantic + hybrid) queries, delegating to `GrepExecutor` for accelerated path evaluation and enforcing execution safety limits (like `MAX_FALLBACK_WALK_FILES` and `FALLBACK_WALK_BUDGET`) during fallback walks when indexes are building or unavailable -- `crates/aft/src/grep_executor.rs`, `crates/aft/src/commands/grep.rs`, `crates/aft/src/commands/semantic_search.rs`. Under standalone bridge mode, interactive semantic searches support cancellable deferred polling in the main event loop. Borrow-only lexical and semantic snapshot opens bypass the cold-build limiter to prevent fresh-worktree search starvation while first searches wait cancellation-aware for a bounded loading window (2.5s). Interactive query embeddings and search artifact waits are bounded by dedicated budgets (`QueryBudget` and bounded interactive search artifact wait timeouts; `query_timeout_ms` clamped to 500..15000ms, defaulting to 3000ms) to keep interactive requests fast without affecting background build/refresh timeouts, falling back to lexical search if query embedding fails or times out. Downrank generated documentation artifacts (e.g. minified CSS/JS, maps, SVGs) in lexical and hybrid search results. For external search requests, resolve and cache external git roots, querying cached read-only search and semantic indexes from the `borrowed_index_cache` (capped at 4 concurrent entries) to avoid redundant git probes and disk parsing. **File read flow:** @@ -266,6 +275,16 @@ - Location: `crates/aft/src/callgraph.rs` - Pattern: Lazy workspace index with invalidation on watcher events. +**StandingScheduler / ResourcePolicy:** +- Purpose: Share bounded cold-build capacity fairly across configured standing roots while respecting laptop resource pressure. +- Location: `crates/aft/src/standing_scheduler.rs`, `crates/aft/src/resource_policy.rs`, `crates/aft/src/subc/standing.rs` +- Pattern: Process-wide deficit round-robin scheduling over durable artifact slices. The balanced policy pauses admission with hysteresis under power, CPU, memory, or I/O pressure. The performance policy bypasses pressure admission but retains bounded concurrency and fair rotation. + +**ThreadPriority:** +- Purpose: Keep maintenance CPU and I/O work below interactive and transport work. +- Location: `crates/aft/src/thread_priority.rs` +- Pattern: Cross-platform background demotion with restoration guards for Linux, macOS, and Windows maintenance workers. + **SearchIndex:** - Purpose: Provide fast trigram-based full-text search across the project. - Location: `crates/aft/src/search_index.rs` @@ -343,8 +362,8 @@ **MemoryEstimate / MemorySnapshot:** - Purpose: Track, attribute, and report process-wide and subsystem-specific memory usage. - Location: `crates/aft/src/memory.rs` -- Pattern: Diagnostic structures and OS memory allocators hook. -- Contains: Subsystem memory estimation helpers, SQLite allocator query bindings (`sqlite3_memory_used`), platform-specific resident set size (RSS) and macOS kernel physical footprint (`phys_footprint_bytes` via `proc_pid_rusage RUSAGE_INFO_V4`) queries to exclude `MADV_FREE` allocator noise, and macOS-specific pressure relief bindings (`malloc_zone_pressure_relief`) to release unused pages during idle sweeps and periodic ticks. +- Pattern: Diagnostic structures and OS memory allocator hooks. +- Contains: Subsystem memory estimation helpers, SQLite allocator query bindings (`sqlite3_memory_used`), platform-specific resident set size (RSS), and macOS kernel physical footprint (`phys_footprint_bytes` via `proc_pid_rusage RUSAGE_INFO_V4`) queries. Periodic allocator slack scans run on a detached background-priority `aft-mem-relief` thread because allocator inspection can block. Transport and stdin ticks only perform a cheap cadence check. **FleetStatusClient:** - Purpose: Publish AFT's project-scoped status segment to the fleet status-holder plane (`prefrontal-core`). diff --git a/README.md b/README.md index d8c740701..0ea929aa4 100644 --- a/README.md +++ b/README.md @@ -296,6 +296,7 @@ Adding a command means implementing it in Rust (`crates/aft/src/commands/`) and --- ## Documentation +- [Architecture for new contributors](docs/architecture-for-contributors.md): a visual guide to the request path and main code areas - [Tool reference](docs/tools.md): complete documentation for every tool - [Configuration](docs/config.md): config schema, LSP, auto-install diff --git a/STRUCTURE.md b/STRUCTURE.md index 146cb9b2f..924a63acb 100644 --- a/STRUCTURE.md +++ b/STRUCTURE.md @@ -73,10 +73,15 @@ opencode-aft/ - Key files: `crates/aft/src/lsp/manager.rs`, `crates/aft/src/lsp/client.rs`, `crates/aft/src/lsp/diagnostics.rs`, `crates/aft/src/lsp/roots.rs`, `crates/aft/src/lsp/child_registry.rs` **`crates/aft/src/executor/`:** -- Purpose: Orchestrate background maintenance, interactive tools, and job queues. -- Contains: Actor scheduler, job classes and priority queues, worker thread loop, and cooperative cancellation tokens. +- Purpose: Orchestrate bounded background maintenance and interactive tool queues across project-root actors. +- Contains: Process-wide and per-actor capacity accounting, interactive and maintenance job classes, reader-first admission, deadline-aware writer promotion, queue-deadline pruning, deficit round-robin actor scheduling, worker lanes, dispatch telemetry, and cooperative cancellation tokens. - Key files: `crates/aft/src/executor/mod.rs`, `crates/aft/src/executor/tests.rs` +**Standing-root scheduling and resource control:** +- Purpose: Share cold-build slots fairly across standing roots without making a developer laptop unresponsive. +- Contains: Process-wide deficit round-robin root scheduling, balanced and performance resource policies, pressure sampling with hysteresis, durable slice coordination, and cross-platform background thread priority control. +- Key files: `crates/aft/src/standing_scheduler.rs`, `crates/aft/src/resource_policy.rs`, `crates/aft/src/subc/standing.rs`, `crates/aft/src/thread_priority.rs` + **`crates/aft/src/bash_background/`:** - Purpose: Manage background bash tasks, PTY sessions, async pattern watches, and output compression. - Contains: Process pool, PTY runtime, watchdog thread, persistence, restart fate preservation (`FateUnknown`), process start-time liveness checks, buffer management, async pattern watches diff --git a/docs/architecture-for-contributors.md b/docs/architecture-for-contributors.md new file mode 100644 index 000000000..074a91411 --- /dev/null +++ b/docs/architecture-for-contributors.md @@ -0,0 +1,272 @@ +# ELI5: Architecture for New Contributors + +AFT gives coding agents precise tools for reading, changing, and checking code. + +This explanation follows one tool call from the agent to the Rust engine and back. + +## What it is + +**Analogy:** AFT is a workshop with reception desks, a courier, and one shared machine room. + +```mermaid +flowchart TD + A[Coding agent] -->|calls a tool| B[Harness adapter] + B -->|uses| C[Shared bridge] + C -->|sends request| D[Rust engine] + D -->|reads or changes| E[Project files] +``` + +The diagram shows the main path between an agent and a project. + +A harness adapter connects one coding agent to AFT. The Rust engine owns the real tool behavior. + +This split keeps each harness adapter small. It also gives every harness the same results. + +## How a tool call works + +```mermaid +sequenceDiagram + participant Agent + participant Adapter + participant Bridge + participant Engine + participant Project + Agent->>Adapter: Call read + Adapter->>Bridge: Send tool_call + Bridge->>Engine: Send request + Engine->>Project: Read file + Project-->>Engine: Return bytes + Engine-->>Bridge: Return result + Bridge-->>Adapter: Return result + Adapter-->>Agent: Show text +``` + +The diagram shows one `read` request and its response. + +1. The agent calls a tool registered by its harness adapter. +1. The adapter sends a common `tool_call` request through the shared bridge. +1. The bridge uses a standalone process or the Subconscious daemon transport. +1. The Rust engine validates the request and executes the command. +1. The result returns through the same layers to the agent. + +The tool protocol defines the shared request and response format. New harnesses reuse this protocol and the same engine. + +## The main parts + +```mermaid +flowchart TD + A[Harness adapters] -->|depend on| B[Shared bridge] + B -->|connects to| C[Protocol commands] + C -->|schedule work| D[Executor] + C -->|use| E[Analysis engines] + C -->|use| F[Runtime state] +``` + +The diagram shows the main code areas and their dependencies. + +| Part | Purpose | Start Here | +| --- | --- | --- | +| Harness adapters | Register tools for OpenCode and Pi. | `packages/opencode-plugin/src/index.ts`, `packages/pi-plugin/src/index.ts` | +| Shared bridge | Select transport, manage processes, and carry requests. | `packages/aft-bridge/src/transport-factory.ts`, `packages/aft-bridge/src/transport.ts` | +| Protocol commands | Translate a tool name into Rust command logic. | `crates/aft/src/run_tool_call.rs`, `crates/aft/src/commands/` | +| Executor | Give interactive work priority over maintenance work. | `crates/aft/src/executor/mod.rs` | +| Analysis engines | Parse, search, inspect, format, and change code. | `crates/aft/src/search_index.rs`, `crates/aft/src/inspect/`, `crates/aft/src/edit.rs` | +| Runtime state | Store project state, caches, watchers, and language servers. | `crates/aft/src/context.rs` | +| Subconscious transport | Serve many project roots through one daemon connection. | `crates/aft/src/subc/mod.rs` | + +## Two transport modes + +```mermaid +flowchart TD + A[Shared bridge] -->|standalone mode| B[Project process] + A -->|daemon mode| C[Subconscious route] + C -->|reaches| D[Root actor] + B -->|runs| E[Rust commands] + D -->|runs| E +``` + +The diagram shows both paths to the same Rust command layer. + +Standalone mode keeps one AFT process for a project root. It uses newline-delimited JSON over standard input and output. + +Daemon mode sends requests through Subconscious routes. A root actor owns the state for each active project root. + +Both modes use the same command handlers. A feature should behave the same in both modes. + +## How the executor protects tool calls + +The daemon can serve many project roots at the same time. Each root has an actor. An actor keeps the state and queues for one project root. + +```mermaid +flowchart LR + A[Incoming jobs] --> B{Job class} + B -->|Interactive| C[Bounded interactive queue] + B -->|Maintenance| D[Bounded maintenance queue] + C --> E[Reader-first admission] + E --> F[Deadline-aware writer promotion] + D --> G[Reserved maintenance capacity] + F --> H[Deficit round-robin actor scheduler] + G --> H + H --> I[Worker lanes] +``` + +The diagram shows how the executor classifies and schedules work. + +The executor separates interactive jobs from maintenance jobs. Reads, writes, and language-server requests are interactive jobs. Index refreshes and watcher drains are maintenance jobs. + +Each queue has a fixed capacity. The executor rejects excess work with a structured backpressure error. It does not allow an unbounded queue to consume memory. + +The interactive queue normally admits readers before writers. A waiting writer moves forward as its deadline approaches. This rule prevents reader traffic from starving a mutation. + +The actor scheduler uses deficit round-robin scheduling. This scheduling method gives each active root a service allowance. A root rotates to the queue tail after it uses that allowance. + +The executor removes expired jobs before dispatch. It returns a deadline error without starting obsolete work. Cancellation and every other removal path release the exact queue capacity that the job used. + +Start with `crates/aft/src/executor/mod.rs`. Read `crates/aft/src/executor/tests.rs` for the queue contracts. + +## How a request keeps one time budget + +Pi stops a synchronous tool call after 30 seconds. AFT keeps its own deadlines below that host limit. + +```mermaid +sequenceDiagram + participant Pi + participant Adapter + participant Bridge + participant Subc as Subconscious + participant Executor + Pi->>Adapter: Start tool call + Adapter->>Adapter: Set 25 second transport budget + Adapter->>Bridge: Send absolute budget + Bridge->>Subc: Open route and send within same budget + Subc->>Executor: Submit with 24 second execution deadline + Executor-->>Subc: Result or deadline error + Subc-->>Bridge: Return result + Bridge-->>Adapter: Return before host timeout + Adapter-->>Pi: Show result +``` + +The diagram shows one budget across all transport stages. + +The Pi adapter allows at most 25 seconds for synchronous transport. The Rust engine receives at most 24 seconds for interactive execution. The difference leaves time to encode and return the result before the host stops the call. + +The bridge does not restart the budget when it opens a route. Route discovery, request dispatch, queue waiting, execution, and response delivery consume the same absolute budget. + +The Pi adapter sends a progress update every five seconds while a tool runs. A progress update informs the user. It does not extend the host deadline. + +A long `bash` request becomes a background task before the synchronous budget expires. The agent can inspect the task later. AFT does not lose the running process when the foreground wait ends. + +Start with `packages/pi-plugin/src/tools/_shared.ts`, `packages/aft-bridge/src/subc-transport.ts`, and `crates/aft/src/subc/mod.rs`. + +## How standing roots share index capacity + +A standing root is a project that AFT indexes before an agent asks for it. Many standing roots must share a small amount of background capacity. + +```mermaid +flowchart TD + A[Standing roots] --> B[Process-wide deficit round-robin scheduler] + B --> C{Resource policy admits work?} + C -->|No| D[Pause with reason] + C -->|Yes| E[Acquire cold-build permit] + E --> F[Run one durable slice] + F --> G{Artifact complete?} + G -->|No| H[Save cursor and rotate root] + H --> B + G -->|Yes| I[Publish complete artifact atomically] +``` + +The diagram shows fair, resumable index construction. + +The scheduler runs one bounded slice for a root. It charges the measured slice cost to that root. An unfinished root then rotates to the queue tail. + +Search, semantic, and call-graph builders store durable cursors. A later slice resumes from the cursor. A restart or a scheduler rotation does not discard completed stages. + +Readers continue to use the old published artifact during a rebuild. The builder publishes the replacement only after the full corpus is complete. + +The `balanced` resource policy pauses new slices during battery saving or CPU, memory, and input/output pressure. It uses hysteresis so short signal changes do not repeatedly stop and start work. The `performance` policy ignores these pressure signals. Both policies keep the concurrency limit and fair rotation. + +Start with `crates/aft/src/standing_scheduler.rs`, `crates/aft/src/resource_policy.rs`, and `crates/aft/src/subc/standing.rs`. + +## How the daemon stays responsive + +The transport thread must answer control traffic even when background indexing uses the machine. + +```mermaid +flowchart LR + A[Subconscious frames] --> B{Frame channel} + B -->|Channel 0 control| C[Priority control queue] + B -->|Tool data| D[Data queue] + C --> E[Biased receive loop] + D --> E + E --> F[Transport handling] + G[Maintenance work] --> H[Background CPU and I/O priority] + I[Allocator slack scan] --> J[Detached aft-mem-relief thread] + K[250 ms maintenance tick] --> L{Standing config changed?} + L -->|No| M[Skip root reconciliation] + L -->|Yes| N[Reconcile standing roots] +``` + +The diagram shows the safeguards around the transport loop. + +Channel 0 carries heartbeats and health checks. The daemon keeps control frames in a separate queue. A biased receive operation processes a ready control frame before buffered data frames. + +Maintenance workers use background CPU and input/output priority. This rule reduces competition with transport and interactive worker threads. + +Allocator inspection can pause inside the system allocator. AFT runs this scan on a detached `aft-mem-relief` thread. The transport tick only checks whether the scan is due. + +Standing-root reconciliation can open SQLite and resolve paths. The standing actor caches a reconciliation key made from `storage_dir` and `index.roots`. An unchanged key makes the 250 millisecond maintenance tick skip that work. A change to only `index.resource_policy` does not require root reconciliation. + +Start with `crates/aft/src/subc/mod.rs`, `crates/aft/src/thread_priority.rs`, `crates/aft/src/memory.rs`, and `crates/aft/src/subc/standing.rs`. + +## Where new work belongs + +Use the narrowest existing layer that owns the behavior. + +| Change | Primary Location | +| --- | --- | +| Add a new agent tool | `crates/aft/src/commands/` and both harness tool directories | +| Change request translation | `crates/aft/src/subc_translate.rs` | +| Change agent-facing result text | `crates/aft/src/subc_format.rs` | +| Change transport behavior | `packages/aft-bridge/src/` | +| Change queue priority or admission | `crates/aft/src/executor/` | +| Change search behavior | `crates/aft/src/search_index.rs` or `crates/aft/src/grep_executor.rs` | +| Change health analysis | `crates/aft/src/inspect/` | +| Change shared runtime state | `crates/aft/src/context.rs` | + +A command usually needs a Rust handler and one definition in each harness adapter. + +Keep protocol dispatch thin. Put reusable behavior in a shared Rust engine outside `commands/`. + +## Why it matters + +The architecture separates the harness integration from the code analysis. Harness details cannot change the core behavior. + +The persistent Rust engine keeps the indexes and the project state ready. The executor protects interactive requests from maintenance work. + +## Words + +| Word | What It Means | +| --- | --- | +| Absolute budget | One deadline that all transport and execution stages share. | +| Adapter | TypeScript code that connects a coding harness to AFT. | +| Bridge | Shared TypeScript code that carries requests to the Rust engine. | +| Command handler | Rust code that executes one protocol command. | +| Deficit round-robin | A fair scheduler that gives each active item a service allowance. | +| Durable slice | A bounded unit of index work that records a cursor for later resumption. | +| Executor | The scheduler that orders interactive and maintenance work. | +| Harness | A coding-agent host such as OpenCode or Pi. | +| Hysteresis | Separate pause and resume thresholds that prevent rapid state changes. | +| Newline-delimited JSON | One JSON message on each text line. | +| Reconciliation key | The configuration inputs that determine whether standing roots need reconciliation. | +| Root actor | The daemon state and queue for one project root. | +| Standing root | A configured project that AFT indexes before an interactive request. | +| Subconscious | The daemon transport that routes messages between modules. | +| Tool protocol | The shared request and response format used by each AFT transport. | +| Transport | The connection that carries a request and its response. | + +## Where to look next + +- [Architecture](../ARCHITECTURE.md) gives the complete system layer and data-flow map. +- [Codebase Structure](../STRUCTURE.md) maps each capability to its source directory. +- [Tool Reference](tools.md) describes every agent-facing tool. +- [Configuration Reference](config.md) describes runtime configuration. From 1c79774539f6c196f26977668e399437f4a02dd1 Mon Sep 17 00:00:00 2001 From: Naadir Jeewa Date: Mon, 31 Aug 2026 17:18:56 +0100 Subject: [PATCH 06/14] fix(index): reset changed root cursors Signed-off-by: Naadir Jeewa --- crates/aft/src/subc/standing.rs | 48 ++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/crates/aft/src/subc/standing.rs b/crates/aft/src/subc/standing.rs index cc9bf7703..0eddd6c11 100644 --- a/crates/aft/src/subc/standing.rs +++ b/crates/aft/src/subc/standing.rs @@ -224,17 +224,34 @@ impl StandingActor { .map(|entry| entry.literal_path.clone()) .collect::>(); schedule.queue.reconcile(keys.iter().cloned()); + Self::reconcile_kind_cursors(&mut schedule, &entries); schedule.entries = entries .into_iter() .map(|entry| (entry.literal_path.clone(), entry)) .collect(); - schedule.next_kind.retain(|key, _| keys.contains(key)); - for key in keys { - schedule.next_kind.entry(key).or_insert(0); - } Self::publish_schedule_telemetry(&schedule); } + fn reconcile_kind_cursors(schedule: &mut StandingScheduleState, entries: &[StandingRootEntry]) { + schedule + .next_kind + .retain(|key, _| entries.iter().any(|entry| entry.literal_path == *key)); + for entry in entries { + let selection_changed = schedule + .entries + .get(&entry.literal_path) + .is_some_and(|previous| previous.indexes != entry.indexes); + if selection_changed { + schedule.next_kind.insert(entry.literal_path.clone(), 0); + } else { + schedule + .next_kind + .entry(entry.literal_path.clone()) + .or_insert(0); + } + } + } + fn publish_schedule_telemetry(schedule: &StandingScheduleState) { crate::standing_scheduler::publish_telemetry( crate::standing_scheduler::StandingSchedulerTelemetry { @@ -719,6 +736,29 @@ mod tests { assert!(key.requires_reconcile(&config)); } + #[test] + fn index_selection_change_resets_kind_cursor() { + let mut schedule = StandingScheduleState::default(); + let mut entry = StandingRootEntry { + literal_path: "/tmp/root".to_string(), + resolved_target: std::path::PathBuf::from("/tmp/root"), + resolved_git_toplevel: None, + scoped_relative_path: None, + artifact_key: "root".to_string(), + indexes: vec![IndexKind::Search, IndexKind::Semantic], + config_order: 0, + }; + schedule + .entries + .insert(entry.literal_path.clone(), entry.clone()); + schedule.next_kind.insert(entry.literal_path.clone(), 1); + + entry.indexes = vec![IndexKind::Search]; + StandingActor::reconcile_kind_cursors(&mut schedule, std::slice::from_ref(&entry)); + + assert_eq!(schedule.next_kind.get(&entry.literal_path), Some(&0)); + } + #[test] fn strict_search_verification_accepts_metadata_only_drift() { let storage = tempfile::tempdir().unwrap(); From 208d75760bf3885099bcc191814d679d062b56b2 Mon Sep 17 00:00:00 2001 From: Naadir Jeewa Date: Mon, 31 Aug 2026 17:47:23 +0100 Subject: [PATCH 07/14] fix(index): address standing-root review defects - Retire removed standing actors on session-bind reconciliation - Publish admission decisions to scheduler telemetry each tick - Fence standing semantic publication through WriterLease and admission epoch - Build standing callgraph slices via resume_cold_build_slice_with_lease - Admit balanced indexing when portable resource signals are unavailable - Treat unreadable battery capacity as Unknown power, not Battery - Preserve arrival order when pruning elapsed deadline jobs - Derive DRR reconciliation membership from one HashSet - Pass the bounded foreground window to Pi kill-timeout coherence Assisted-by: Claude Opus 4.5 Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/executor/mod.rs | 49 ++------ crates/aft/src/resource_policy.rs | 43 ++++--- crates/aft/src/standing_scheduler.rs | 7 +- crates/aft/src/subc/standing.rs | 163 +++++++++++++++++++++++---- packages/pi-plugin/src/tools/bash.ts | 2 +- 5 files changed, 176 insertions(+), 88 deletions(-) diff --git a/crates/aft/src/executor/mod.rs b/crates/aft/src/executor/mod.rs index 67bd21724..17c4e0e59 100644 --- a/crates/aft/src/executor/mod.rs +++ b/crates/aft/src/executor/mod.rs @@ -2054,45 +2054,20 @@ impl ClassQueues { /// survivor order in both the ladder and the lane queues. fn prune_elapsed(&mut self, now: Instant) -> Vec { let mut drained = Vec::new(); - for lane in [ - Lane::PureRead, - Lane::SerialLspStatus, - Lane::HeavyInit, - Lane::Mutating, - Lane::MaintenanceCommit, - ] { - let queue = self.queue_mut(lane); - let mut index = 0; - while index < queue.len() { - if queue[index] - .deadline - .is_some_and(|deadline| now >= deadline) - { - if let Some(job) = queue.remove(index) { - drained.push(job); - } - } else { - index += 1; - } - } - } - if drained.is_empty() { - return drained; - } - // Rebuild the ladder from the survivors; each lane queue is FIFO, so - // the per-lane counts are the ladder multiplicities. - self.order.clear(); - for lane in [ - Lane::PureRead, - Lane::SerialLspStatus, - Lane::HeavyInit, - Lane::Mutating, - Lane::MaintenanceCommit, - ] { - for _ in 0..self.queue(lane).len() { - self.order.push_back(lane); + let mut surviving_order = VecDeque::with_capacity(self.order.len()); + while let Some(lane) = self.order.pop_front() { + let Some(job) = self.queue_mut(lane).pop_front() else { + debug_assert!(false, "order ladder references an empty lane"); + continue; + }; + if job.deadline.is_some_and(|deadline| now >= deadline) { + drained.push(job); + } else { + self.queue_mut(lane).push_back(job); + surviving_order.push_back(lane); } } + self.order = surviving_order; drained } diff --git a/crates/aft/src/resource_policy.rs b/crates/aft/src/resource_policy.rs index 351f48fc0..29184fbf3 100644 --- a/crates/aft/src/resource_policy.rs +++ b/crates/aft/src/resource_policy.rs @@ -85,26 +85,16 @@ impl ResourceAdmissionGate { } fn pause_reason(snapshot: ResourceSnapshot) -> Option { - match snapshot.power { - PowerState::BatterySaving => return Some(PauseReason::BatterySaving), - PowerState::Unknown => return Some(PauseReason::UnknownPower), - PowerState::External | PowerState::Battery | PowerState::NoBattery => {} + if snapshot.power == PowerState::BatterySaving { + return Some(PauseReason::BatterySaving); } - match snapshot.memory_pressure { - SignalState::High => return Some(PauseReason::MemoryPressure), - SignalState::Unknown => return Some(PauseReason::UnknownMemoryPressure), - SignalState::Healthy => {} + if snapshot.memory_pressure == SignalState::High { + return Some(PauseReason::MemoryPressure); } - match snapshot.io_pressure { - SignalState::High => return Some(PauseReason::IoPressure), - SignalState::Unknown => return Some(PauseReason::UnknownIoPressure), - SignalState::Healthy => {} - } - match snapshot.cpu_pressure { - SignalState::High => Some(PauseReason::CpuPressure), - SignalState::Unknown => Some(PauseReason::UnknownCpuPressure), - SignalState::Healthy => None, + if snapshot.io_pressure == SignalState::High { + return Some(PauseReason::IoPressure); } + (snapshot.cpu_pressure == SignalState::High).then_some(PauseReason::CpuPressure) } #[cfg(target_os = "linux")] @@ -163,8 +153,10 @@ fn sample_linux_power_at(root: &std::path::Path) -> PowerState { PowerState::NoBattery } else if battery_capacity.is_some_and(|capacity| capacity <= 10) { PowerState::BatterySaving - } else { + } else if battery_capacity.is_some() { PowerState::Battery + } else { + PowerState::Unknown } } @@ -244,13 +236,17 @@ mod tests { } #[test] - fn balanced_reports_unknown_portable_pressure_conservatively() { + fn balanced_admits_when_portable_signals_are_unavailable() { let mut gate = ResourceAdmissionGate::default(); - let mut unknown = healthy(); - unknown.io_pressure = SignalState::Unknown; + let unknown = ResourceSnapshot { + power: PowerState::Unknown, + cpu_pressure: SignalState::Unknown, + memory_pressure: SignalState::Unknown, + io_pressure: SignalState::Unknown, + }; assert_eq!( gate.observe(IndexResourcePolicy::Balanced, unknown), - AdmissionDecision::Paused(PauseReason::UnknownIoPressure) + AdmissionDecision::Admit ); } @@ -370,5 +366,8 @@ mod tests { std::fs::write(battery.join("capacity"), "5\n").unwrap(); assert_eq!(sample_linux_power_at(dir.path()), PowerState::BatterySaving); + + std::fs::remove_file(battery.join("capacity")).unwrap(); + assert_eq!(sample_linux_power_at(dir.path()), PowerState::Unknown); } } diff --git a/crates/aft/src/standing_scheduler.rs b/crates/aft/src/standing_scheduler.rs index b41f67b1a..1a6d761dc 100644 --- a/crates/aft/src/standing_scheduler.rs +++ b/crates/aft/src/standing_scheduler.rs @@ -28,9 +28,10 @@ where I: IntoIterator, { let keys = keys.into_iter().collect::>(); - self.queue.retain(|key| keys.contains(key)); - self.deficits.retain(|key, _| keys.contains(key)); - self.in_flight.retain(|key| keys.contains(key)); + let active = keys.iter().cloned().collect::>(); + self.queue.retain(|key| active.contains(key)); + self.deficits.retain(|key, _| active.contains(key)); + self.in_flight.retain(|key| active.contains(key)); for key in keys { if !self.deficits.contains_key(&key) { self.deficits.insert(key.clone(), 0); diff --git a/crates/aft/src/subc/standing.rs b/crates/aft/src/subc/standing.rs index 0eddd6c11..0f3678487 100644 --- a/crates/aft/src/subc/standing.rs +++ b/crates/aft/src/subc/standing.rs @@ -149,10 +149,14 @@ impl StandingActor { pub(super) fn begin_session_bind(&self, ctx: &AppContext) { let snapshot = ctx.config(); let snapshot = snapshot.as_ref().clone(); - if let Err(error) = self.roots.reconcile(&snapshot) { - log::warn!("standing roots bind reconciliation refused: {error}"); - return; - } + let report = match self.roots.reconcile(&snapshot) { + Ok(report) => report, + Err(error) => { + log::warn!("standing roots bind reconciliation refused: {error}"); + return; + } + }; + self.retire_removed_actors(&report.removed); *self.reconciled_config.lock() = Some(StandingReconcileKey::from_config(&snapshot)); let Some(session_root) = ctx .canonical_cache_root_opt() @@ -206,13 +210,21 @@ impl StandingActor { self.resume_entries_without_bound_session(&entries); self.reconcile_schedule(entries); self.drain_completed_slices(); - if matches!( - self.schedule - .lock() - .resource_gate - .observe(snapshot.index.resource_policy, sample_resources(),), - AdmissionDecision::Admit - ) { + let decision = self + .schedule + .lock() + .resource_gate + .observe(snapshot.index.resource_policy, sample_resources()); + { + let mut schedule = self.schedule.lock(); + schedule.resource_policy = snapshot.index.resource_policy.as_str().to_string(); + schedule.pause_reason = match decision { + AdmissionDecision::Admit => None, + AdmissionDecision::Paused(reason) => Some(format!("{reason:?}").to_lowercase()), + }; + Self::publish_schedule_telemetry(&schedule); + } + if decision == AdmissionDecision::Admit { self.dispatch_ready_slices(&snapshot); } } @@ -463,9 +475,21 @@ impl StandingActor { permit.admission_epoch, ) } else if kind == IndexKind::Semantic { - build_missing_semantic_after_strict_check(ctx, &entry) + build_missing_semantic_after_strict_check( + ctx, + &roots, + &entry, + &admission, + permit.admission_epoch, + ) } else { - (false, true) + build_missing_callgraph_after_strict_check( + ctx, + &roots, + &entry, + &admission, + permit.admission_epoch, + ) }; if kind_complete { if let Err(error) = roots.record_strict_verification(&literal_path, kind) { @@ -662,11 +686,18 @@ fn build_missing_search_after_strict_check( fn build_missing_semantic_after_strict_check( ctx: &AppContext, + roots: &StandingRoots, entry: &StandingRootEntry, + admission: &crate::standing_roots::StandingBuildAdmission, + permit_epoch: u64, ) -> (bool, bool) { + if admission.cancellation_requested() || crate::executor::current_job_cancelled() { + return (false, true); + } let config = ctx.config(); let semantic_config = config.semantic.clone(); let storage_dir = config.storage_dir.clone(); + let configure_generation = ctx.configure_generation(); drop(config); let Some(storage_dir) = storage_dir else { return (false, true); @@ -685,23 +716,105 @@ fn build_missing_semantic_after_strict_check( return (false, true); } }; - match crate::semantic_index::SemanticIndex::resume_cold_build_slice( - &entry.resolved_target, - &files, - &mut model, - &semantic_config, - &storage_dir, + let cache_dir = storage_dir.join("semantic").join(&entry.artifact_key); + let lease = match crate::root_cache::WriterLease::acquire_shared( + crate::root_cache::RootCacheDomain::Index, + &cache_dir, &entry.artifact_key, + &entry.resolved_target, ) { - Ok(crate::semantic_index::SemanticBuildSliceOutcome::Complete) => (true, false), - Ok(crate::semantic_index::SemanticBuildSliceOutcome::Yielded) => (false, true), - Err(error) => { - log::warn!("standing semantic slice failed: {}", error); - (false, true) - } + Ok(Some(lease)) => lease, + Ok(None) | Err(_) => return (false, true), + }; + let outcome = roots + .publish_if_current( + &entry.literal_path, + admission.publication, + &lease, + || true, + || { + permit_epoch == admission.publication.admission_epoch + && ctx.configure_generation() == configure_generation + && !admission.cancellation_requested() + }, + || { + crate::semantic_index::SemanticIndex::resume_cold_build_slice( + &entry.resolved_target, + &files, + &mut model, + &semantic_config, + &storage_dir, + &entry.artifact_key, + ) + .is_ok() + }, + ) + .ok() + .flatten(); + match outcome { + Some(true) => (true, false), + Some(false) | None => (false, true), } } +fn build_missing_callgraph_after_strict_check( + ctx: &AppContext, + roots: &StandingRoots, + entry: &StandingRootEntry, + admission: &crate::standing_roots::StandingBuildAdmission, + permit_epoch: u64, +) -> (bool, bool) { + if admission.cancellation_requested() || crate::executor::current_job_cancelled() { + return (false, true); + } + let config = ctx.config(); + let storage_dir = config.storage_dir.clone(); + let configure_generation = ctx.configure_generation(); + drop(config); + let Some(storage_dir) = storage_dir else { + return (false, true); + }; + let files = crate::callgraph::walk_project_files(&entry.resolved_target).collect::>(); + let cache_dir = storage_dir.join("callgraph").join(&entry.artifact_key); + let lease = match crate::root_cache::WriterLease::acquire_shared( + crate::root_cache::RootCacheDomain::Callgraph, + &cache_dir, + &entry.artifact_key, + &entry.resolved_target, + ) { + Ok(Some(lease)) => lease, + Ok(None) | Err(_) => return (false, true), + }; + let outcome = roots + .publish_if_current( + &entry.literal_path, + admission.publication, + &lease, + || true, + || { + permit_epoch == admission.publication.admission_epoch + && ctx.configure_generation() == configure_generation + && !admission.cancellation_requested() + }, + || { + matches!( + crate::callgraph_store::CallGraphStore::resume_cold_build_slice_with_lease( + cache_dir.clone(), + entry.resolved_target.clone(), + &files, + 0, + ), + Ok(crate::callgraph_store::ColdBuildSlice::Complete { .. }) + ) + }, + ) + .ok() + .flatten(); + match outcome { + Some(true) => (true, false), + Some(false) | None => (false, true), + } +} #[cfg(test)] mod tests { use super::*; diff --git a/packages/pi-plugin/src/tools/bash.ts b/packages/pi-plugin/src/tools/bash.ts index 3495db51f..076b15c22 100644 --- a/packages/pi-plugin/src/tools/bash.ts +++ b/packages/pi-plugin/src/tools/bash.ts @@ -538,7 +538,7 @@ export function registerBashTool( const effectiveTimeout = requestedWait || effectiveBackground || backgroundDisabled ? timeout - : resolveBashKillTimeout(timeout, foregroundWaitMs); + : resolveBashKillTimeout(timeout, boundedForegroundWaitMs); // Build spawn context for potential hook modification let spawnContext: BashSpawnContext = { From 1d9bba19bde16d04b7752a73de670d2944b5c924 Mon Sep 17 00:00:00 2001 From: Naadir Jeewa Date: Mon, 31 Aug 2026 18:49:07 +0100 Subject: [PATCH 08/14] fix(index): address standing-root review defects across executors and publishers - bash: pass client-bounded foreground wait (foreground_wait_ms) from Pi through translate into select_foreground_wait_window_ms so the Rust promotion window cannot exceed the client request budget - subc bash: promote still-running commands and retire wait-mode bookkeeping when a poll phase is rejected after the request deadline instead of leaking registrations and returning a bare deadline error - callgraph: hoist method-dispatch edge insertion out of the publication transaction into durable per-slice chunks keyed by staged cursor keys so slice-budget exhaustion can no longer roll back dispatch work and wedge the build in an infinite resume loop - configure: acquire the root-keyed Index WriterLease around session search and semantic persistence so standing-root and session builds cannot interleave publications over the same cache directories - search: mid-build slices resume from the staged manifest without rewalking and refingerprinting the whole corpus; the publication slice revalidates the corpus and re-hashes staged contents so same-size mtime-preserving edits restart the build instead of publishing stale postings; spill file ids are allocated over included entries only to match publication compaction Signed-off-by: Naadir Sheriffdean Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/callgraph_store/mod.rs | 101 +++++----- crates/aft/src/commands/bash_orchestrate.rs | 31 ++- crates/aft/src/commands/configure.rs | 62 +++++- crates/aft/src/search_index.rs | 201 +++++++++++++++++--- crates/aft/src/subc/bash.rs | 58 ++++++ crates/aft/src/subc_translate.rs | 11 ++ packages/pi-plugin/src/tools/bash.ts | 1 + 7 files changed, 382 insertions(+), 83 deletions(-) diff --git a/crates/aft/src/callgraph_store/mod.rs b/crates/aft/src/callgraph_store/mod.rs index 5c2ad9bca..67215203b 100644 --- a/crates/aft/src/callgraph_store/mod.rs +++ b/crates/aft/src/callgraph_store/mod.rs @@ -76,6 +76,8 @@ const COLD_BUILD_EXTRACT_BATCH_BYTES: u64 = 32 * 1024 * 1024; const COLD_BUILD_RESOLVE_WINDOW: usize = 20_000; const STAGED_COMMITTED_EXTRACTED_BYTES: &str = "committed_extracted_bytes"; const STAGED_RESOLVE_CURSOR: &str = "resolve_cursor"; +const STAGED_DISPATCH_CURSOR_FILE: &str = "dispatch_cursor_file"; +const STAGED_DISPATCH_COMPLETED_FILES: &str = "dispatch_completed_files"; const STAGED_BUILD_PHASE: &str = "staged_build_phase"; const STAGED_CORPUS_FINGERPRINT: &str = "staged_corpus_fingerprint"; @@ -3903,12 +3905,65 @@ impl CallGraphStore { } ensure_cold_build_current("resolution", resolved_refs, total_refs)?; + // Dispatch edges are inserted per-slice so a slice budget exhaustion + // cannot roll back work inside the publication transaction. Each + // chunk commits its own cursor before the final publish. + if staged_build_phase(&conn)?.as_deref() != Some("dispatching") { + let total_changes_before = conn.total_changes(); + let tx = conn.transaction()?; + set_staged_build_phase(&tx, "dispatching")?; + set_staged_string(&tx, STAGED_DISPATCH_CURSOR_FILE, "")?; + set_staged_u64(&tx, STAGED_DISPATCH_COMPLETED_FILES, 0)?; + tx.commit()?; + self.record_commit(total_changes_before, &conn); + ensure_cold_build_current("method-dispatch-phase", 1, 1)?; + } + let total_dispatch_files = query_count( + &conn, + "SELECT COUNT(*) FROM (SELECT DISTINCT caller_file FROM refs)", + )? as usize; + let mut dispatch_completed = staged_u64(&conn, STAGED_DISPATCH_COMPLETED_FILES)? as usize; + let mut dispatch_after_file = + staged_string(&conn, STAGED_DISPATCH_CURSOR_FILE)?.unwrap_or_default(); + loop { + let caller_files = { + let mut statement = conn.prepare( + "SELECT DISTINCT caller_file + FROM refs + WHERE caller_file > ?1 + ORDER BY caller_file + LIMIT ?2", + )?; + let rows = statement + .query_map(params![dispatch_after_file, batch_files as i64], |row| { + row.get::<_, String>(0) + })?; + rows.collect::, _>>()? + }; + let Some(last_file) = caller_files.last().cloned() else { + break; + }; + self.verify_writer_lease()?; + let total_changes_before = conn.total_changes(); + let tx = conn.transaction()?; + insert_method_dispatch_edges(&tx, &self.project_root, Some(&caller_files))?; + set_staged_string(&tx, STAGED_DISPATCH_CURSOR_FILE, &last_file)?; + set_staged_u64( + &tx, + STAGED_DISPATCH_COMPLETED_FILES, + (dispatch_completed + caller_files.len()) as u64, + )?; + tx.commit()?; + self.record_commit(total_changes_before, &conn); + dispatch_after_file = last_file; + dispatch_completed += caller_files.len(); + ensure_cold_build_current("method-dispatch", dispatch_completed, total_dispatch_files)?; + } + note_cold_build_phase("publication"); self.verify_writer_lease()?; let total_changes_before = conn.total_changes(); let tx = conn.transaction()?; - let _supplemental_edge_count = - insert_method_dispatch_edges_chunked(&tx, &self.project_root, batch_files)?; set_meta_ready(&tx, true)?; set_staged_build_phase(&tx, "ready")?; tx.execute("DELETE FROM staging_file_inventory", [])?; @@ -11151,48 +11206,6 @@ fn insert_method_dispatch_edges( Ok(inserted) } -fn insert_method_dispatch_edges_chunked( - tx: &Transaction<'_>, - project_root: &Path, - chunk_size: usize, -) -> Result { - let total_files = query_count( - tx, - "SELECT COUNT(*) FROM (SELECT DISTINCT caller_file FROM refs)", - )? as usize; - let mut completed_files = 0usize; - ensure_cold_build_current("method-dispatch", completed_files, total_files)?; - let mut inserted = 0usize; - let mut after_file = String::new(); - loop { - let caller_files = { - let mut statement = tx.prepare( - "SELECT DISTINCT caller_file - FROM refs - WHERE caller_file > ?1 - ORDER BY caller_file - LIMIT ?2", - )?; - let rows = statement - .query_map(params![after_file, chunk_size.max(1) as i64], |row| { - row.get::<_, String>(0) - })?; - rows.collect::, _>>()? - }; - let Some(last_file) = caller_files.last().cloned() else { - break; - }; - inserted += insert_method_dispatch_edges(tx, project_root, Some(&caller_files))?; - after_file = last_file; - completed_files = completed_files - .saturating_add(caller_files.len()) - .min(total_files); - ensure_cold_build_current("method-dispatch", completed_files, total_files)?; - } - ensure_cold_build_current("method-dispatch", completed_files, total_files)?; - Ok(inserted) -} - fn insert_method_dispatch_edge( tx: &Transaction<'_>, reference: &NameMatchRef, diff --git a/crates/aft/src/commands/bash_orchestrate.rs b/crates/aft/src/commands/bash_orchestrate.rs index 873187b95..a3fd23ee4 100644 --- a/crates/aft/src/commands/bash_orchestrate.rs +++ b/crates/aft/src/commands/bash_orchestrate.rs @@ -22,6 +22,9 @@ struct BashOrchestrateParams { background: bool, pty: bool, timeout: Option, + /// Client-bounded foreground wait window (ms), sent by Pi so the + /// promotion window cannot exceed the client's request budget. + foreground_wait_ms: Option, } /// Port of `packages/aft-bridge/src/bash-format.ts` `formatForegroundResult` (lines 8-25). @@ -145,6 +148,7 @@ pub fn build_bash_outcome( ctx.config().foreground_wait_window_ms, params.timeout, params.wait, + params.foreground_wait_ms, ); let deadline = Instant::now() + Duration::from_millis(wait_window_ms); let block_to_completion = params.block_to_completion || params.wait; @@ -411,11 +415,12 @@ pub(crate) fn select_foreground_wait_window_ms( configured: u64, timeout: Option, wait: bool, + request_wait_ms: Option, ) -> u64 { if wait { timeout.unwrap_or(DEFAULT_FOREGROUND_WAIT_TIMEOUT_MS) } else { - resolve_foreground_wait_window_ms(configured) + resolve_foreground_wait_window_ms(configured).min(request_wait_ms.unwrap_or(u64::MAX)) } } @@ -573,15 +578,35 @@ mod tests { #[test] fn select_foreground_wait_window_uses_timeout_budget_for_wait_true() { assert_eq!( - select_foreground_wait_window_ms(8_000, Some(250), true), + select_foreground_wait_window_ms(8_000, Some(250), true, None), 250 ); assert_eq!( - select_foreground_wait_window_ms(8_000, None, true), + select_foreground_wait_window_ms(8_000, None, true, None), DEFAULT_FOREGROUND_WAIT_TIMEOUT_MS ); } + #[test] + fn select_foreground_wait_window_caps_to_client_request_budget() { + // Pi's 24s request budget must cap the promotion window even when + // the server config allows a longer foreground wait. + assert_eq!( + select_foreground_wait_window_ms(60_000, Some(120_000), false, Some(24_000)), + 24_000 + ); + // The server-configured window still wins when it is shorter. + assert_eq!( + select_foreground_wait_window_ms(8_000, Some(120_000), false, Some(24_000)), + 8_000 + ); + // Without a client bound the configured window is used unchanged. + assert_eq!( + select_foreground_wait_window_ms(60_000, None, false, None), + 60_000 + ); + } + #[test] fn foreground_result_format_matches_typescript_order() { let snapshot = snapshot( diff --git a/crates/aft/src/commands/configure.rs b/crates/aft/src/commands/configure.rs index f619be9b3..bb4b158ec 100644 --- a/crates/aft/src/commands/configure.rs +++ b/crates/aft/src/commands/configure.rs @@ -3193,6 +3193,7 @@ fn schedule_artifact_loads( let symbol_cache = ctx.symbol_cache(); let symbol_storage = storage_dir.clone(); let symbol_project_key = project_key.clone(); + let search_project_key = project_key.clone(); let is_worktree_bridge_for_search = is_worktree_bridge; let search_loads_shared_artifacts_read_only = is_worktree_bridge_for_search || ctx.shared_artifacts_read_only(); @@ -3403,11 +3404,35 @@ fn schedule_artifact_loads( // Borrow-only / worktree roots never persist, overlay or not. // `write_to_disk` also fail-closes via artifact_write_allowed. if generation_current() && !is_worktree_bridge_for_search && persist_to_disk { - let published = - search_persist_epoch_flag.run_if_current(search_persist_epoch, || { - let head = index.stored_git_head().map(str::to_owned); - index.write_to_disk(&cache_dir, head.as_deref()) - }); + // Standing-root search builders publish into this same + // cache dir under a root-keyed WriterLease; hold the same + let search_lease = crate::root_cache::WriterLease::acquire_shared( + crate::root_cache::RootCacheDomain::Index, + &cache_dir, + &search_project_key, + &root_for_search, + ); + let published = match search_lease { + Ok(Some(_lease)) => { + search_persist_epoch_flag.run_if_current(search_persist_epoch, || { + let head = index.stored_git_head().map(str::to_owned); + index.write_to_disk(&cache_dir, head.as_deref()) + }) + } + Ok(None) => { + slog_info!( + "search index persistence skipped: writer lease not allowed" + ); + None + } + Err(error) => { + slog_warn!( + "search index persistence writer lease unavailable: {}", + error + ); + None + } + }; persistence_succeeded = published == Some(true); if published.is_none() { slog_info!( @@ -4075,6 +4100,33 @@ fn schedule_artifact_loads( ); return false; } + // A standing-root builder publishes into the same + // semantic cache dir through a root-keyed WriterLease. + // Acquire the same lease here so session-driven and + // standing publications cannot interleave over one + // directory. + let semantic_cache_dir = dir.join("semantic").join(&semantic_project_key); + let _standing_lease = match crate::root_cache::WriterLease::acquire_shared( + crate::root_cache::RootCacheDomain::Index, + &semantic_cache_dir, + &semantic_project_key, + &root_clone, + ) { + Ok(Some(lease)) => Some(lease), + Ok(None) => { + slog_info!( + "semantic index persistence skipped for {reason}: writer lease not allowed" + ); + return false; + } + Err(error) => { + slog_warn!( + "semantic index persistence writer lease unavailable for {reason}: {}", + error + ); + return false; + } + }; let Ok(_cache_lock) = SemanticIndexLock::acquire(dir, &semantic_project_key, &root_clone) else { diff --git a/crates/aft/src/search_index.rs b/crates/aft/src/search_index.rs index 50346a617..f29e5e6d4 100644 --- a/crates/aft/src/search_index.rs +++ b/crates/aft/src/search_index.rs @@ -938,49 +938,91 @@ impl SearchIndex { root: &Path, max_file_size: u64, cache_dir: &Path, + ) -> std::io::Result { + Self::resume_cold_build_slice_sized(root, max_file_size, cache_dir, SEARCH_SLICE_FILES) + } + + pub(crate) fn resume_cold_build_slice_sized( + root: &Path, + max_file_size: u64, + cache_dir: &Path, + slice_size: usize, ) -> std::io::Result { fs::create_dir_all(cache_dir)?; let staging_dir = cache_dir.join(SEARCH_STAGING_DIR); let manifest_path = staging_dir.join(SEARCH_STAGING_MANIFEST); let canonical_root = fs::canonicalize(root).unwrap_or_else(|_| root.to_path_buf()); - let ignore_fingerprint = ignore_rules_fingerprint(&canonical_root); - let filters = PathFilters::default(); - let paths = walk_project_files(&canonical_root, &filters); - let corpus_fingerprint = - search_corpus_fingerprint(&canonical_root, &ignore_fingerprint, max_file_size, &paths); - let mut manifest = load_search_staging_manifest(&manifest_path) - .filter(|manifest| { - manifest.version == SEARCH_STAGING_VERSION - && manifest.corpus_fingerprint == corpus_fingerprint - && manifest.canonical_root == canonical_root - && manifest.ignore_fingerprint == ignore_fingerprint - && manifest.max_file_size == max_file_size - && manifest.paths == paths - && manifest.cursor <= manifest.paths.len() - && manifest.files.len() == manifest.cursor - }) - .unwrap_or_else(|| { - let _ = fs::remove_dir_all(&staging_dir); - SearchStagingManifest { - version: SEARCH_STAGING_VERSION, - corpus_fingerprint: corpus_fingerprint.clone(), - canonical_root: canonical_root.clone(), - ignore_fingerprint: ignore_fingerprint.clone(), + let existing = load_search_staging_manifest(&manifest_path).filter(|manifest| { + manifest.version == SEARCH_STAGING_VERSION + && manifest.canonical_root == canonical_root + && manifest.cursor <= manifest.paths.len() + && manifest.files.len() == manifest.cursor + }); + // Fresh start (or structurally invalid staging): walk and fingerprint + // the full corpus once to seed the manifest. Mid-build slices skip the + // walk and continue from the staged manifest; the publication slice + // re-validates the corpus before publishing. + let manifest_valid_mid_build = existing + .as_ref() + .is_some_and(|manifest| manifest.cursor > 0 && manifest.cursor < manifest.paths.len()); + let mut manifest = match existing { + Some(manifest) if manifest_valid_mid_build => manifest, + _ => { + let ignore_fingerprint = ignore_rules_fingerprint(&canonical_root); + let filters = PathFilters::default(); + let paths = walk_project_files(&canonical_root, &filters); + let corpus_fingerprint = search_corpus_fingerprint( + &canonical_root, + &ignore_fingerprint, max_file_size, - paths: paths.clone(), - cursor: 0, - spill_seq: 0, - files: Vec::new(), + &paths, + ); + let compatible = load_search_staging_manifest(&manifest_path).filter(|manifest| { + manifest.version == SEARCH_STAGING_VERSION + && manifest.corpus_fingerprint == corpus_fingerprint + && manifest.canonical_root == canonical_root + && manifest.ignore_fingerprint == ignore_fingerprint + && manifest.max_file_size == max_file_size + && manifest.paths == paths + && manifest.cursor <= manifest.paths.len() + && manifest.files.len() == manifest.cursor + }); + match compatible { + Some(manifest) => manifest, + None => { + let _ = fs::remove_dir_all(&staging_dir); + SearchStagingManifest { + version: SEARCH_STAGING_VERSION, + corpus_fingerprint, + canonical_root: canonical_root.clone(), + ignore_fingerprint, + max_file_size, + paths, + cursor: 0, + spill_seq: 0, + files: Vec::new(), + } + } } - }); + } + }; fs::create_dir_all(&staging_dir)?; if manifest.cursor < manifest.paths.len() { - let end = (manifest.cursor + SEARCH_SLICE_FILES).min(manifest.paths.len()); + let end = (manifest.cursor + slice_size).min(manifest.paths.len()); let mut block = Vec::new(); for path in &manifest.paths[manifest.cursor..end] { - let file_id = u32::try_from(manifest.files.len()) - .map_err(|_| std::io::Error::other("too many files to index"))?; + // Publication compacts ids over included entries only, so + // spill ids must be allocated the same way: excluded entries + // never consume an id. + let file_id = u32::try_from( + manifest + .files + .iter() + .filter(|staged| staged.included) + .count(), + ) + .map_err(|_| std::io::Error::other("too many files to index"))?; match prepare_search_path(path, max_file_size) { PreparedSearchPath::Indexed(file) => { let trigram_count = @@ -1034,6 +1076,25 @@ impl SearchIndex { return Ok(SearchBuildSliceOutcome::Yielded); } + // Publication slice: the corpus fingerprint only covers path, size, + // and mtime, so an edit that preserves both would silently publish + // stale postings. Re-hash every staged included file and restart the + // build when any content no longer matches its staged hash. + let ignore_fingerprint = ignore_rules_fingerprint(&canonical_root); + let filters = PathFilters::default(); + let paths = walk_project_files(&canonical_root, &filters); + let corpus_fingerprint = + search_corpus_fingerprint(&canonical_root, &ignore_fingerprint, max_file_size, &paths); + let corpus_unchanged = manifest.corpus_fingerprint == corpus_fingerprint + && manifest.ignore_fingerprint == ignore_fingerprint + && manifest.max_file_size == max_file_size + && manifest.paths == paths + && staged_contents_match_disk(&manifest.files); + if !corpus_unchanged { + let _ = fs::remove_dir_all(&staging_dir); + return Ok(SearchBuildSliceOutcome::Yielded); + } + let mut files = Vec::with_capacity(manifest.files.len()); let mut path_to_id = HashMap::with_capacity(manifest.files.len()); let mut unindexed_files = HashSet::new(); @@ -3421,6 +3482,19 @@ fn build_postings_header_bytes(plan: &CacheWritePlan) -> std::io::Result .into_inner()) } +/// Re-hashes every staged included file against disk. The corpus +/// fingerprint covers only path, size, and mtime, so an edit that preserves +/// both can otherwise resume stale postings. +fn staged_contents_match_disk(staged_files: &[SearchStagingFile]) -> bool { + staged_files + .iter() + .filter(|staged| staged.included && staged.indexed) + .all(|staged| match fs::read(&staged.path) { + Ok(content) => cache_freshness::hash_bytes(&content).as_bytes() == &staged.content_hash, + Err(_) => false, + }) +} + fn build_lookup_section_bytes(lookup_entries: &[LookupEntry]) -> std::io::Result> { let mut writer = BufWriter::new(Cursor::new(Vec::new())); let entry_count = u32::try_from(lookup_entries.len()) @@ -8526,6 +8600,71 @@ mod tests { ); } + #[test] + fn resumable_search_spill_ids_skip_excluded_entries() { + let dir = tempfile::tempdir().expect("create temp dir"); + let project = dir.path().join("project"); + let cache = dir.path().join("cache"); + fs::create_dir_all(&project).expect("create project"); + // First entry is a binary (unindexed but included — it participates + // in the published file table without postings), remaining are + // indexable text. Skipped entries never consume an id; the critical + // invariant is that grep works on the text files regardless. + fs::write(project.join("000_blob.bin"), [0u8; 64]).expect("write binary"); + for index in 0..5 { + fs::write( + project.join(format!("file_{index:03}.rs")), + format!("pub fn marker_{index}() {{ println!(\"id_marker_{index}\"); }}\n"), + ) + .expect("write source"); + } + while SearchIndex::resume_cold_build_slice(&project, DEFAULT_MAX_FILE_SIZE, &cache) + .expect("resume slice") + == SearchBuildSliceOutcome::Yielded + {} + let published = SearchIndex::read_from_disk(&cache, &project).expect("published index"); + let result = published.grep("id_marker_4", true, &[], &[], &project, 10); + } + + fn resumable_search_restarts_when_content_changes_without_metadata_change() { + let dir = tempfile::tempdir().expect("create temp dir"); + let project = dir.path().join("project"); + let cache = dir.path().join("cache"); + fs::create_dir_all(&project).expect("create project"); + // Two files so the build spans two slices of one file each: the + // first slice stages stale content under a valid corpus fingerprint. + fs::write(project.join("a.rs"), "fn stale() {}\n").expect("write a"); + fs::write(project.join("b.rs"), "fn stable() {}\n").expect("write b"); + assert_eq!( + SearchIndex::resume_cold_build_slice_sized(&project, DEFAULT_MAX_FILE_SIZE, &cache, 1,) + .expect("first slice"), + SearchBuildSliceOutcome::Yielded + ); + // Rewrite file a with identical size and restored mtime: the corpus + // fingerprint stays identical, but staged content is now stale. + let metadata = fs::metadata(project.join("a.rs")).expect("stat"); + fs::write(project.join("a.rs"), "fn fresh() {}\n").expect("rewrite a"); + let mtime = filetime::FileTime::from_last_modification_time(&metadata); + filetime::set_file_mtime(project.join("a.rs"), mtime).expect("restore mtime"); + // The next slice reaches publication and must detect the stale + // staged content, reset staging, and yield for a fresh build. + assert_eq!( + SearchIndex::resume_cold_build_slice_sized(&project, DEFAULT_MAX_FILE_SIZE, &cache, 1,) + .expect("publication slice"), + SearchBuildSliceOutcome::Yielded + ); + // Draining the fresh build publishes content that matches disk. + while SearchIndex::resume_cold_build_slice(&project, DEFAULT_MAX_FILE_SIZE, &cache) + .expect("drain slices") + == SearchBuildSliceOutcome::Yielded + {} + let published = SearchIndex::read_from_disk(&cache, &project).expect("published index"); + let fresh = published.grep("fresh", true, &[], &[], &project, 10); + assert_eq!(fresh.total_matches, 1); + let stale = published.grep("stale", true, &[], &[], &project, 10); + assert_eq!(stale.total_matches, 0); + } + #[test] fn resumable_search_build_yields_then_matches_monolithic_results() { let dir = tempfile::tempdir().expect("create temp dir"); diff --git a/crates/aft/src/subc/bash.rs b/crates/aft/src/subc/bash.rs index 0f94143f3..404f97a38 100644 --- a/crates/aft/src/subc/bash.rs +++ b/crates/aft/src/subc/bash.rs @@ -46,6 +46,9 @@ struct BashTranslatedSettings { wait: bool, block_to_completion: bool, timeout: Option, + /// Client-bounded foreground wait window (ms). Pi sends this so the + /// Rust-side promotion window cannot exceed the client's request budget. + foreground_wait_ms: Option, } enum BashSpawnControl { @@ -82,6 +85,10 @@ fn bash_settings_from_translated(args: &serde_json::Map) -> BashT .and_then(Value::as_bool) .unwrap_or(false), timeout: args.get("timeout").and_then(Value::as_u64), + foreground_wait_ms: args + .get("foreground_wait_ms") + .and_then(Value::as_u64) + .filter(|ms| *ms > 0), } } @@ -418,6 +425,7 @@ pub(super) fn submit_deferred_bash( ctx.config().foreground_wait_window_ms, settings.timeout, settings.wait, + settings.foreground_wait_ms, ); let deadline = Instant::now() + Duration::from_millis(wait_window_ms); let storage_dir = @@ -751,6 +759,49 @@ async fn run_deferred_bash_wait( .await; match poll_control_rx.await.unwrap_or(BashPollControl::Done) { BashPollControl::Done => { + // If the executor rejected the poll after the request + // deadline elapsed, the command is still running with + // wait-mode bookkeeping registered. Promote it to a + // background task instead of leaking the registration + // and delivering a bare deadline error. + if poll_was_deadline_rejected(&poll_response) { + if let Some(ctx) = executor.actor_context(&root) { + if detach_on_user_message { + ctx.bash_background() + .end_wait_mode_session(&session_id, &task_id); + } else { + ctx.bash_background() + .unregister_foreground_task(&session_id, &task_id); + } + } + let result = submit_bash_promote( + &executor, + root.clone(), + request_id.clone(), + task_id.clone(), + session_id.clone(), + timeout, + wait_window_ms, + format_context.clone(), + None, + ) + .await; + let fatal = response_is_fatal_panic(&result.response); + send_bash_deferred_completion( + &completion_tx, + &metrics, + route, + corr, + flags, + ver, + root, + request_id, + Some(result), + fatal, + ) + .await; + break; + } let text = poll_text_rx.await.unwrap_or_else(|_| { crate::subc_format::format_response_with_context( "bash", @@ -880,6 +931,13 @@ async fn submit_bash_promote( ToolCallResult { text, response } } +/// True when a poll-phase executor response proves the request budget +/// elapsed before the poll could be admitted (queued rejection or prune). +/// The command may still be running, so the caller must promote it. +fn poll_was_deadline_rejected(response: &Response) -> bool { + response.data.get("code").and_then(Value::as_str) == Some("request_deadline_exceeded") +} + #[allow(clippy::too_many_arguments)] async fn send_bash_deferred_completion( completion_tx: &mpsc::Sender, diff --git a/crates/aft/src/subc_translate.rs b/crates/aft/src/subc_translate.rs index 9f4870e86..ae1208d98 100644 --- a/crates/aft/src/subc_translate.rs +++ b/crates/aft/src/subc_translate.rs @@ -1184,6 +1184,17 @@ fn translate_bash(args: Value, project_root: &Path) -> Result Date: Mon, 31 Aug 2026 19:08:28 +0100 Subject: [PATCH 09/14] fix(review): P3 cleanup across fixtures, executor tests, thread priority, docs - callgraph fixture copy handles nested directories (recursive helper) - restore pub mod symbol_diff dropped from lib.rs (776 lines dead code) - executor test covers global interactive backpressure with second actor - retirement test proves no process-capacity leak via new-actor admission - deadline-elapsed test syncs on job start (removes submit/dispatch race) - halfway deadline case labeled for true 12s-budget halfway point - thread_priority: macOS warning carries rc; Windows formats last_os_error (was std::process::id()); BackgroundGuard restores prior state not default - docs: index.resource_policy marked USER-only in config snippet + prose Signed-off-by: Naadir Sheriffdean Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- packages/opencode-plugin/src/tools/_shared.ts | 23 ++++++++++++------- .../pi-plugin/src/__tests__/_shared.test.ts | 2 +- packages/pi-plugin/src/tools/_shared.ts | 13 +++++++---- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/packages/opencode-plugin/src/tools/_shared.ts b/packages/opencode-plugin/src/tools/_shared.ts index 11a2ab379..5df53697b 100644 --- a/packages/opencode-plugin/src/tools/_shared.ts +++ b/packages/opencode-plugin/src/tools/_shared.ts @@ -21,6 +21,7 @@ import * as os from "node:os"; import * as path from "node:path"; import type { AftProjectTransport, + AftTransportOptions, BridgeRequestOptions, ToolCallOptions, ToolCallResult, @@ -240,7 +241,7 @@ export async function callBridge( runtime: ToolRuntime, command: string, params: Record = {}, - options?: BridgeRequestOptions, + options?: AftTransportOptions, ): Promise> { // Resolve the session's stored project directory once on first call — // OpenCode sets `runtime.directory = process.cwd()` even for resumed @@ -254,11 +255,17 @@ export async function callBridge( if (runtime.sessionID) { merged.session_id = runtime.sessionID; } - const timeoutMs = timeoutForCommand(command); + // One canonical budget field: caller `transportTimeoutMs` (if any) wins, + // otherwise the command budget. A caller-provided `timeoutMs` is consumed + // as an override input but never re-emitted — bridge.ts resolves + // `transportTimeoutMs ?? timeoutMs`, so emitting both fields lets the + // legacy alias silently shadow the winner. + const { timeoutMs: callerTimeoutMs, ...restOptions } = options ?? {}; + const timeoutMs = restOptions.transportTimeoutMs ?? callerTimeoutMs ?? timeoutForCommand(command); const sendOptions = { - ...(timeoutMs !== undefined ? { timeoutMs } : {}), + ...(timeoutMs !== undefined ? { transportTimeoutMs: timeoutMs } : {}), configureWarningClient: ctx.client, - ...options, + ...restOptions, }; markBridgeStart(); let response: Awaited>; @@ -300,13 +307,13 @@ export async function callToolCall( await getSessionDirectory(ctx.client, runtime.sessionID, runtime.directory); } - const timeoutMs = timeoutForCommand(name); + const { timeoutMs: callerTimeoutMs, ...restOptions } = options ?? {}; + const timeoutMs = restOptions.transportTimeoutMs ?? callerTimeoutMs ?? timeoutForCommand(name); const sendOptions = { - ...(timeoutMs !== undefined ? { timeoutMs } : {}), + ...(timeoutMs !== undefined ? { transportTimeoutMs: timeoutMs } : {}), configureWarningClient: ctx.client, - ...options, + ...restOptions, }; - markBridgeStart(); let response: Awaited>; try { response = await bridgeFor(ctx, runtime).toolCall( diff --git a/packages/pi-plugin/src/__tests__/_shared.test.ts b/packages/pi-plugin/src/__tests__/_shared.test.ts index 8a4262de2..bc67e0454 100644 --- a/packages/pi-plugin/src/__tests__/_shared.test.ts +++ b/packages/pi-plugin/src/__tests__/_shared.test.ts @@ -58,7 +58,7 @@ describe("tool shared helpers", () => { expect(calls).toHaveLength(1); expect(calls[0].command).toBe("grep"); expect(calls[0].params).toEqual({ pattern: "needle", session_id: "pi-session-123" }); - expect(calls[0].options?.timeoutMs).toBe(25_000); + expect(calls[0].options?.timeoutMs).toBeUndefined(); expect(calls[0].options?.transportTimeoutMs).toBe(25_000); expect(calls[0].options?.executionDeadlineMs).toBe(24_000); expect(calls[0].options?.configureWarningClient).toBe(extCtx); diff --git a/packages/pi-plugin/src/tools/_shared.ts b/packages/pi-plugin/src/tools/_shared.ts index f8461c75f..f9034f60f 100644 --- a/packages/pi-plugin/src/tools/_shared.ts +++ b/packages/pi-plugin/src/tools/_shared.ts @@ -43,13 +43,18 @@ function piTransportOptions( command: string, options: BridgeRequestOptions = {}, ): BridgeRequestOptions { - const requested = options.transportTimeoutMs ?? bridgeTimeoutForCommand(command); - const transportTimeoutMs = Math.min(requested ?? PI_TOOL_TRANSPORT_TIMEOUT_MS, PI_TOOL_TRANSPORT_TIMEOUT_MS); + const { timeoutMs: callerTimeoutMs, ...rest } = options; + const requested = + rest.transportTimeoutMs ?? callerTimeoutMs ?? bridgeTimeoutForCommand(command); + const transportTimeoutMs = Math.min( + requested ?? PI_TOOL_TRANSPORT_TIMEOUT_MS, + PI_TOOL_TRANSPORT_TIMEOUT_MS, + ); return { - ...options, + ...rest, transportTimeoutMs, executionDeadlineMs: Math.min( - options.executionDeadlineMs ?? PI_TOOL_EXECUTION_TIMEOUT_MS, + rest.executionDeadlineMs ?? PI_TOOL_EXECUTION_TIMEOUT_MS, PI_TOOL_EXECUTION_TIMEOUT_MS, ), }; From 6b1840f3235098b9bb9d2d538481af7fab47d901 Mon Sep 17 00:00:00 2001 From: Naadir Jeewa Date: Mon, 31 Aug 2026 19:23:24 +0100 Subject: [PATCH 10/14] fix(index): complete scheduler review corrections Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/checkpoint.rs | 74 ++------ crates/aft/src/cold_build_limiter.rs | 56 ++++++ crates/aft/src/standing_scheduler.rs | 44 ++++- crates/aft/src/subc/mod.rs | 168 +++++++++++++++++- .../aft/tests/integration/subc_storm_test.rs | 14 +- 5 files changed, 280 insertions(+), 76 deletions(-) diff --git a/crates/aft/src/checkpoint.rs b/crates/aft/src/checkpoint.rs index 2638af757..58a890908 100644 --- a/crates/aft/src/checkpoint.rs +++ b/crates/aft/src/checkpoint.rs @@ -727,16 +727,10 @@ impl CheckpointStore { if let Some(checkpoints_dir) = self.durable_checkpoints_dir() { sweep_expired_durable_checkpoints(&checkpoints_dir, now); } - if let Some(checkpoints_root) = self.lock_path.parent().and_then(Path::parent) { - // Fail-closed guard: the sweep root is DERIVED from lock_path depth, and a - // caller with a nonstandard (shallower) lock path would resolve this to an - // unrelated directory - in tests, the OS temp root itself, where removing - // "empty scope dirs" deletes other processes' freshly created temp dirs. - // Only a directory actually named `checkpoints` is a legitimate sweep root. - if checkpoints_root.file_name() == Some(std::ffi::OsStr::new("checkpoints")) { - sweep_empty_scope_dirs(checkpoints_root); - } - } + // Empty scope dirs are intentionally retained: another process may + // hold a scope-dir lock between create_dir_all and its first + // checkpoint write, so sweeping "empty" dirs deletes concurrently + // in-use scopes. Real data expires via sweep_expired_durable_checkpoints. Ok(()) } @@ -1475,31 +1469,10 @@ fn rollback_created_dirs(dirs: &[PathBuf]) -> bool { ok } -/// Remove one project scope directory without ever deleting its contents. -/// Another process may acquire the lock or create a file between inspection and -/// removal, so every failure is intentionally ignored. -fn remove_empty_scope_dir(scope_dir: &Path) { - let _ = fs::remove_dir(scope_dir); -} - -/// Sweep only the direct children of the checkpoints root. Scope directories -/// contain lockfiles, not durable checkpoint data, so an empty one is safe to -/// remove while a non-empty one is left untouched by `remove_dir`. -fn sweep_empty_scope_dirs(checkpoints_root: &Path) { - let entries = match fs::read_dir(checkpoints_root) { - Ok(entries) => entries, - Err(_) => return, - }; - - for entry in entries.flatten() { - let Ok(file_type) = entry.file_type() else { - continue; - }; - if file_type.is_dir() { - remove_empty_scope_dir(&entry.path()); - } - } -} +// The empty-scope-dir sweep (remove_empty_scope_dir + sweep_empty_scope_dirs) +// was removed: it raced concurrent scope creation between create_dir_all and +// lock acquisition. Empty scope dirs are harmless and expire naturally with +// their session data via sweep_expired_durable_checkpoints. fn current_timestamp() -> u64 { std::time::SystemTime::now() @@ -1980,16 +1953,16 @@ mod tests { assert_eq!(store.list(DEFAULT_SESSION_ID).unwrap()[0].name, "recent"); assert!(store.list("other").unwrap().is_empty()); } - #[test] - fn cleanup_sweeps_empty_scope_dirs_but_keeps_live_lock_scope() { + fn cleanup_retains_empty_scope_dirs_and_live_lock_scope() { + // The empty-scope sweep was removed because it raced concurrent + // scope creation (create_dir_all vs lock acquisition). Cleanup must + // now leave scope directories in place regardless of emptiness. let dir = tempfile::tempdir().unwrap(); let checkpoints_root = dir.path().join("checkpoints"); let empty_a = checkpoints_root.join("empty-a"); - let empty_b = checkpoints_root.join("empty-b"); let live_scope = checkpoints_root.join("live-scope"); fs::create_dir_all(&empty_a).unwrap(); - fs::create_dir_all(&empty_b).unwrap(); fs::create_dir_all(&live_scope).unwrap(); fs::write(live_scope.join("checkpoint.lock"), "live lock").unwrap(); @@ -1999,32 +1972,11 @@ mod tests { let mut store = CheckpointStore::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT); store.cleanup(); - assert!(!empty_a.exists()); - assert!(!empty_b.exists()); + assert!(empty_a.is_dir(), "empty scope dirs must be retained"); assert!(live_scope.is_dir()); assert!(live_scope.join("checkpoint.lock").is_file()); } - #[test] - fn cleanup_ignores_non_empty_scope_dir_removal_failure() { - let dir = tempfile::tempdir().unwrap(); - let checkpoints_root = dir.path().join("checkpoints"); - let scope_dir = checkpoints_root.join("racing-scope"); - fs::create_dir_all(&scope_dir).unwrap(); - // Model the post-race state where a concurrent lock acquisition adds - // this file after the root readdir but before remove_dir. - fs::write(scope_dir.join("checkpoint.lock"), "lock appeared").unwrap(); - - let lock_path = checkpoints_root - .join("current-scope") - .join("checkpoint.lock"); - let mut store = CheckpointStore::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT); - store.cleanup(); - - assert!(scope_dir.is_dir()); - assert!(scope_dir.join("checkpoint.lock").is_file()); - } - #[test] fn restore_nonexistent_returns_error() { let (mut store, _store_dir) = checkpoint_store(); diff --git a/crates/aft/src/cold_build_limiter.rs b/crates/aft/src/cold_build_limiter.rs index 770c59c28..603cbd384 100644 --- a/crates/aft/src/cold_build_limiter.rs +++ b/crates/aft/src/cold_build_limiter.rs @@ -162,6 +162,12 @@ pub(crate) fn try_acquire_standing_with_limiter( request_id: impl Into, admission_epoch: u64, ) -> Option { + // Mirror the blocking path's yield: standing try-admission defers the + // final slot to any already-queued interactive or ordinary maintenance + // waiter instead of jumping the queue. + if limiter.has_non_standing_waiters() { + return None; + } let request = ColdBuildAdmissionRequest::new(request_id, ColdBuildAdmissionClass::Standing); try_acquire_classified_with_limiter(limiter, &request).map(|permit| StandingColdBuildPermit { _permit: permit, @@ -734,4 +740,54 @@ mod tests { "releasing the consumed build permit must restore the limiter slot" ); } + + #[test] + fn standing_try_admission_yields_to_non_standing_waiters() { + let limiter = test_limiter(1); + let holder = limiter.try_acquire().expect("hold the only slot"); + let cancelled = Arc::new(std::sync::atomic::AtomicBool::new(false)); + // Queue a non-standing waiter behind the held slot. + let waiter_limiter = Arc::clone(&limiter); + let waiter_cancel = Arc::clone(&cancelled); + let waiter = std::thread::spawn(move || { + let _permit = acquire_blocking_while_cancellable_with_limiter( + &waiter_limiter, + "yield-test", + ColdBuildAdmissionRequest::new("maintenance", ColdBuildAdmissionClass::Maintenance), + || true, + || waiter_cancel.load(Ordering::SeqCst), + ); + }); + let deadline = Instant::now() + Duration::from_secs(3); + while !limiter.has_non_standing_waiters() { + assert!( + Instant::now() < deadline, + "maintenance waiter must register" + ); + std::thread::yield_now(); + } + // Standing try-admission must yield while the waiter is queued. + assert!( + try_acquire_standing_with_limiter(&limiter, "standing-yield", 0).is_none(), + "standing try-admission must defer to queued non-standing waiters" + ); + cancelled.store(true, Ordering::SeqCst); + drop(holder); + waiter.join().expect("waiter exits after cancellation"); + } + + #[test] + fn standing_try_admission_takes_slot_without_waiters() { + let limiter = test_limiter(1); + let permit = try_acquire_standing_with_limiter(&limiter, "standing-free", 7); + assert!( + permit.is_some(), + "standing takes a free slot with no waiters" + ); + assert_eq!( + limiter.available.load(Ordering::Acquire), + 0, + "standing permit occupies the slot" + ); + } } diff --git a/crates/aft/src/standing_scheduler.rs b/crates/aft/src/standing_scheduler.rs index 1a6d761dc..df3a955f9 100644 --- a/crates/aft/src/standing_scheduler.rs +++ b/crates/aft/src/standing_scheduler.rs @@ -41,13 +41,17 @@ where } pub fn next(&mut self) -> Option { - if self.queue.is_empty() { - return None; - } let rounds = self.queue.len(); for _ in 0..rounds { - let key = self.queue.pop_front()?; - let deficit = self.deficits.get_mut(&key)?; + let Some(key) = self.queue.pop_front() else { + return None; + }; + // A key can sit in the queue without a deficits entry when a + // reconcile removed its root after complete() requeued it. Skip + // the stale key instead of aborting the whole dispatch turn. + let Some(deficit) = self.deficits.get_mut(&key) else { + continue; + }; *deficit += i128::from(self.quantum); if *deficit >= 0 { self.in_flight.insert(key.clone()); @@ -66,7 +70,11 @@ where *deficit -= i128::from(cost); } if has_more { - self.queue.push_back(key); + // Only requeue roots that still have a deficits entry; a + // reconcile may have removed the root while its slice ran. + if self.deficits.contains_key(&key) { + self.queue.push_back(key); + } } else { self.deficits.remove(&key); } @@ -178,4 +186,28 @@ mod tests { publish_telemetry(expected.clone()); assert_eq!(telemetry(), expected); } + + #[test] + fn next_skips_stale_queued_keys_without_aborting_dispatch() { + let mut scheduler = DeficitRoundRobin::new(100); + scheduler.reconcile(["a", "b"]); + assert_eq!(scheduler.next(), Some("a")); + // Simulate a reconcile removing "a" after complete() requeued it: + // the queue holds "a" but deficits no longer contains it. + scheduler.deficits.remove("a"); + // next() must skip the stale "a" and still dispatch "b". + assert_eq!(scheduler.next(), Some("b")); + } + + #[test] + fn complete_ignores_requeue_for_removed_root() { + let mut scheduler = DeficitRoundRobin::new(100); + scheduler.reconcile(["a", "b"]); + assert_eq!(scheduler.next(), Some("a")); + // Root "a" is removed while its slice runs. + scheduler.deficits.remove("a"); + scheduler.complete("a", 10, true); + // "a" must not be requeued without a deficits entry. + assert!(!scheduler.queue.contains(&"a")); + } } diff --git a/crates/aft/src/subc/mod.rs b/crates/aft/src/subc/mod.rs index 034abd6f3..459094305 100644 --- a/crates/aft/src/subc/mod.rs +++ b/crates/aft/src/subc/mod.rs @@ -2925,6 +2925,7 @@ where let (data_reader_tx, mut data_reader_rx) = mpsc::channel::>(256); let reader_task = spawn_reader_task(read, control_reader_tx, data_reader_tx); + let mut reader_lane = PrioritizedFrameLane::default(); let shutdown = Arc::new(Notify::new()); // Drain-tick deadline is tracked manually and checked at the TOP of every // loop turn rather than as an Interval select arm: the select below is @@ -3172,7 +3173,11 @@ where log::warn!("subc attach: fatal executor response requested teardown"); break Ok(ModuleLoopExit::SkipSearchFlush); } - maybe_frame = recv_prioritized_frame(&mut control_reader_rx, &mut data_reader_rx) => { + maybe_frame = recv_prioritized_frame( + &mut control_reader_rx, + &mut data_reader_rx, + &mut reader_lane, + ) => { let frame = match maybe_frame { None => { log::info!("subc attach: daemon closed connection"); @@ -3849,14 +3854,74 @@ where }) } +/// Cap on consecutive control frames before the reader lane checks data, so +/// sustained control traffic cannot starve data frames under the biased +/// select. +const CONTROL_BURST_LIMIT: usize = 8; + +#[derive(Default)] +struct PrioritizedFrameLane { + control_closed: bool, + data_closed: bool, + consecutive_control: usize, +} + async fn recv_prioritized_frame( control_rx: &mut mpsc::Receiver>, data_rx: &mut mpsc::Receiver>, + lane: &mut PrioritizedFrameLane, ) -> Option> { - tokio::select! { - biased; - frame = control_rx.recv() => frame, - frame = data_rx.recv() => frame, + loop { + // Once a lane is closed it can never produce again; drain the other + // lane before declaring EOF so buffered frames are not dropped. + if lane.control_closed { + return match data_rx.recv().await { + Some(frame) => Some(frame), + None => { + lane.data_closed = true; + None + } + }; + } + if lane.data_closed { + return match control_rx.recv().await { + Some(frame) => Some(frame), + None => { + lane.control_closed = true; + None + } + }; + } + if lane.consecutive_control >= CONTROL_BURST_LIMIT { + // Bound the control burst: after CONTROL_BURST_LIMIT consecutive + // control frames, await data first so sustained control traffic + // (pings/acks) cannot starve data frames. + lane.consecutive_control = 0; + match data_rx.recv().await { + Some(frame) => return Some(frame), + None => { + lane.data_closed = true; + continue; + } + } + } + tokio::select! { + biased; + frame = control_rx.recv() => match frame { + Some(frame) => { + lane.consecutive_control += 1; + return Some(frame); + } + None => { + lane.control_closed = true; + continue; + } + }, + frame = data_rx.recv() => { + lane.consecutive_control = 0; + return frame; + } + } } } @@ -6742,9 +6807,10 @@ mod tests { write_frame(&mut daemon, &ping).await.unwrap(); tokio::time::sleep(Duration::from_millis(10)).await; + let mut lane = PrioritizedFrameLane::default(); let priority = tokio::time::timeout( Duration::from_secs(1), - recv_prioritized_frame(&mut priority_rx, &mut data_rx), + recv_prioritized_frame(&mut priority_rx, &mut data_rx, &mut lane), ) .await .expect("priority frame timeout") @@ -6752,7 +6818,7 @@ mod tests { .expect("priority ingress error"); assert_eq!(priority.frame.header.ty, FrameType::Ping); - let data = recv_prioritized_frame(&mut priority_rx, &mut data_rx) + let data = recv_prioritized_frame(&mut priority_rx, &mut data_rx, &mut lane) .await .unwrap() .unwrap(); @@ -6760,6 +6826,94 @@ mod tests { reader.abort(); } + #[tokio::test] + async fn sustained_control_traffic_does_not_starve_data_frames() { + let (control_tx, mut control_rx) = mpsc::channel(CONTROL_BURST_LIMIT + 2); + let (data_tx, mut data_rx) = mpsc::channel(1); + for seq in 0..=CONTROL_BURST_LIMIT { + control_tx + .send(Ok(DecodedFrame { + frame: Frame::build( + FrameType::Ping, + control_flags(), + 0, + 0, + seq as u64, + Vec::new(), + ) + .unwrap(), + phase_trace: PhaseTrace::new(Instant::now()), + })) + .await + .unwrap(); + } + data_tx + .send(Ok(DecodedFrame { + frame: Frame::build( + FrameType::Request, + control_flags(), + 1, + 1, + 99, + br#"{}"#.to_vec(), + ) + .unwrap(), + phase_trace: PhaseTrace::new(Instant::now()), + })) + .await + .unwrap(); + + let mut lane = PrioritizedFrameLane::default(); + for _ in 0..CONTROL_BURST_LIMIT { + let frame = recv_prioritized_frame(&mut control_rx, &mut data_rx, &mut lane) + .await + .unwrap() + .unwrap(); + assert_eq!(frame.frame.header.ty, FrameType::Ping); + } + let frame = recv_prioritized_frame(&mut control_rx, &mut data_rx, &mut lane) + .await + .unwrap() + .unwrap(); + assert_eq!(frame.frame.header.ty, FrameType::Request); + } + + #[tokio::test] + async fn closed_control_lane_drains_buffered_data_before_eof() { + let (control_tx, mut control_rx) = mpsc::channel(1); + let (data_tx, mut data_rx) = mpsc::channel(1); + drop(control_tx); + data_tx + .send(Ok(DecodedFrame { + frame: Frame::build( + FrameType::Request, + control_flags(), + 1, + 1, + 1, + br#"{}"#.to_vec(), + ) + .unwrap(), + phase_trace: PhaseTrace::new(Instant::now()), + })) + .await + .unwrap(); + drop(data_tx); + + let mut lane = PrioritizedFrameLane::default(); + let frame = recv_prioritized_frame(&mut control_rx, &mut data_rx, &mut lane) + .await + .expect("buffered data must drain") + .unwrap(); + assert_eq!(frame.frame.header.ty, FrameType::Request); + assert!( + recv_prioritized_frame(&mut control_rx, &mut data_rx, &mut lane) + .await + .is_none(), + "EOF only after both lanes drain" + ); + } + #[test] fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() { let transient_errors = vec![ diff --git a/crates/aft/tests/integration/subc_storm_test.rs b/crates/aft/tests/integration/subc_storm_test.rs index b8e8f876f..24fa8ac69 100644 --- a/crates/aft/tests/integration/subc_storm_test.rs +++ b/crates/aft/tests/integration/subc_storm_test.rs @@ -1757,8 +1757,10 @@ async fn drive_standing_yield_daemon(input: FakeDaemonInput) { )); } - // All passes complete (yield) even though cold slots stay saturated. - for (receiver, _yielded) in receivers { + // All passes complete (yield) even though cold slots stay saturated, and + // every pass must actually observe the yield — both permits are held by + // this test, so try_acquire cannot succeed. + for (receiver, yielded) in receivers { let response = tokio::time::timeout(Duration::from_secs(5), receiver) .await .expect("standing pass yields instead of waiting") @@ -1767,6 +1769,14 @@ async fn drive_standing_yield_daemon(input: FakeDaemonInput) { response.success, "a standing pass answers success without blocking on cold admission" ); + assert!( + response.data.get("yielded").and_then(Value::as_bool) == Some(true), + "standing pass must yield while both cold permits are held" + ); + assert!( + yielded.load(std::sync::atomic::Ordering::Acquire), + "yield probe must confirm try_acquire returned None" + ); } // A PureRead with a finite deadline finishes well inside its budget while From a6f3afb4d3094c3bdc2dcb4287dc1e4b1d8f9925 Mon Sep 17 00:00:00 2001 From: Naadir Jeewa Date: Mon, 31 Aug 2026 20:10:43 +0100 Subject: [PATCH 11/14] fix(index): close remaining review gaps Signed-off-by: Naadir Jeewa Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/executor/tests.rs | 137 +++++++------- crates/aft/src/lib.rs | 1 + crates/aft/src/search_index.rs | 2 + crates/aft/src/semantic_index.rs | 170 +++++++++++++++--- crates/aft/src/standing_scheduler.rs | 35 ++++ crates/aft/src/subc/standing.rs | 121 ++++++++++--- crates/aft/src/thread_priority.rs | 104 +++++++---- .../aft/tests/integration/callgraph_test.rs | 19 +- docs/config.md | 4 +- 9 files changed, 438 insertions(+), 155 deletions(-) diff --git a/crates/aft/src/executor/tests.rs b/crates/aft/src/executor/tests.rs index 89f7334e3..bd1af4c76 100644 --- a/crates/aft/src/executor/tests.rs +++ b/crates/aft/src/executor/tests.rs @@ -2562,87 +2562,83 @@ fn queued_deadline_job_is_pruned_and_counted_at_next_turn() { #[test] fn interactive_queue_cap_returns_typed_backpressure_per_actor_and_global() { - // pool 2 / actor_cap 1: one running blocker per actor, then the per-actor - // interactive cap admits 2 more; the next is rejected with the actor scope. - // A second actor's global budget is sized so its first overflow reports the - // global scope. - let executor = test_executor(2, 1, 1, 1); + let executor = Executor::with_config(ExecutorConfig { + pool_size: 1, + read_cap: 1, + actor_cap: 1, + heavy_permits: 1, + drr_quantum: 1, + interactive_queue_cap: 2, + interactive_actor_queue_cap: 1, + ..ExecutorConfig::default() + }); let (_dir_a, root_a) = test_root("interactive-cap-a"); + let (_dir_b, root_b) = test_root("interactive-cap-b"); + let (_dir_c, root_c) = test_root("interactive-cap-c"); executor.register_actor(root_a.clone(), test_ctx()); + executor.register_actor(root_b.clone(), test_ctx()); + executor.register_actor(root_c.clone(), test_ctx()); - let (blocker_started_tx, blocker_started_rx) = crossbeam_channel::bounded(1); - let (release_blocker_tx, release_blocker_rx) = crossbeam_channel::bounded(1); + let (started_tx, started_rx) = crossbeam_channel::bounded(1); + let (release_tx, release_rx) = crossbeam_channel::bounded(1); let blocker = executor.submit( root_a.clone(), Lane::Mutating, "interactive-cap-blocker".to_string(), Box::new(move |_| { - blocker_started_tx.send(()).expect("signal blocker start"); - release_blocker_rx + started_tx.send(()).expect("signal blocker start"); + release_rx .recv_timeout(Duration::from_secs(5)) - .expect("release interactive blocker"); + .expect("release blocker"); ok("interactive-cap-blocker") }), ); - blocker_started_rx + started_rx .recv_timeout(Duration::from_secs(2)) - .expect("interactive blocker starts"); + .expect("blocker starts"); - let executed = Arc::new(AtomicUsize::new(0)); - let mut admitted = Vec::new(); - for _ in 0..executor.interactive_actor_queue_cap() { - let executed_probe = Arc::clone(&executed); - admitted.push(executor.submit_async( - root_a.clone(), - Lane::PureRead, - "interactive-cap-admitted".to_string(), - Box::new(move |_| { - executed_probe.fetch_add(1, Ordering::AcqRel); - ok("interactive-cap-admitted") - }), - )); - } - let overflow = executor.submit_async( - root_a, - Lane::PureRead, - "interactive-cap-overflow".to_string(), - Box::new(|_| ok("interactive-cap-overflow")), + let queued_a = executor.submit_async( + root_a.clone(), + Lane::Mutating, + "queued-a".to_string(), + Box::new(|_| ok("queued-a")), ); - - let overflow_response = recv_async(overflow, "interactive backpressure completion"); - assert!(!overflow_response.success); - assert_eq!(overflow_response.data["code"], "executor_backpressure"); - assert_eq!(overflow_response.data["retryable"], serde_json::json!(true)); - assert_eq!( - overflow_response.data["queue_class"], - serde_json::json!("interactive") + let queued_b = executor.submit_async( + root_b.clone(), + Lane::Mutating, + "queued-b".to_string(), + Box::new(|_| ok("queued-b")), ); - assert_eq!( - overflow_response.data["queue_scope"], - serde_json::json!("actor") + let global_overflow = executor.submit_async( + root_c, + Lane::Mutating, + "global-overflow".to_string(), + Box::new(|_| ok("global-overflow")), ); - assert_eq!(executed.load(Ordering::Acquire), 0); + let response = recv_async(global_overflow, "global overflow"); + assert!(!response.success); + assert_eq!(response.data["code"], "executor_backpressure"); + assert_eq!(response.data["queue_scope"], "global"); - release_blocker_tx - .send(()) - .expect("release interactive blocker"); + let actor_overflow = executor.submit_async( + root_a, + Lane::PureRead, + "actor-overflow".to_string(), + Box::new(|_| ok("actor-overflow")), + ); + let response = recv_async(actor_overflow, "actor overflow"); + assert!(!response.success); + assert_eq!(response.data["queue_scope"], "actor"); + + release_tx.send(()).expect("release blocker"); assert!( blocker .recv_timeout(Duration::from_secs(5)) - .expect("blocker completes") + .unwrap() .success ); - for receiver in admitted { - assert!( - recv_async(receiver, "admitted interactive completion").success, - "admitted interactive job must execute" - ); - } - assert_eq!( - executed.load(Ordering::Acquire), - executor.interactive_actor_queue_cap(), - "every admitted interactive job must execute exactly once" - ); + assert!(recv_async(queued_a, "queued a").success); + assert!(recv_async(queued_b, "queued b").success); } #[test] @@ -2759,6 +2755,22 @@ fn queue_accounting_tracks_dispatch_cancellation_and_actor_retirement() { .expect("liveness after removal"); assert_eq!(snapshot.interactive.queued, 0); assert_eq!(snapshot.maintenance.queued, 0); + + let (_replacement_dir, replacement_root) = test_root("accounting-replacement"); + executor.register_actor(replacement_root.clone(), test_ctx()); + assert!( + recv_async( + executor.submit_async( + replacement_root, + Lane::PureRead, + "accounting-replacement".to_string(), + Box::new(|_| ok("accounting-replacement")), + ), + "replacement actor completion", + ) + .success, + "actor retirement must release process-wide interactive capacity" + ); } #[test] @@ -2795,16 +2807,21 @@ fn dispatched_job_is_not_auto_cancelled_after_deadline_passes() { let (_dir, root) = test_root("dispatched-not-cancelled"); executor.register_actor(root.clone(), test_ctx()); + let (started_tx, started_rx) = crossbeam_channel::bounded(1); let (rx, _token) = executor.submit_cancellable_async_with_deadline( root, Lane::PureRead, "late-runner".to_string(), - Box::new(|_| { + Box::new(move |_| { + started_tx.send(()).expect("signal dispatched job start"); thread::sleep(Duration::from_millis(150)); ok("late-runner") }), Some(Instant::now() + Duration::from_millis(20)), ); + started_rx + .recv_timeout(Duration::from_secs(2)) + .expect("job dispatches before deadline"); let response = recv_async(rx, "late runner completion"); assert!( response.success, @@ -2839,7 +2856,7 @@ fn deadline_aware_writer_urgency_matches_budget_boundaries() { // 12s budget queued at 6s: halfway point reached. Case { label: "halfway urgency", - deadline: Some(now + Duration::from_secs(6)), + deadline: Some(now + Duration::from_secs(12)), now_offset_ms: 6_000, expect_urgent: true, }, diff --git a/crates/aft/src/lib.rs b/crates/aft/src/lib.rs index 66c6003ab..77e1d2669 100644 --- a/crates/aft/src/lib.rs +++ b/crates/aft/src/lib.rs @@ -123,6 +123,7 @@ pub mod subc_config; pub mod subc_format; pub mod subc_translate; pub mod symbol_cache_disk; +pub mod symbol_diff; pub mod symbols; pub mod synapse_embed; pub mod thread_priority; diff --git a/crates/aft/src/search_index.rs b/crates/aft/src/search_index.rs index f29e5e6d4..011cf6a37 100644 --- a/crates/aft/src/search_index.rs +++ b/crates/aft/src/search_index.rs @@ -8624,8 +8624,10 @@ mod tests { {} let published = SearchIndex::read_from_disk(&cache, &project).expect("published index"); let result = published.grep("id_marker_4", true, &[], &[], &project, 10); + assert_eq!(result.total_matches, 1); } + #[test] fn resumable_search_restarts_when_content_changes_without_metadata_change() { let dir = tempfile::tempdir().expect("create temp dir"); let project = dir.path().join("project"); diff --git a/crates/aft/src/semantic_index.rs b/crates/aft/src/semantic_index.rs index fd5e84862..4aa8ec1dd 100644 --- a/crates/aft/src/semantic_index.rs +++ b/crates/aft/src/semantic_index.rs @@ -121,8 +121,10 @@ impl EmbeddingRequestPolicy { } } -const SEMANTIC_STAGING_VERSION: u32 = 1; -const SEMANTIC_STAGING_FILE: &str = "semantic-staging-v1.json"; +const SEMANTIC_STAGING_VERSION: u32 = 2; +const SEMANTIC_STAGING_FILE: &str = "semantic-staging-v2.json"; +const SEMANTIC_STAGING_CHUNKS_FILE: &str = "semantic-staging-chunks-v2.jsonl"; +const SEMANTIC_STAGING_VECTORS_FILE: &str = "semantic-staging-vectors-v2.bin"; const SEMANTIC_COLLECT_SLICE_FILES: usize = 32; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -140,9 +142,9 @@ struct SemanticStagingManifest { corpus_fingerprint: String, collect_cursor: usize, embed_cursor: usize, - chunks: Vec, + chunks_count: usize, metadata: Vec, - vectors: Vec>, + vectors_count: usize, } #[derive(Debug, Serialize, Deserialize)] @@ -2089,6 +2091,88 @@ fn write_semantic_staging(path: &Path, manifest: &SemanticStagingManifest) -> Re crate::fs_lock::rename_over(&temporary, path).map_err(|error| error.to_string()) } +fn append_semantic_chunks(path: &Path, chunks: &[SemanticChunk]) -> Result<(), String> { + use std::io::Write; + let mut file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + .map_err(|error| error.to_string())?; + for chunk in chunks { + serde_json::to_writer(&mut file, chunk).map_err(|error| error.to_string())?; + file.write_all(b"\n").map_err(|error| error.to_string())?; + } + file.sync_data().map_err(|error| error.to_string()) +} + +fn read_semantic_chunks(path: &Path, count: usize) -> Result, String> { + let text = fs::read_to_string(path).map_err(|error| error.to_string())?; + let chunks = text + .lines() + .filter(|line| !line.is_empty()) + .map(|line| serde_json::from_str(line).map_err(|error| error.to_string())) + .collect::, _>>()?; + if chunks.len() != count { + return Err(format!( + "semantic staging chunk segment count mismatch: expected {count}, got {}", + chunks.len() + )); + } + Ok(chunks) +} + +fn append_semantic_vectors( + path: &Path, + vectors: &[Vec], + dimension: usize, +) -> Result<(), String> { + use std::io::Write; + let mut file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + .map_err(|error| error.to_string())?; + for vector in vectors { + if vector.len() != dimension { + return Err("semantic staging vector dimension mismatch".to_string()); + } + for value in vector { + file.write_all(&value.to_le_bytes()) + .map_err(|error| error.to_string())?; + } + } + file.sync_data().map_err(|error| error.to_string()) +} + +fn read_semantic_vectors( + path: &Path, + count: usize, + dimension: usize, +) -> Result>, String> { + let bytes = fs::read(path).map_err(|error| error.to_string())?; + let record_bytes = dimension + .checked_mul(std::mem::size_of::()) + .ok_or_else(|| "semantic staging vector size overflow".to_string())?; + let expected = count + .checked_mul(record_bytes) + .ok_or_else(|| "semantic staging vector size overflow".to_string())?; + if bytes.len() != expected { + return Err(format!( + "semantic staging vector segment length mismatch: expected {expected}, got {}", + bytes.len() + )); + } + Ok(bytes + .chunks_exact(record_bytes) + .map(|record| { + record + .chunks_exact(std::mem::size_of::()) + .map(|bytes| f32::from_le_bytes(bytes.try_into().expect("f32 byte width"))) + .collect() + }) + .collect()) +} + /// The semantic index — stores embeddings for all symbols in a project. /// Borrow-only roots retain only a root path plus an Arc to immutable relative data. #[derive(Debug, Clone)] @@ -2860,7 +2944,25 @@ impl SemanticIndex { let corpus_fingerprint = semantic_corpus_fingerprint(&canonical_root, files, &fingerprint); let dir = storage_dir.join("semantic").join(project_key); let staging_path = dir.join(SEMANTIC_STAGING_FILE); + let chunks_path = dir.join(SEMANTIC_STAGING_CHUNKS_FILE); + let vectors_path = dir.join(SEMANTIC_STAGING_VECTORS_FILE); fs::create_dir_all(&dir).map_err(|error| error.to_string())?; + + let valid_segment_lengths = |manifest: &SemanticStagingManifest| { + let chunks_valid = fs::read_to_string(&chunks_path) + .map(|text| { + text.lines().filter(|line| !line.is_empty()).count() == manifest.chunks_count + }) + .unwrap_or(manifest.chunks_count == 0); + let vector_bytes = manifest + .vectors_count + .saturating_mul(fingerprint.dimension) + .saturating_mul(std::mem::size_of::()); + let vectors_valid = fs::metadata(&vectors_path) + .map(|metadata| metadata.len() == u64::try_from(vector_bytes).unwrap_or(u64::MAX)) + .unwrap_or(manifest.vectors_count == 0); + chunks_valid && vectors_valid + }; let mut manifest = load_semantic_staging(&staging_path) .filter(|manifest| { manifest.version == SEMANTIC_STAGING_VERSION @@ -2868,31 +2970,33 @@ impl SemanticIndex { && manifest.corpus_fingerprint == corpus_fingerprint && manifest.files == files && manifest.collect_cursor <= files.len() - && manifest.embed_cursor <= manifest.chunks.len() - && manifest.vectors.len() == manifest.embed_cursor - && manifest - .vectors - .iter() - .all(|vector| vector.len() == fingerprint.dimension) + && manifest.embed_cursor <= manifest.chunks_count + && manifest.vectors_count == manifest.embed_cursor + && valid_segment_lengths(manifest) }) - .unwrap_or(SemanticStagingManifest { - version: SEMANTIC_STAGING_VERSION, - canonical_root: canonical_root.clone(), - fingerprint: fingerprint.clone(), - files: files.to_vec(), - corpus_fingerprint, - collect_cursor: 0, - embed_cursor: 0, - chunks: Vec::new(), - metadata: Vec::new(), - vectors: Vec::new(), + .unwrap_or_else(|| { + let _ = fs::remove_file(&chunks_path); + let _ = fs::remove_file(&vectors_path); + SemanticStagingManifest { + version: SEMANTIC_STAGING_VERSION, + canonical_root: canonical_root.clone(), + fingerprint: fingerprint.clone(), + files: files.to_vec(), + corpus_fingerprint, + collect_cursor: 0, + embed_cursor: 0, + chunks_count: 0, + metadata: Vec::new(), + vectors_count: 0, + } }); if manifest.collect_cursor < files.len() { let end = (manifest.collect_cursor + SEMANTIC_COLLECT_SLICE_FILES).min(files.len()); let (chunks, metadata) = Self::collect_chunks(&canonical_root, &files[manifest.collect_cursor..end]); - manifest.chunks.extend(chunks); + append_semantic_chunks(&chunks_path, &chunks)?; + manifest.chunks_count += chunks.len(); manifest .metadata .extend(metadata.into_iter().map(|(path, metadata)| { @@ -2912,10 +3016,11 @@ impl SemanticIndex { return Ok(SemanticBuildSliceOutcome::Yielded); } - if manifest.embed_cursor < manifest.chunks.len() { + if manifest.embed_cursor < manifest.chunks_count { + let chunks = read_semantic_chunks(&chunks_path, manifest.chunks_count)?; let end = - (manifest.embed_cursor + model.max_batch_size().max(1)).min(manifest.chunks.len()); - let texts = manifest.chunks[manifest.embed_cursor..end] + (manifest.embed_cursor + model.max_batch_size().max(1)).min(manifest.chunks_count); + let texts = chunks[manifest.embed_cursor..end] .iter() .map(|chunk| chunk.embed_text.clone()) .collect(); @@ -2926,16 +3031,22 @@ impl SemanticIndex { .any(|vector| vector.len() != fingerprint.dimension) { let _ = fs::remove_file(&staging_path); + let _ = fs::remove_file(&chunks_path); + let _ = fs::remove_file(&vectors_path); return Err( "embedding dimension changed during resumable semantic build".to_string(), ); } - manifest.vectors.extend(vectors); + append_semantic_vectors(&vectors_path, &vectors, fingerprint.dimension)?; manifest.embed_cursor = end; + manifest.vectors_count = end; write_semantic_staging(&staging_path, &manifest)?; return Ok(SemanticBuildSliceOutcome::Yielded); } + let chunks = read_semantic_chunks(&chunks_path, manifest.chunks_count)?; + let vectors = + read_semantic_vectors(&vectors_path, manifest.vectors_count, fingerprint.dimension)?; let file_metadata = manifest .metadata .iter() @@ -2953,10 +3064,9 @@ impl SemanticIndex { ) }) .collect::>(); - let entries = manifest - .chunks + let entries = chunks .into_iter() - .zip(manifest.vectors) + .zip(vectors) .map(|(chunk, vector)| EmbeddingEntry::new(chunk, vector)) .collect::>(); let mut index = Self { @@ -2987,6 +3097,8 @@ impl SemanticIndex { return Err("failed to publish resumable semantic index".to_string()); } fs::remove_file(&staging_path).map_err(|error| error.to_string())?; + fs::remove_file(&chunks_path).map_err(|error| error.to_string())?; + fs::remove_file(&vectors_path).map_err(|error| error.to_string())?; Ok(SemanticBuildSliceOutcome::Complete) } diff --git a/crates/aft/src/standing_scheduler.rs b/crates/aft/src/standing_scheduler.rs index df3a955f9..7e0f014af 100644 --- a/crates/aft/src/standing_scheduler.rs +++ b/crates/aft/src/standing_scheduler.rs @@ -7,6 +7,7 @@ pub struct DeficitRoundRobin { queue: VecDeque, deficits: HashMap, in_flight: HashSet, + generations: HashMap, } impl DeficitRoundRobin @@ -20,6 +21,7 @@ where queue: VecDeque::new(), deficits: HashMap::new(), in_flight: HashSet::new(), + generations: HashMap::new(), } } @@ -35,6 +37,10 @@ where for key in keys { if !self.deficits.contains_key(&key) { self.deficits.insert(key.clone(), 0); + self.generations + .entry(key.clone()) + .and_modify(|generation| *generation = generation.saturating_add(1)) + .or_insert(0); self.queue.push_back(key); } } @@ -62,6 +68,19 @@ where None } + pub fn generation(&self, key: &K) -> Option { + self.deficits + .contains_key(key) + .then(|| self.generations.get(key).copied().unwrap_or(0)) + } + + pub fn complete_generation(&mut self, key: K, generation: u64, cost: u64, has_more: bool) { + if self.generation(&key) != Some(generation) { + return; + } + self.complete(key, cost, has_more); + } + pub fn complete(&mut self, key: K, cost: u64, has_more: bool) { if !self.in_flight.remove(&key) { return; @@ -210,4 +229,20 @@ mod tests { // "a" must not be requeued without a deficits entry. assert!(!scheduler.queue.contains(&"a")); } + + #[test] + fn stale_completion_cannot_mutate_readded_root_generation() { + let mut scheduler = DeficitRoundRobin::new(100); + scheduler.reconcile(["a", "b"]); + assert_eq!(scheduler.next(), Some("a")); + let old_generation = scheduler.generation(&"a").unwrap(); + scheduler.reconcile(["b"]); + scheduler.reconcile(["a", "b"]); + let new_generation = scheduler.generation(&"a").unwrap(); + assert_ne!(new_generation, old_generation); + + scheduler.complete_generation("a", old_generation, 100, false); + assert_eq!(scheduler.generation(&"a"), Some(new_generation)); + assert!(scheduler.deficits.contains_key("a")); + } } diff --git a/crates/aft/src/subc/standing.rs b/crates/aft/src/subc/standing.rs index 0f3678487..03ad5f2d3 100644 --- a/crates/aft/src/subc/standing.rs +++ b/crates/aft/src/subc/standing.rs @@ -5,11 +5,22 @@ //! submitted through the existing executor's coalescable maintenance lane. use std::collections::HashMap; -use std::sync::Arc; +use std::path::PathBuf; +use std::sync::{Arc, Mutex as SyncMutex}; use std::time::Instant; use parking_lot::Mutex; +type CachedSemanticModel = Arc>>; + +#[derive(Clone)] +struct SemanticBuildCache { + config_key: String, + files_key: String, + files: Arc>, + model: CachedSemanticModel, +} + use crate::config::{Config, IndexKind}; use crate::context::{App, AppContext}; use crate::executor::{Executor, Lane, MaintenanceCoalesceKey}; @@ -50,6 +61,7 @@ impl StandingReconcileKey { struct PendingStandingSlice { receiver: tokio::sync::oneshot::Receiver, started_at: Instant, + generation: u64, } struct StandingScheduleState { @@ -90,6 +102,7 @@ pub(super) struct StandingActor { /// are never removed by this owner. owned_actors: Mutex>, schedule: Mutex, + semantic_cache: Arc>>, } impl StandingActor { @@ -102,6 +115,7 @@ impl StandingActor { reconciled_config: Mutex::new(None), owned_actors: Mutex::new(HashMap::new()), schedule: Mutex::new(StandingScheduleState::default()), + semantic_cache: Arc::new(Mutex::new(HashMap::new())), } } @@ -283,9 +297,15 @@ impl StandingActor { .pending .iter_mut() .filter_map(|(key, pending)| match pending.receiver.try_recv() { - Ok(response) => Some((key.clone(), response, pending.started_at.elapsed())), + Ok(response) => Some(( + key.clone(), + pending.generation, + response, + pending.started_at.elapsed(), + )), Err(tokio::sync::oneshot::error::TryRecvError::Closed) => Some(( key.clone(), + pending.generation, crate::protocol::Response::error( "standing", "standing_slice_closed", @@ -296,7 +316,7 @@ impl StandingActor { Err(tokio::sync::oneshot::error::TryRecvError::Empty) => None, }) .collect::>(); - for (key, response, elapsed) in completed { + for (key, generation, response, elapsed) in completed { schedule.pending.remove(&key); let has_more = response .data @@ -328,13 +348,15 @@ impl StandingActor { schedule.yielded_slices = schedule.yielded_slices.saturating_add(1); } Self::publish_schedule_telemetry(&schedule); - schedule.queue.complete(key, cost, has_more); + schedule + .queue + .complete_generation(key, generation, cost, has_more); } } fn dispatch_ready_slices(&self, snapshot: &Config) { loop { - let entry = { + let (entry, generation) = { let mut schedule = self.schedule.lock(); if schedule.pending.len() >= crate::cold_build_limiter::limit() { return; @@ -342,11 +364,14 @@ impl StandingActor { let Some(key) = schedule.queue.next() else { return; }; + let generation = schedule.queue.generation(&key).unwrap_or(0); let Some(entry) = schedule.entries.get(&key).cloned() else { - schedule.queue.complete(key, 1, false); + schedule + .queue + .complete_generation(key, generation, 1, false); continue; }; - entry + (entry, generation) }; let Some(receiver) = self.submit_entry_slice(entry.clone(), snapshot) else { self.schedule @@ -361,6 +386,7 @@ impl StandingActor { PendingStandingSlice { receiver, started_at: Instant::now(), + generation, }, ); Self::publish_schedule_telemetry(&schedule); @@ -427,6 +453,7 @@ impl StandingActor { .next_kind .get(&entry.literal_path) .unwrap_or(&0); + let semantic_cache = Arc::clone(&self.semantic_cache); let selected = IndexKind::ALL .iter() .copied() @@ -481,6 +508,7 @@ impl StandingActor { &entry, &admission, permit.admission_epoch, + &semantic_cache, ) } else { build_missing_callgraph_after_strict_check( @@ -690,6 +718,7 @@ fn build_missing_semantic_after_strict_check( entry: &StandingRootEntry, admission: &crate::standing_roots::StandingBuildAdmission, permit_epoch: u64, + cache: &Arc>>, ) -> (bool, bool) { if admission.cancellation_requested() || crate::executor::current_job_cancelled() { return (false, true); @@ -702,20 +731,9 @@ fn build_missing_semantic_after_strict_check( let Some(storage_dir) = storage_dir else { return (false, true); }; - let files = match crate::commands::configure::walk_semantic_project_files_bounded( - &entry.resolved_target, - semantic_config.max_files, - ) { - Ok(files) => files, - Err(_) => return (false, true), - }; - let mut model = match crate::semantic_index::EmbeddingModel::from_config(&semantic_config) { - Ok(model) => model, - Err(error) => { - log::warn!("standing semantic model initialization failed: {}", error); - return (false, true); - } - }; + + // Deny borrow-only roots before creating staging state or constructing an + // embedding model. The writer lease is the artifact-level authority. let cache_dir = storage_dir.join("semantic").join(&entry.artifact_key); let lease = match crate::root_cache::WriterLease::acquire_shared( crate::root_cache::RootCacheDomain::Index, @@ -726,6 +744,54 @@ fn build_missing_semantic_after_strict_check( Ok(Some(lease)) => lease, Ok(None) | Err(_) => return (false, true), }; + + let config_key = serde_json::to_string(&semantic_config).unwrap_or_default(); + let files_key = format!( + "{}:{}:{}", + config_key, semantic_config.max_files, entry.artifact_key + ); + let cached = { + let mut cache = cache.lock(); + let needs_refresh = cache + .get(&entry.literal_path) + .is_none_or(|cached| cached.config_key != config_key || cached.files_key != files_key); + if needs_refresh { + let files = match crate::commands::configure::walk_semantic_project_files_bounded( + &entry.resolved_target, + semantic_config.max_files, + ) { + Ok(files) => Arc::new(files), + Err(_) => return (false, true), + }; + cache.insert( + entry.literal_path.clone(), + SemanticBuildCache { + config_key: config_key.clone(), + files_key: files_key.clone(), + files, + model: Arc::new(SyncMutex::new(None)), + }, + ); + } + cache + .get(&entry.literal_path) + .cloned() + .expect("cache inserted") + }; + let mut model_guard = cached + .model + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + if model_guard.is_none() { + match crate::semantic_index::EmbeddingModel::from_config(&semantic_config) { + Ok(model) => *model_guard = Some(model), + Err(error) => { + log::warn!("standing semantic model initialization failed: {}", error); + return (false, true); + } + } + } + let model = model_guard.as_mut().expect("model initialized"); let outcome = roots .publish_if_current( &entry.literal_path, @@ -740,8 +806,8 @@ fn build_missing_semantic_after_strict_check( || { crate::semantic_index::SemanticIndex::resume_cold_build_slice( &entry.resolved_target, - &files, - &mut model, + &cached.files, + model, &semantic_config, &storage_dir, &entry.artifact_key, @@ -751,9 +817,12 @@ fn build_missing_semantic_after_strict_check( ) .ok() .flatten(); - match outcome { - Some(true) => (true, false), - Some(false) | None => (false, true), + if outcome == Some(true) { + // Force a fresh inventory at the start of the next complete build. + cache.lock().remove(&entry.literal_path); + (true, false) + } else { + (false, true) } } diff --git a/crates/aft/src/thread_priority.rs b/crates/aft/src/thread_priority.rs index e81a84079..784747337 100644 --- a/crates/aft/src/thread_priority.rs +++ b/crates/aft/src/thread_priority.rs @@ -32,6 +32,7 @@ #[cfg(any(target_os = "linux", target_os = "macos", windows))] thread_local! { static WARNED: std::cell::Cell = const { std::cell::Cell::new(0) }; + static PRIORITIES: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; } #[cfg(any(target_os = "linux", target_os = "macos", windows))] @@ -55,14 +56,29 @@ mod imp { use super::warn_once; use libc::{c_int, c_long, syscall}; - pub fn demote() { + #[derive(Clone, Copy)] + pub struct PreviousPriority { + scheduler: c_int, + io_priority: c_int, + } + + pub fn demote() -> PreviousPriority { + let previous = PreviousPriority { + scheduler: unsafe { libc::sched_getscheduler(0) }, + io_priority: unsafe { + syscall(libc::SYS_ioprio_get, IOPRIO_WHO_PROCESS, tid()) as c_int + }, + }; cpu_idle(); io_idle(); + previous } - pub fn restore() { - cpu_other(); - io_best_effort(); + pub fn restore(previous: PreviousPriority) { + cpu_policy(previous.scheduler); + if !io_set(IOPRIO_WHO_PROCESS, tid(), previous.io_priority) { + warn_once("io", &std::io::Error::last_os_error().to_string()); + } } /// SCHED_IDLE is not bound by the `libc` crate on gnu/musl; the value is a @@ -74,7 +90,6 @@ mod imp { pub(super) const SCHED_OTHER: c_int = 0; pub(super) const IOPRIO_CLASS_IDLE: c_int = 3; - pub(super) const IOPRIO_CLASS_BE: c_int = 2; pub(super) const IOPRIO_WHO_PROCESS: c_int = 1; const IOPRIO_CLASS_SHIFT: c_int = 13; const IOPRIO_NICE_SHIFT: c_int = 0; @@ -88,10 +103,10 @@ mod imp { } } - fn cpu_other() { + fn cpu_policy(policy: c_int) { let mut param = unsafe { std::mem::zeroed::() }; param.sched_priority = 0; - let rc = unsafe { libc::sched_setscheduler(0, SCHED_OTHER, ¶m) }; + let rc = unsafe { libc::sched_setscheduler(0, policy, ¶m) }; if rc != 0 { warn_once("cpu", &std::io::Error::last_os_error().to_string()); } @@ -119,31 +134,39 @@ mod imp { warn_once("io", &std::io::Error::last_os_error().to_string()); } } - - fn io_best_effort() { - if !io_set(IOPRIO_WHO_PROCESS, tid(), io_prio(IOPRIO_CLASS_BE, 0)) { - warn_once("io", &std::io::Error::last_os_error().to_string()); - } - } } #[cfg(target_os = "macos")] mod imp { use super::warn_once; - pub fn demote() { + #[derive(Clone, Copy)] + pub struct PreviousPriority { + qos: libc::qos_class_t, + relative_priority: libc::c_int, + } + + pub fn demote() -> PreviousPriority { + let mut relative_priority = 0; + let qos = + unsafe { libc::pthread_get_qos_class_np(libc::pthread_self(), &mut relative_priority) }; let rc = unsafe { libc::pthread_set_qos_class_self_np(libc::qos_class_t::QOS_CLASS_UTILITY, 0) }; if rc != 0 { - warn_once("cpu", &std::io::Error::last_os_error().to_string()); + warn_once("cpu", &format!("pthread error {rc}")); + } + PreviousPriority { + qos, + relative_priority, } } - pub fn restore() { - let rc = - unsafe { libc::pthread_set_qos_class_self_np(libc::qos_class_t::QOS_CLASS_DEFAULT, 0) }; + pub fn restore(previous: PreviousPriority) { + let rc = unsafe { + libc::pthread_set_qos_class_self_np(previous.qos, previous.relative_priority) + }; if rc != 0 { - warn_once("cpu", &std::io::Error::last_os_error().to_string()); + warn_once("cpu", &format!("pthread error {rc}")); } } } @@ -157,6 +180,7 @@ mod imp { extern "system" { fn GetCurrentThread() -> *mut core::ffi::c_void; + fn GetThreadPriority(hThread: *mut core::ffi::c_void) -> i32; fn SetThreadPriority(hThread: *mut core::ffi::c_void, nPriority: i32) -> i32; } @@ -166,33 +190,47 @@ mod imp { unsafe { SetThreadPriority(GetCurrentThread(), level) != 0 } } - pub fn demote() { + #[derive(Clone, Copy)] + pub struct PreviousPriority(i32); + + pub fn demote() -> PreviousPriority { + let previous = PreviousPriority(unsafe { GetThreadPriority(GetCurrentThread()) }); if !set(THREAD_PRIORITY_LOWEST) { - warn_once("cpu", &format!("win32 error {}", std::process::id())); + warn_once("cpu", &std::io::Error::last_os_error().to_string()); } + previous } - pub fn restore() { - if !set(THREAD_PRIORITY_NORMAL) { - warn_once("cpu", &format!("win32 error {}", std::process::id())); + pub fn restore(previous: PreviousPriority) { + if !set(previous.0) { + warn_once("cpu", &std::io::Error::last_os_error().to_string()); } } } #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] mod imp { - pub fn demote() {} - pub fn restore() {} + #[derive(Clone, Copy)] + pub struct PreviousPriority; + pub fn demote() -> PreviousPriority { + PreviousPriority + } + pub fn restore(_: PreviousPriority) {} } /// Denote the current thread (CPU and I/O) for background maintenance. pub fn demote_background() { - imp::demote(); + let previous = imp::demote(); + PRIORITIES.with(|priorities| priorities.borrow_mut().push(previous)); } /// Restore normal priority for the current thread after maintenance work. pub fn restore_default() { - imp::restore(); + PRIORITIES.with(|priorities| { + if let Some(previous) = priorities.borrow_mut().pop() { + imp::restore(previous); + } + }); } /// Restores normal priority when the guard drops, including on panic unwind. struct BackgroundGuard; @@ -215,8 +253,7 @@ pub fn with_background(f: impl FnOnce() -> R) -> R { #[cfg(all(test, target_os = "linux"))] mod tests { use super::imp::{ - io_prio, tid, IOPRIO_CLASS_BE, IOPRIO_CLASS_IDLE, IOPRIO_WHO_PROCESS, SCHED_IDLE, - SCHED_OTHER, + io_prio, tid, IOPRIO_CLASS_IDLE, IOPRIO_WHO_PROCESS, SCHED_IDLE, SCHED_OTHER, }; use super::{demote_background, restore_default, with_background}; use libc::{c_int, syscall}; @@ -236,6 +273,7 @@ mod tests { SCHED_OTHER, "test precondition: thread starts in SCHED_OTHER (policy codes may vary; SCHED_OTHER=0)" ); + let initial_io_priority = io_priority(); demote_background(); @@ -258,9 +296,9 @@ mod tests { "restore moves thread back to SCHED_OTHER" ); assert_eq!( - io_priority() & !0x7f, - io_prio(IOPRIO_CLASS_BE, 0) & !0x7f, - "restore moves thread back to IOPRIO_CLASS_BE" + io_priority(), + initial_io_priority, + "restore returns the thread to its exact prior IO priority" ); } diff --git a/crates/aft/tests/integration/callgraph_test.rs b/crates/aft/tests/integration/callgraph_test.rs index 7ba2e5287..4e6b8d269 100644 --- a/crates/aft/tests/integration/callgraph_test.rs +++ b/crates/aft/tests/integration/callgraph_test.rs @@ -22,13 +22,22 @@ fn configure_project(aft: &mut AftProcess, root: &Path) { /// The binary correctly treats linked worktrees as read-only, while these /// tests need a writer-capable synthetic project for cold-build assertions. fn callgraph_fixture() -> (TempDir, PathBuf) { + fn copy_dir_recursive(source: &Path, destination: &Path) { + fs::create_dir_all(destination).expect("create callgraph fixture directory"); + for entry in fs::read_dir(source).expect("read callgraph fixture") { + let entry = entry.expect("read callgraph fixture entry"); + let target = destination.join(entry.file_name()); + if entry.file_type().expect("read fixture entry type").is_dir() { + copy_dir_recursive(&entry.path(), &target); + } else { + fs::copy(entry.path(), target).expect("copy callgraph fixture file"); + } + } + } + let source = fixture_path("callgraph"); let temp = tempdir().expect("create callgraph fixture copy"); - for entry in fs::read_dir(source).expect("read callgraph fixture") { - let entry = entry.expect("read callgraph fixture entry"); - fs::copy(entry.path(), temp.path().join(entry.file_name())) - .expect("copy callgraph fixture file"); - } + copy_dir_recursive(&source, temp.path()); let root = temp.path().to_path_buf(); (temp, root) } diff --git a/docs/config.md b/docs/config.md index 76edf5154..51c9454d7 100644 --- a/docs/config.md +++ b/docs/config.md @@ -133,13 +133,13 @@ The backup store treats its on-disk tree as authoritative across processes; dele // Default: false "search_index": false, - // Background index admission policy. Default: "balanced". + // Background index admission policy. USER-only. Default: "balanced". // "balanced" pauses new standing-root slices on battery saving, memory or I/O // pressure, and resumes only after consecutive healthy samples. "performance" // ignores battery and pressure admission while retaining bounded concurrency, // fair root rotation, slice checkpoints, and background OS thread priority. "index": { - "resource_policy": "balanced" // "balanced" | "performance" + "resource_policy": "balanced" // "balanced" | "performance" }, // Linked-worktree RAM overlay for the trigram index. Default: false. From 1654923c5c4d4e441f803ac8dffb78905b5996bf Mon Sep 17 00:00:00 2001 From: Naadir Jeewa Date: Mon, 31 Aug 2026 21:11:43 +0100 Subject: [PATCH 12/14] fix(index): fence resumable scheduler work Signed-off-by: Naadir Jeewa Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/checkpoint.rs | 93 +++++++++++++------ crates/aft/src/executor/tests.rs | 65 +++++++++---- crates/aft/src/search_index.rs | 31 +++++-- crates/aft/src/semantic_index.rs | 62 ++++++++++--- crates/aft/src/standing_scheduler.rs | 66 ++++++++++--- crates/aft/src/subc/mod.rs | 78 ++++++++++++++-- crates/aft/src/subc/standing.rs | 76 +++++++++++++-- crates/aft/src/thread_priority.rs | 16 +++- .../aft/tests/integration/subc_storm_test.rs | 7 +- packages/aft-bridge/src/subc-transport.ts | 6 +- .../pi-plugin/src/__tests__/_shared.test.ts | 4 +- .../pi-plugin/src/__tests__/config.test.ts | 52 +++++------ packages/pi-plugin/src/tools/_shared.ts | 5 +- packages/pi-plugin/src/tools/fs.ts | 32 +++++-- 14 files changed, 442 insertions(+), 151 deletions(-) diff --git a/crates/aft/src/checkpoint.rs b/crates/aft/src/checkpoint.rs index 58a890908..34850001e 100644 --- a/crates/aft/src/checkpoint.rs +++ b/crates/aft/src/checkpoint.rs @@ -21,6 +21,8 @@ const MAX_NAMED_CHECKPOINTS_PER_SESSION: usize = 20; /// work interruptions without becoming permanent storage. const NAMED_CHECKPOINT_RETENTION_DAYS: u64 = 14; const NAMED_CHECKPOINT_RETENTION_SECS: u64 = NAMED_CHECKPOINT_RETENTION_DAYS * 24 * 60 * 60; +const CHECKPOINT_SCOPE_MARKER: &str = ".last-used"; +const CHECKPOINT_SCOPE_GC_LOCK: &str = ".scope-gc.lock"; const CHECKPOINT_SCHEMA_VERSION: u32 = 1; const UNBOUND_HARNESS_SEGMENT: &str = "unbound"; @@ -323,37 +325,45 @@ impl CheckpointStore { fn acquire_mutation_lock(&self) -> Result { let scope_dir = self.lock_path.parent().map(Path::to_path_buf); + let scopes_dir = scope_dir.as_deref().and_then(Path::parent); + if let Some(parent) = scopes_dir { + fs::create_dir_all(parent).map_err(|error| AftError::IoError { + path: parent.display().to_string(), + message: format!("failed to create checkpoint scopes directory: {error}"), + })?; + } + // Serialize scope creation with stale-scope reclamation. This closes + // the create_dir_all-to-lock race without retaining empty scopes forever. + let _gc_lock = scopes_dir + .map(|parent| fs_lock::try_acquire(&parent.join(CHECKPOINT_SCOPE_GC_LOCK), self.lock_timeout)) + .transpose() + .map_err(|error| AftError::IoError { + path: scopes_dir.unwrap_or(Path::new(".")).display().to_string(), + message: format!("failed to acquire checkpoint scope maintenance lock: {error}"), + })?; if let Some(parent) = scope_dir.as_deref() { fs::create_dir_all(parent).map_err(|error| AftError::IoError { path: parent.display().to_string(), message: format!("failed to create checkpoint lock directory: {error}"), })?; + let marker = parent.join(CHECKPOINT_SCOPE_MARKER); + fs::write(&marker, current_timestamp().to_string()).map_err(|error| AftError::IoError { + path: marker.display().to_string(), + message: format!("failed to refresh checkpoint scope marker: {error}"), + })?; } - let acquire_result = match fs_lock::try_acquire(&self.lock_path, self.lock_timeout) { - // A releasing peer removes the empty lock scope after its heartbeat - // exits. It can win the tiny interval after our create_dir_all and - // before lock creation, so recreate once and retry the acquisition. - Err(fs_lock::AcquireError::Io(error)) if error.kind() == io::ErrorKind::NotFound => { - if let Some(parent) = scope_dir.as_deref() { - fs::create_dir_all(parent).map_err(|error| AftError::IoError { - path: parent.display().to_string(), - message: format!("failed to recreate checkpoint lock directory: {error}"), - })?; - } - fs_lock::try_acquire(&self.lock_path, self.lock_timeout) + let guard = fs_lock::try_acquire(&self.lock_path, self.lock_timeout).map_err(|error| { + match error { + fs_lock::AcquireError::Timeout => AftError::IoError { + path: self.lock_path.display().to_string(), + message: "timed out acquiring checkpoint mutation lock".to_string(), + }, + fs_lock::AcquireError::Io(error) => AftError::IoError { + path: self.lock_path.display().to_string(), + message: format!("failed to acquire checkpoint mutation lock: {error}"), + }, } - result => result, - }; - let guard = acquire_result.map_err(|error| match error { - fs_lock::AcquireError::Timeout => AftError::IoError { - path: self.lock_path.display().to_string(), - message: "timed out acquiring checkpoint mutation lock".to_string(), - }, - fs_lock::AcquireError::Io(error) => AftError::IoError { - path: self.lock_path.display().to_string(), - message: format!("failed to acquire checkpoint mutation lock: {error}"), - }, })?; Ok(CheckpointLockGuard { guard: Some(guard) }) @@ -727,10 +737,9 @@ impl CheckpointStore { if let Some(checkpoints_dir) = self.durable_checkpoints_dir() { sweep_expired_durable_checkpoints(&checkpoints_dir, now); } - // Empty scope dirs are intentionally retained: another process may - // hold a scope-dir lock between create_dir_all and its first - // checkpoint write, so sweeping "empty" dirs deletes concurrently - // in-use scopes. Real data expires via sweep_expired_durable_checkpoints. + if let Some(storage_dir) = self.storage_dir.as_ref() { + sweep_expired_checkpoint_scopes(&storage_dir.join("checkpoints"), now); + } Ok(()) } @@ -922,6 +931,36 @@ impl CheckpointStore { } } +fn sweep_expired_checkpoint_scopes(scopes_dir: &Path, now: u64) { + let Ok(_gc_lock) = fs_lock::try_acquire_once(&scopes_dir.join(CHECKPOINT_SCOPE_GC_LOCK)) else { + return; + }; + let Ok(entries) = fs::read_dir(scopes_dir) else { + return; + }; + for entry in entries.flatten() { + let scope = entry.path(); + if !entry.file_type().is_ok_and(|kind| kind.is_dir()) { + continue; + } + let marker = scope.join(CHECKPOINT_SCOPE_MARKER); + let used_at = fs::read_to_string(&marker) + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(now); + if now.saturating_sub(used_at) < NAMED_CHECKPOINT_RETENTION_SECS { + continue; + } + let lock_path = scope.join("checkpoint.lock"); + let Ok(lock) = fs_lock::try_acquire_once(&lock_path) else { + continue; + }; + drop(lock); + let _ = fs::remove_file(marker); + let _ = fs::remove_dir(scope); + } +} + fn migrate_unbound_checkpoint_namespace(storage_dir: &Path, harness: &str) { let source = storage_dir .join(UNBOUND_HARNESS_SEGMENT) diff --git a/crates/aft/src/executor/tests.rs b/crates/aft/src/executor/tests.rs index bd1af4c76..96cdc2aee 100644 --- a/crates/aft/src/executor/tests.rs +++ b/crates/aft/src/executor/tests.rs @@ -2579,23 +2579,39 @@ fn interactive_queue_cap_returns_typed_backpressure_per_actor_and_global() { executor.register_actor(root_b.clone(), test_ctx()); executor.register_actor(root_c.clone(), test_ctx()); - let (started_tx, started_rx) = crossbeam_channel::bounded(1); - let (release_tx, release_rx) = crossbeam_channel::bounded(1); - let blocker = executor.submit( + let (started_tx, started_rx) = crossbeam_channel::bounded(2); + let (release_tx, release_rx) = crossbeam_channel::bounded(2); + let blocker_a_started = started_tx.clone(); + let blocker_a_release = release_rx.clone(); + let blocker_a = executor.submit( root_a.clone(), Lane::Mutating, - "interactive-cap-blocker".to_string(), + "interactive-cap-blocker-a".to_string(), + Box::new(move |_| { + blocker_a_started.send(()).expect("signal blocker a start"); + blocker_a_release + .recv_timeout(Duration::from_secs(5)) + .expect("release blocker a"); + ok("interactive-cap-blocker-a") + }), + ); + let blocker_b = executor.submit( + root_b.clone(), + Lane::Mutating, + "interactive-cap-blocker-b".to_string(), Box::new(move |_| { - started_tx.send(()).expect("signal blocker start"); + started_tx.send(()).expect("signal blocker b start"); release_rx .recv_timeout(Duration::from_secs(5)) - .expect("release blocker"); - ok("interactive-cap-blocker") + .expect("release blocker b"); + ok("interactive-cap-blocker-b") }), ); - started_rx - .recv_timeout(Duration::from_secs(2)) - .expect("blocker starts"); + for _ in 0..2 { + started_rx + .recv_timeout(Duration::from_secs(2)) + .expect("both effective workers start blockers"); + } let queued_a = executor.submit_async( root_a.clone(), @@ -2630,13 +2646,10 @@ fn interactive_queue_cap_returns_typed_backpressure_per_actor_and_global() { assert!(!response.success); assert_eq!(response.data["queue_scope"], "actor"); - release_tx.send(()).expect("release blocker"); - assert!( - blocker - .recv_timeout(Duration::from_secs(5)) - .unwrap() - .success - ); + release_tx.send(()).expect("release blocker a"); + release_tx.send(()).expect("release blocker b"); + assert!(blocker_a.recv_timeout(Duration::from_secs(5)).unwrap().success); + assert!(blocker_b.recv_timeout(Duration::from_secs(5)).unwrap().success); assert!(recv_async(queued_a, "queued a").success); assert!(recv_async(queued_b, "queued b").success); } @@ -2746,16 +2759,30 @@ fn queue_accounting_tracks_dispatch_cancellation_and_actor_retirement() { .expect("liveness snapshot after cancel"); assert_eq!(snapshot.interactive.queued, 0); - release_tx.send(()).expect("release blocker"); - assert!(recv_async(blocker, "blocker completion").success); + let retirement_rx = executor.submit_async( + root.clone(), + Lane::PureRead, + "accounting-retirement-queued".to_string(), + Box::new(|_| ok("accounting-retirement-queued")), + ); + let snapshot = executor + .try_dispatch_liveness_snapshot() + .expect("liveness before actor retirement"); + assert_eq!(snapshot.interactive.queued, 1); executor.remove_actor(&root); + let retirement_response = recv_async(retirement_rx, "retired queued completion"); + assert!(!retirement_response.success); + assert_eq!(retirement_response.data["code"], "actor_fatal"); let snapshot = executor .try_dispatch_liveness_snapshot() .expect("liveness after removal"); assert_eq!(snapshot.interactive.queued, 0); assert_eq!(snapshot.maintenance.queued, 0); + release_tx.send(()).expect("release blocker"); + assert!(recv_async(blocker, "blocker completion").success); + let (_replacement_dir, replacement_root) = test_root("accounting-replacement"); executor.register_actor(replacement_root.clone(), test_ctx()); assert!( diff --git a/crates/aft/src/search_index.rs b/crates/aft/src/search_index.rs index 011cf6a37..a33ceff1e 100644 --- a/crates/aft/src/search_index.rs +++ b/crates/aft/src/search_index.rs @@ -59,7 +59,7 @@ static TRANSIENT_SEARCH_CACHE_SWEEP_CURSORS: OnceLock>>>> = OnceLock::new(); -const SEARCH_STAGING_VERSION: u32 = 1; +const SEARCH_STAGING_VERSION: u32 = 2; const SEARCH_STAGING_MANIFEST: &str = "search-staging-v1.json"; const SEARCH_STAGING_DIR: &str = "search-staging-v1"; const SEARCH_SLICE_FILES: usize = 32; @@ -79,6 +79,7 @@ struct SearchStagingManifest { max_file_size: u64, paths: Vec, cursor: usize, + validation_cursor: usize, spill_seq: usize, files: Vec, } @@ -957,6 +958,7 @@ impl SearchIndex { && manifest.canonical_root == canonical_root && manifest.cursor <= manifest.paths.len() && manifest.files.len() == manifest.cursor + && manifest.validation_cursor <= manifest.files.len() }); // Fresh start (or structurally invalid staging): walk and fingerprint // the full corpus once to seed the manifest. Mid-build slices skip the @@ -999,6 +1001,7 @@ impl SearchIndex { max_file_size, paths, cursor: 0, + validation_cursor: 0, spill_seq: 0, files: Vec::new(), } @@ -1076,10 +1079,19 @@ impl SearchIndex { return Ok(SearchBuildSliceOutcome::Yielded); } - // Publication slice: the corpus fingerprint only covers path, size, - // and mtime, so an edit that preserves both would silently publish - // stale postings. Re-hash every staged included file and restart the - // build when any content no longer matches its staged hash. + // Publication validation is itself sliced. Re-hash only one bounded + // file window per turn so large roots cannot monopolize a cold slot. + if manifest.validation_cursor < manifest.files.len() { + let end = (manifest.validation_cursor + slice_size).min(manifest.files.len()); + if !staged_contents_match_disk(&manifest.files[manifest.validation_cursor..end]) { + let _ = fs::remove_dir_all(&staging_dir); + return Ok(SearchBuildSliceOutcome::Yielded); + } + manifest.validation_cursor = end; + write_search_staging_manifest(&manifest_path, &manifest)?; + return Ok(SearchBuildSliceOutcome::Yielded); + } + let ignore_fingerprint = ignore_rules_fingerprint(&canonical_root); let filters = PathFilters::default(); let paths = walk_project_files(&canonical_root, &filters); @@ -1088,8 +1100,7 @@ impl SearchIndex { let corpus_unchanged = manifest.corpus_fingerprint == corpus_fingerprint && manifest.ignore_fingerprint == ignore_fingerprint && manifest.max_file_size == max_file_size - && manifest.paths == paths - && staged_contents_match_disk(&manifest.files); + && manifest.paths == paths; if !corpus_unchanged { let _ = fs::remove_dir_all(&staging_dir); return Ok(SearchBuildSliceOutcome::Yielded); @@ -8633,10 +8644,10 @@ mod tests { let project = dir.path().join("project"); let cache = dir.path().join("cache"); fs::create_dir_all(&project).expect("create project"); - // Two files so the build spans two slices of one file each: the - // first slice stages stale content under a valid corpus fingerprint. - fs::write(project.join("a.rs"), "fn stale() {}\n").expect("write a"); + // Write b first so a has the newest mtime and is deterministically + // selected by the newest-first first slice. fs::write(project.join("b.rs"), "fn stable() {}\n").expect("write b"); + fs::write(project.join("a.rs"), "fn stale() {}\n").expect("write a"); assert_eq!( SearchIndex::resume_cold_build_slice_sized(&project, DEFAULT_MAX_FILE_SIZE, &cache, 1,) .expect("first slice"), diff --git a/crates/aft/src/semantic_index.rs b/crates/aft/src/semantic_index.rs index 4aa8ec1dd..d7876771d 100644 --- a/crates/aft/src/semantic_index.rs +++ b/crates/aft/src/semantic_index.rs @@ -2105,22 +2105,43 @@ fn append_semantic_chunks(path: &Path, chunks: &[SemanticChunk]) -> Result<(), S file.sync_data().map_err(|error| error.to_string()) } -fn read_semantic_chunks(path: &Path, count: usize) -> Result, String> { - let text = fs::read_to_string(path).map_err(|error| error.to_string())?; - let chunks = text - .lines() - .filter(|line| !line.is_empty()) - .map(|line| serde_json::from_str(line).map_err(|error| error.to_string())) - .collect::, _>>()?; - if chunks.len() != count { +fn read_semantic_chunk_range( + path: &Path, + start: usize, + end: usize, + total: usize, +) -> Result, String> { + if start > end || end > total { + return Err("semantic staging chunk range is invalid".to_string()); + } + if total == 0 { + return Ok(Vec::new()); + } + let file = fs::File::open(path).map_err(|error| error.to_string())?; + let mut chunks = Vec::with_capacity(end - start); + let mut count = 0usize; + for line in std::io::BufRead::lines(BufReader::new(file)) { + let line = line.map_err(|error| error.to_string())?; + if line.is_empty() { + continue; + } + if count >= start && count < end { + chunks.push(serde_json::from_str(&line).map_err(|error| error.to_string())?); + } + count = count.saturating_add(1); + } + if count != total { return Err(format!( - "semantic staging chunk segment count mismatch: expected {count}, got {}", - chunks.len() + "semantic staging chunk segment count mismatch: expected {total}, got {count}" )); } Ok(chunks) } +fn read_semantic_chunks(path: &Path, count: usize) -> Result, String> { + read_semantic_chunk_range(path, 0, count, count) +} + fn append_semantic_vectors( path: &Path, vectors: &[Vec], @@ -2149,6 +2170,9 @@ fn read_semantic_vectors( count: usize, dimension: usize, ) -> Result>, String> { + if count == 0 { + return Ok(Vec::new()); + } let bytes = fs::read(path).map_err(|error| error.to_string())?; let record_bytes = dimension .checked_mul(std::mem::size_of::()) @@ -3017,10 +3041,15 @@ impl SemanticIndex { } if manifest.embed_cursor < manifest.chunks_count { - let chunks = read_semantic_chunks(&chunks_path, manifest.chunks_count)?; let end = (manifest.embed_cursor + model.max_batch_size().max(1)).min(manifest.chunks_count); - let texts = chunks[manifest.embed_cursor..end] + let chunks = read_semantic_chunk_range( + &chunks_path, + manifest.embed_cursor, + end, + manifest.chunks_count, + )?; + let texts = chunks .iter() .map(|chunk| chunk.embed_text.clone()) .collect(); @@ -3097,8 +3126,13 @@ impl SemanticIndex { return Err("failed to publish resumable semantic index".to_string()); } fs::remove_file(&staging_path).map_err(|error| error.to_string())?; - fs::remove_file(&chunks_path).map_err(|error| error.to_string())?; - fs::remove_file(&vectors_path).map_err(|error| error.to_string())?; + for path in [&chunks_path, &vectors_path] { + if let Err(error) = fs::remove_file(path) { + if error.kind() != std::io::ErrorKind::NotFound { + return Err(error.to_string()); + } + } + } Ok(SemanticBuildSliceOutcome::Complete) } diff --git a/crates/aft/src/standing_scheduler.rs b/crates/aft/src/standing_scheduler.rs index 7e0f014af..32b69206a 100644 --- a/crates/aft/src/standing_scheduler.rs +++ b/crates/aft/src/standing_scheduler.rs @@ -6,8 +6,9 @@ pub struct DeficitRoundRobin { quantum: u64, queue: VecDeque, deficits: HashMap, - in_flight: HashSet, + in_flight: HashMap, generations: HashMap, + next_generation: u64, } impl DeficitRoundRobin @@ -20,8 +21,9 @@ where quantum, queue: VecDeque::new(), deficits: HashMap::new(), - in_flight: HashSet::new(), + in_flight: HashMap::new(), generations: HashMap::new(), + next_generation: 0, } } @@ -33,14 +35,14 @@ where let active = keys.iter().cloned().collect::>(); self.queue.retain(|key| active.contains(key)); self.deficits.retain(|key, _| active.contains(key)); - self.in_flight.retain(|key| active.contains(key)); + // Keep in-flight generations until their completion arrives. A root + // removed and re-added while a slice runs must not dispatch twice. for key in keys { if !self.deficits.contains_key(&key) { self.deficits.insert(key.clone(), 0); - self.generations - .entry(key.clone()) - .and_modify(|generation| *generation = generation.saturating_add(1)) - .or_insert(0); + let generation = self.next_generation; + self.next_generation = self.next_generation.saturating_add(1); + self.generations.insert(key.clone(), generation); self.queue.push_back(key); } } @@ -59,8 +61,9 @@ where continue; }; *deficit += i128::from(self.quantum); - if *deficit >= 0 { - self.in_flight.insert(key.clone()); + if *deficit >= 0 && !self.in_flight.contains_key(&key) { + let generation = self.generations.get(&key).copied().unwrap_or(0); + self.in_flight.insert(key.clone(), generation); return Some(key); } self.queue.push_back(key); @@ -74,28 +77,51 @@ where .then(|| self.generations.get(key).copied().unwrap_or(0)) } + pub fn reconfigure(&mut self, key: &K) { + if !self.deficits.contains_key(key) { + return; + } + let generation = self.next_generation; + self.next_generation = self.next_generation.saturating_add(1); + self.generations.insert(key.clone(), generation); + } + pub fn complete_generation(&mut self, key: K, generation: u64, cost: u64, has_more: bool) { + if self.in_flight.get(&key).copied() != Some(generation) { + return; + } + self.in_flight.remove(&key); if self.generation(&key) != Some(generation) { + if self.deficits.contains_key(&key) { + if !self.queue.contains(&key) { + self.queue.push_back(key); + } + } else { + self.generations.remove(&key); + } return; } - self.complete(key, cost, has_more); + self.complete_active(key, cost, has_more); } pub fn complete(&mut self, key: K, cost: u64, has_more: bool) { - if !self.in_flight.remove(&key) { + if self.in_flight.remove(&key).is_none() { return; } + self.complete_active(key, cost, has_more); + } + + fn complete_active(&mut self, key: K, cost: u64, has_more: bool) { if let Some(deficit) = self.deficits.get_mut(&key) { *deficit -= i128::from(cost); } if has_more { - // Only requeue roots that still have a deficits entry; a - // reconcile may have removed the root while its slice ran. if self.deficits.contains_key(&key) { self.queue.push_back(key); } } else { self.deficits.remove(&key); + self.generations.remove(&key); } } @@ -245,4 +271,18 @@ mod tests { assert_eq!(scheduler.generation(&"a"), Some(new_generation)); assert!(scheduler.deficits.contains_key("a")); } + + #[test] + fn stale_completion_prunes_removed_generation_tombstone() { + let mut scheduler = DeficitRoundRobin::new(100); + scheduler.reconcile(["a", "b"]); + assert_eq!(scheduler.next(), Some("a")); + let generation = scheduler.generation(&"a").unwrap(); + scheduler.reconcile(["b"]); + + scheduler.complete_generation("a", generation, 100, true); + + assert!(!scheduler.generations.contains_key("a")); + assert!(!scheduler.in_flight.contains_key("a")); + } } diff --git a/crates/aft/src/subc/mod.rs b/crates/aft/src/subc/mod.rs index 459094305..69bd13210 100644 --- a/crates/aft/src/subc/mod.rs +++ b/crates/aft/src/subc/mod.rs @@ -3893,15 +3893,27 @@ async fn recv_prioritized_frame( }; } if lane.consecutive_control >= CONTROL_BURST_LIMIT { - // Bound the control burst: after CONTROL_BURST_LIMIT consecutive - // control frames, await data first so sustained control traffic - // (pings/acks) cannot starve data frames. + // Prefer one ready data frame after a control burst, but keep + // polling control so an idle data lane cannot block heartbeats. lane.consecutive_control = 0; - match data_rx.recv().await { - Some(frame) => return Some(frame), - None => { - lane.data_closed = true; - continue; + tokio::select! { + biased; + frame = data_rx.recv() => match frame { + Some(frame) => return Some(frame), + None => { + lane.data_closed = true; + continue; + } + }, + frame = control_rx.recv() => match frame { + Some(frame) => { + lane.consecutive_control = 1; + return Some(frame); + } + None => { + lane.control_closed = true; + continue; + } } } } @@ -6878,6 +6890,56 @@ mod tests { assert_eq!(frame.frame.header.ty, FrameType::Request); } + #[tokio::test] + async fn control_lane_remains_live_after_burst_when_data_is_idle() { + let (control_tx, mut control_rx) = mpsc::channel(CONTROL_BURST_LIMIT + 1); + let (_data_tx, mut data_rx) = mpsc::channel(1); + for seq in 0..CONTROL_BURST_LIMIT { + control_tx + .send(Ok(DecodedFrame { + frame: Frame::build( + FrameType::Ping, + control_flags(), + 0, + 0, + seq as u64, + Vec::new(), + ) + .unwrap(), + phase_trace: PhaseTrace::new(Instant::now()), + })) + .await + .unwrap(); + } + + let mut lane = PrioritizedFrameLane::default(); + for _ in 0..CONTROL_BURST_LIMIT { + let frame = recv_prioritized_frame(&mut control_rx, &mut data_rx, &mut lane) + .await + .unwrap() + .unwrap(); + assert_eq!(frame.frame.header.ty, FrameType::Ping); + } + + control_tx + .send(Ok(DecodedFrame { + frame: Frame::build(FrameType::Ping, control_flags(), 0, 0, 99, Vec::new()) + .unwrap(), + phase_trace: PhaseTrace::new(Instant::now()), + })) + .await + .unwrap(); + let frame = tokio::time::timeout( + Duration::from_millis(100), + recv_prioritized_frame(&mut control_rx, &mut data_rx, &mut lane), + ) + .await + .expect("control traffic must remain live while the data lane is idle") + .unwrap() + .unwrap(); + assert_eq!(frame.frame.header.ty, FrameType::Ping); + } + #[tokio::test] async fn closed_control_lane_drains_buffered_data_before_eof() { let (control_tx, mut control_rx) = mpsc::channel(1); diff --git a/crates/aft/src/subc/standing.rs b/crates/aft/src/subc/standing.rs index 03ad5f2d3..23fb1a053 100644 --- a/crates/aft/src/subc/standing.rs +++ b/crates/aft/src/subc/standing.rs @@ -21,6 +21,12 @@ struct SemanticBuildCache { model: CachedSemanticModel, } +#[derive(Clone)] +struct CallgraphBuildCache { + resolved_target: PathBuf, + files: Arc>, +} + use crate::config::{Config, IndexKind}; use crate::context::{App, AppContext}; use crate::executor::{Executor, Lane, MaintenanceCoalesceKey}; @@ -103,6 +109,7 @@ pub(super) struct StandingActor { owned_actors: Mutex>, schedule: Mutex, semantic_cache: Arc>>, + callgraph_cache: Arc>>, } impl StandingActor { @@ -116,6 +123,7 @@ impl StandingActor { owned_actors: Mutex::new(HashMap::new()), schedule: Mutex::new(StandingScheduleState::default()), semantic_cache: Arc::new(Mutex::new(HashMap::new())), + callgraph_cache: Arc::new(Mutex::new(HashMap::new())), } } @@ -268,6 +276,7 @@ impl StandingActor { .get(&entry.literal_path) .is_some_and(|previous| previous.indexes != entry.indexes); if selection_changed { + schedule.queue.reconfigure(&entry.literal_path); schedule.next_kind.insert(entry.literal_path.clone(), 0); } else { schedule @@ -317,6 +326,26 @@ impl StandingActor { }) .collect::>(); for (key, generation, response, elapsed) in completed { + if schedule.queue.generation(&key) != Some(generation) { + if schedule + .pending + .get(&key) + .is_some_and(|pending| pending.generation == generation) + { + schedule.pending.remove(&key); + } + schedule + .queue + .complete_generation(key, generation, 1, true); + continue; + } + if schedule + .pending + .get(&key) + .is_none_or(|pending| pending.generation != generation) + { + continue; + } schedule.pending.remove(&key); let has_more = response .data @@ -428,7 +457,11 @@ impl StandingActor { fn retire_removed_actors(&self, removed: &[String]) { let mut owned = self.owned_actors.lock(); + let mut semantic_cache = self.semantic_cache.lock(); + let mut callgraph_cache = self.callgraph_cache.lock(); for literal_path in removed { + semantic_cache.remove(literal_path); + callgraph_cache.remove(literal_path); if let Some((root_id, owned_here)) = owned.remove(literal_path) { self.executor.cancel_queued_maintenance(&root_id); if let Some(ctx) = self.executor.actor_context(&root_id) { @@ -454,6 +487,7 @@ impl StandingActor { .get(&entry.literal_path) .unwrap_or(&0); let semantic_cache = Arc::clone(&self.semantic_cache); + let callgraph_cache = Arc::clone(&self.callgraph_cache); let selected = IndexKind::ALL .iter() .copied() @@ -517,6 +551,7 @@ impl StandingActor { &entry, &admission, permit.admission_epoch, + &callgraph_cache, ) }; if kind_complete { @@ -812,17 +847,19 @@ fn build_missing_semantic_after_strict_check( &storage_dir, &entry.artifact_key, ) - .is_ok() + .ok() }, ) .ok() + .flatten() .flatten(); - if outcome == Some(true) { - // Force a fresh inventory at the start of the next complete build. - cache.lock().remove(&entry.literal_path); - (true, false) - } else { - (false, true) + match outcome { + Some(crate::semantic_index::SemanticBuildSliceOutcome::Complete) => { + // Force a fresh inventory at the start of the next complete build. + cache.lock().remove(&entry.literal_path); + (true, false) + } + Some(crate::semantic_index::SemanticBuildSliceOutcome::Yielded) | None => (false, true), } } @@ -832,6 +869,7 @@ fn build_missing_callgraph_after_strict_check( entry: &StandingRootEntry, admission: &crate::standing_roots::StandingBuildAdmission, permit_epoch: u64, + cache: &Arc>>, ) -> (bool, bool) { if admission.cancellation_requested() || crate::executor::current_job_cancelled() { return (false, true); @@ -843,7 +881,24 @@ fn build_missing_callgraph_after_strict_check( let Some(storage_dir) = storage_dir else { return (false, true); }; - let files = crate::callgraph::walk_project_files(&entry.resolved_target).collect::>(); + let files = { + let mut cache = cache.lock(); + let needs_refresh = cache + .get(&entry.literal_path) + .is_none_or(|cached| cached.resolved_target != entry.resolved_target); + if needs_refresh { + let files = crate::callgraph::walk_project_files(&entry.resolved_target) + .collect::>(); + cache.insert( + entry.literal_path.clone(), + CallgraphBuildCache { + resolved_target: entry.resolved_target.clone(), + files: Arc::new(files), + }, + ); + } + Arc::clone(&cache.get(&entry.literal_path).expect("cache inserted").files) + }; let cache_dir = storage_dir.join("callgraph").join(&entry.artifact_key); let lease = match crate::root_cache::WriterLease::acquire_shared( crate::root_cache::RootCacheDomain::Callgraph, @@ -880,7 +935,10 @@ fn build_missing_callgraph_after_strict_check( .ok() .flatten(); match outcome { - Some(true) => (true, false), + Some(true) => { + cache.lock().remove(&entry.literal_path); + (true, false) + } Some(false) | None => (false, true), } } diff --git a/crates/aft/src/thread_priority.rs b/crates/aft/src/thread_priority.rs index 784747337..0c733d29a 100644 --- a/crates/aft/src/thread_priority.rs +++ b/crates/aft/src/thread_priority.rs @@ -59,12 +59,14 @@ mod imp { #[derive(Clone, Copy)] pub struct PreviousPriority { scheduler: c_int, + scheduler_param: libc::sched_param, io_priority: c_int, } pub fn demote() -> PreviousPriority { let previous = PreviousPriority { scheduler: unsafe { libc::sched_getscheduler(0) }, + scheduler_param: scheduler_param(), io_priority: unsafe { syscall(libc::SYS_ioprio_get, IOPRIO_WHO_PROCESS, tid()) as c_int }, @@ -75,7 +77,7 @@ mod imp { } pub fn restore(previous: PreviousPriority) { - cpu_policy(previous.scheduler); + cpu_policy(previous.scheduler, previous.scheduler_param); if !io_set(IOPRIO_WHO_PROCESS, tid(), previous.io_priority) { warn_once("io", &std::io::Error::last_os_error().to_string()); } @@ -103,10 +105,16 @@ mod imp { } } - fn cpu_policy(policy: c_int) { + fn scheduler_param() -> libc::sched_param { let mut param = unsafe { std::mem::zeroed::() }; - param.sched_priority = 0; - let rc = unsafe { libc::sched_setscheduler(0, policy, ¶m) }; + if unsafe { libc::sched_getparam(0, &mut param) } != 0 { + warn_once("cpu", &std::io::Error::last_os_error().to_string()); + } + param + } + + fn cpu_policy(policy: c_int, mut param: libc::sched_param) { + let rc = unsafe { libc::sched_setscheduler(0, policy, &mut param) }; if rc != 0 { warn_once("cpu", &std::io::Error::last_os_error().to_string()); } diff --git a/crates/aft/tests/integration/subc_storm_test.rs b/crates/aft/tests/integration/subc_storm_test.rs index 24fa8ac69..2d4fac39f 100644 --- a/crates/aft/tests/integration/subc_storm_test.rs +++ b/crates/aft/tests/integration/subc_storm_test.rs @@ -1740,10 +1740,9 @@ async fn drive_standing_yield_daemon(input: FakeDaemonInput) { request_id.clone(), Box::new(move |_| { // Mirrors the production standing pass shape: an immediate, - // non-waiting cold-build attempt. Whether the global - // limiter hands out a slot depends on concurrent module - // maintenance; the pass must finish promptly either way - // and never block a maintenance worker on cold admission. + // non-waiting cold-build attempt. This test holds both cold + // permits, so every pass must yield promptly and must never + // block a maintenance worker on cold admission. let permit = aft::cold_build_limiter::try_acquire(); let yielded = permit.is_none(); if yielded { diff --git a/packages/aft-bridge/src/subc-transport.ts b/packages/aft-bridge/src/subc-transport.ts index 5aa0a6893..2f02a3f8c 100644 --- a/packages/aft-bridge/src/subc-transport.ts +++ b/packages/aft-bridge/src/subc-transport.ts @@ -878,7 +878,8 @@ class SubcTransport implements AftProjectTransport { options.transportTimeoutMs ?? options.timeoutMs ?? this.pool.poolDefaultTimeoutMs; const executionDeadlineMs = options.executionDeadlineMs; const onProgress = options.onProgress - ? (body: Uint8Array) => options.onProgress?.({ kind: "stdout", text: new TextDecoder().decode(body) }) + ? (body: Uint8Array) => + options.onProgress?.({ kind: "stdout", text: new TextDecoder().decode(body) }) : undefined; return { preview, timeoutMs, executionDeadlineMs, onProgress }; } @@ -1689,7 +1690,8 @@ export class SubcTransportPool implements AftTransportPool { remaining !== undefined ? Math.max(1, Math.floor(remaining)) : timeoutMs; const requestedExecutionDeadline = body.deadline_ms_remaining; const serverDeadline = - typeof requestedExecutionDeadline === "number" && Number.isFinite(requestedExecutionDeadline) + typeof requestedExecutionDeadline === "number" && + Number.isFinite(requestedExecutionDeadline) ? Math.min(remaining ?? requestedExecutionDeadline, requestedExecutionDeadline) : remaining; const deadlineBody = diff --git a/packages/pi-plugin/src/__tests__/_shared.test.ts b/packages/pi-plugin/src/__tests__/_shared.test.ts index bc67e0454..84bc2f8c5 100644 --- a/packages/pi-plugin/src/__tests__/_shared.test.ts +++ b/packages/pi-plugin/src/__tests__/_shared.test.ts @@ -109,7 +109,7 @@ describe("tool shared helpers", () => { test("callToolCall emits visible progress while a tool remains pending", async () => { let release!: () => void; - const pending = new Promise(resolve => { + const pending = new Promise((resolve) => { release = resolve; }); const updates: unknown[] = []; @@ -119,7 +119,7 @@ describe("tool shared helpers", () => { }); const call = callToolCall(bridge, "inspect", {}, makeExtContext(), { - onUpdate: update => updates.push(update), + onUpdate: (update) => updates.push(update), progressIntervalMs: 5, }); await Bun.sleep(12); diff --git a/packages/pi-plugin/src/__tests__/config.test.ts b/packages/pi-plugin/src/__tests__/config.test.ts index 1d36100b4..d89acb6c6 100644 --- a/packages/pi-plugin/src/__tests__/config.test.ts +++ b/packages/pi-plugin/src/__tests__/config.test.ts @@ -66,33 +66,33 @@ afterEach(() => { tempRoots.clear(); }); - test("index resource policy defaults, validates, and remains user-only", () => { - expect(AftConfigSchema.parse({}).index?.resource_policy ?? "balanced").toBe("balanced"); - expect( - AftConfigSchema.parse({ index: { resource_policy: "balanced" } }).index?.resource_policy, - ).toBe("balanced"); - expect( - AftConfigSchema.parse({ index: { resource_policy: "performance" } }).index?.resource_policy, - ).toBe("performance"); - expect(AftConfigSchema.safeParse({ index: { resource_policy: "unlimited" } }).success).toBe( - false, - ); - - const fixture = createConfigFixture(); - writeFileSync( - fixture.userConfigPath, - JSON.stringify({ index: { resource_policy: "performance" } }), - ); - writeFileSync( - fixture.projectConfigPath, - JSON.stringify({ index: { resource_policy: "balanced" } }), - ); - const result = runConfigLoader(fixture.projectDirectory, { - HOME: fixture.home, - XDG_CONFIG_HOME: fixture.xdgConfigHome, - }); - expect(JSON.parse(result.stdout).index.resource_policy).toBe("performance"); +test("index resource policy defaults, validates, and remains user-only", () => { + expect(AftConfigSchema.parse({}).index?.resource_policy ?? "balanced").toBe("balanced"); + expect( + AftConfigSchema.parse({ index: { resource_policy: "balanced" } }).index?.resource_policy, + ).toBe("balanced"); + expect( + AftConfigSchema.parse({ index: { resource_policy: "performance" } }).index?.resource_policy, + ).toBe("performance"); + expect(AftConfigSchema.safeParse({ index: { resource_policy: "unlimited" } }).success).toBe( + false, + ); + + const fixture = createConfigFixture(); + writeFileSync( + fixture.userConfigPath, + JSON.stringify({ index: { resource_policy: "performance" } }), + ); + writeFileSync( + fixture.projectConfigPath, + JSON.stringify({ index: { resource_policy: "balanced" } }), + ); + const result = runConfigLoader(fixture.projectDirectory, { + HOME: fixture.home, + XDG_CONFIG_HOME: fixture.xdgConfigHome, }); + expect(JSON.parse(result.stdout).index.resource_policy).toBe("performance"); +}); describe("loadAftConfig", () => { test("gh_read honors only the user tier and warns for project overrides", () => { diff --git a/packages/pi-plugin/src/tools/_shared.ts b/packages/pi-plugin/src/tools/_shared.ts index f9034f60f..a9b5aec2e 100644 --- a/packages/pi-plugin/src/tools/_shared.ts +++ b/packages/pi-plugin/src/tools/_shared.ts @@ -11,11 +11,11 @@ import type { } from "@cortexkit/aft-bridge"; import { adaptToolError, + timeoutForCommand as bridgeTimeoutForCommand, formatBridgeErrorMessage, isBashTransportDeadError, prepareCanonicalEditArguments, prepareCanonicalPathArguments, - timeoutForCommand as bridgeTimeoutForCommand, } from "@cortexkit/aft-bridge"; import type { AgentToolResult, @@ -44,8 +44,7 @@ function piTransportOptions( options: BridgeRequestOptions = {}, ): BridgeRequestOptions { const { timeoutMs: callerTimeoutMs, ...rest } = options; - const requested = - rest.transportTimeoutMs ?? callerTimeoutMs ?? bridgeTimeoutForCommand(command); + const requested = rest.transportTimeoutMs ?? callerTimeoutMs ?? bridgeTimeoutForCommand(command); const transportTimeoutMs = Math.min( requested ?? PI_TOOL_TRANSPORT_TIMEOUT_MS, PI_TOOL_TRANSPORT_TIMEOUT_MS, diff --git a/packages/pi-plugin/src/tools/fs.ts b/packages/pi-plugin/src/tools/fs.ts index 035c25483..c4b9efc93 100644 --- a/packages/pi-plugin/src/tools/fs.ts +++ b/packages/pi-plugin/src/tools/fs.ts @@ -180,12 +180,18 @@ export function registerFsTools(pi: ExtensionAPI, ctx: PluginContext, surface: F const bridge = bridgeFor(ctx, extCtx.cwd); // Single batched call so every file shares one op_id; one // `aft_safety undo` then restores the whole delete atomically. - const response = await callToolCall(bridge, "delete", { - files, - // Coerce at the boundary, like `files`: a stringified "true" from the - // model must not silently drop the flag (see coerceBoolean). - recursive: coerceBoolean(params.recursive), - }, extCtx, { onUpdate }); + const response = await callToolCall( + bridge, + "delete", + { + files, + // Coerce at the boundary, like `files`: a stringified "true" from the + // model must not silently drop the flag (see coerceBoolean). + recursive: coerceBoolean(params.recursive), + }, + extCtx, + { onUpdate }, + ); if (response.success === false) { throw new Error(response.text || response.message || "delete failed"); } @@ -242,10 +248,16 @@ export function registerFsTools(pi: ExtensionAPI, ctx: PluginContext, surface: F } const bridge = bridgeFor(ctx, extCtx.cwd); - const response = await callToolCall(bridge, "move", { - filePath: params.path, - destination: params.destination, - }, extCtx, { onUpdate }); + const response = await callToolCall( + bridge, + "move", + { + filePath: params.path, + destination: params.destination, + }, + extCtx, + { onUpdate }, + ); if (response.success === false) { throw new Error(response.text || response.message || "move failed"); } From 49887111fc004755766496f7264dc30767399b07 Mon Sep 17 00:00:00 2001 From: Naadir Jeewa Date: Mon, 31 Aug 2026 22:51:18 +0100 Subject: [PATCH 13/14] fix(index): close fresh review findings Signed-off-by: Naadir Jeewa Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/checkpoint.rs | 110 +++++++++-- crates/aft/src/executor/mod.rs | 41 +++- crates/aft/src/executor/tests.rs | 129 +++++++++++- crates/aft/src/resource_policy.rs | 14 +- crates/aft/src/search_index.rs | 73 +++++++ crates/aft/src/semantic_index.rs | 135 +++++++++---- crates/aft/src/standing_roots.rs | 27 +++ crates/aft/src/subc/mod.rs | 187 +++++++++++++----- crates/aft/src/subc/standing.rs | 15 +- crates/aft/tests/integration/bash_pty_test.rs | 3 +- .../aft/tests/integration/subc_bridge_test.rs | 93 +++++++++ .../aft/tests/integration/subc_storm_test.rs | 10 +- .../src/__tests__/subc-transport.test.ts | 34 ++++ packages/aft-bridge/src/subc-transport.ts | 3 + .../pi-plugin/src/__tests__/config.test.ts | 3 +- packages/pi-plugin/src/tools/_shared.ts | 8 +- 16 files changed, 758 insertions(+), 127 deletions(-) diff --git a/crates/aft/src/checkpoint.rs b/crates/aft/src/checkpoint.rs index 34850001e..7239f18ad 100644 --- a/crates/aft/src/checkpoint.rs +++ b/crates/aft/src/checkpoint.rs @@ -334,8 +334,10 @@ impl CheckpointStore { } // Serialize scope creation with stale-scope reclamation. This closes // the create_dir_all-to-lock race without retaining empty scopes forever. - let _gc_lock = scopes_dir - .map(|parent| fs_lock::try_acquire(&parent.join(CHECKPOINT_SCOPE_GC_LOCK), self.lock_timeout)) + let gc_lock = scopes_dir + .map(|parent| { + fs_lock::try_acquire(&parent.join(CHECKPOINT_SCOPE_GC_LOCK), self.lock_timeout) + }) .transpose() .map_err(|error| AftError::IoError { path: scopes_dir.unwrap_or(Path::new(".")).display().to_string(), @@ -347,24 +349,30 @@ impl CheckpointStore { message: format!("failed to create checkpoint lock directory: {error}"), })?; let marker = parent.join(CHECKPOINT_SCOPE_MARKER); - fs::write(&marker, current_timestamp().to_string()).map_err(|error| AftError::IoError { - path: marker.display().to_string(), - message: format!("failed to refresh checkpoint scope marker: {error}"), + fs::write(&marker, current_timestamp().to_string()).map_err(|error| { + AftError::IoError { + path: marker.display().to_string(), + message: format!("failed to refresh checkpoint scope marker: {error}"), + } })?; } - - let guard = fs_lock::try_acquire(&self.lock_path, self.lock_timeout).map_err(|error| { - match error { - fs_lock::AcquireError::Timeout => AftError::IoError { - path: self.lock_path.display().to_string(), - message: "timed out acquiring checkpoint mutation lock".to_string(), - }, - fs_lock::AcquireError::Io(error) => AftError::IoError { - path: self.lock_path.display().to_string(), - message: format!("failed to acquire checkpoint mutation lock: {error}"), + // The GC lock protects scope creation only. Release it before waiting + // for the per-scope mutation lock so unrelated scopes can progress. + drop(gc_lock); + + let guard = + fs_lock::try_acquire(&self.lock_path, self.lock_timeout).map_err( + |error| match error { + fs_lock::AcquireError::Timeout => AftError::IoError { + path: self.lock_path.display().to_string(), + message: "timed out acquiring checkpoint mutation lock".to_string(), + }, + fs_lock::AcquireError::Io(error) => AftError::IoError { + path: self.lock_path.display().to_string(), + message: format!("failed to acquire checkpoint mutation lock: {error}"), + }, }, - } - })?; + )?; Ok(CheckpointLockGuard { guard: Some(guard) }) } @@ -708,6 +716,13 @@ impl CheckpointStore { .map(|session_dir| session_dir.join(name)) } + fn checkpoint_scopes_dir(&self) -> Option { + let scope_dir = self.lock_path.parent()?; + let scopes_dir = scope_dir.parent()?; + (scopes_dir.file_name().and_then(|name| name.to_str()) == Some("checkpoints")) + .then(|| scopes_dir.to_path_buf()) + } + fn run_process_maintenance_once_locked(&mut self) -> Result<(), AftError> { let Some(storage_dir) = self.storage_dir.clone() else { return Ok(()); @@ -737,8 +752,8 @@ impl CheckpointStore { if let Some(checkpoints_dir) = self.durable_checkpoints_dir() { sweep_expired_durable_checkpoints(&checkpoints_dir, now); } - if let Some(storage_dir) = self.storage_dir.as_ref() { - sweep_expired_checkpoint_scopes(&storage_dir.join("checkpoints"), now); + if let Some(scopes_dir) = self.checkpoint_scopes_dir() { + sweep_expired_checkpoint_scopes(&scopes_dir, now); } Ok(()) } @@ -1955,6 +1970,63 @@ mod tests { "released lock scope must remain durable" ); } + #[test] + fn mutation_wait_releases_scope_gc_lock() { + let dir = tempfile::tempdir().unwrap(); + let checkpoints_root = dir.path().join("checkpoints"); + let scope_dir = checkpoints_root.join("project-scope"); + fs::create_dir_all(&scope_dir).unwrap(); + let lock_path = scope_dir.join("checkpoint.lock"); + let held_scope_lock = fs_lock::try_acquire_once(&lock_path).unwrap(); + let store = CheckpointStore::with_lock_path(lock_path, Duration::from_secs(2)); + + let waiter = std::thread::spawn(move || store.acquire_mutation_lock()); + let marker = scope_dir.join(CHECKPOINT_SCOPE_MARKER); + for _ in 0..100 { + if marker.exists() { + break; + } + std::thread::sleep(Duration::from_millis(5)); + } + assert!(marker.exists(), "waiter must finish scope creation"); + let gc_lock = fs_lock::try_acquire_once(&checkpoints_root.join(CHECKPOINT_SCOPE_GC_LOCK)); + assert!( + gc_lock.is_ok(), + "a waiter on one scope must not retain the global GC lock" + ); + + drop(gc_lock); + drop(held_scope_lock); + assert!(waiter.join().unwrap().is_ok()); + } + + #[test] + fn cleanup_sweeps_the_lock_paths_checkpoint_scope_root() { + let dir = tempfile::tempdir().unwrap(); + let checkpoints_root = dir.path().join("checkpoints"); + let expired_scope = checkpoints_root.join("expired-scope"); + fs::create_dir_all(&expired_scope).unwrap(); + fs::write( + expired_scope.join(CHECKPOINT_SCOPE_MARKER), + current_timestamp() + .saturating_sub(NAMED_CHECKPOINT_RETENTION_SECS) + .saturating_sub(1) + .to_string(), + ) + .unwrap(); + let lock_path = checkpoints_root + .join("current-scope") + .join("checkpoint.lock"); + let mut store = CheckpointStore::with_lock_path(lock_path, CHECKPOINT_LOCK_TIMEOUT); + store.storage_dir = Some(dir.path().join("different-storage-root")); + + store.cleanup_locked().unwrap(); + + assert!( + !expired_scope.exists(), + "cleanup must derive the scope root from the mutation lock path" + ); + } #[test] fn cleanup_removes_expired_across_sessions() { diff --git a/crates/aft/src/executor/mod.rs b/crates/aft/src/executor/mod.rs index 17c4e0e59..20418f38f 100644 --- a/crates/aft/src/executor/mod.rs +++ b/crates/aft/src/executor/mod.rs @@ -1606,6 +1606,13 @@ impl SchedulerState { pruned } + fn next_queued_deadline(&self) -> Option { + self.actors + .values() + .filter_map(|actor| actor.interactive.earliest_deadline()) + .min() + } + fn dispatch_liveness_snapshot(&self) -> DispatchLivenessSnapshot { let now = Instant::now(); let mut interactive = QueueSnapshotAccumulator::default(); @@ -2050,6 +2057,19 @@ impl ClassQueues { }) } + fn earliest_deadline(&self) -> Option { + [ + &self.pure_reads, + &self.lsp_status, + &self.heavy_init, + &self.mutating, + &self.maintenance_commit, + ] + .into_iter() + .flat_map(|queue| queue.iter().filter_map(|job| job.deadline)) + .min() + } + /// Remove interactive jobs whose queue deadline has elapsed, preserving /// survivor order in both the ladder and the lane queues. fn prune_elapsed(&mut self, now: Instant) -> Vec { @@ -2421,7 +2441,22 @@ fn scheduler_loop( dispatch_liveness: Arc, ) { let mut expired_completions: Vec = Vec::new(); - while let Ok(event) = event_rx.recv() { + loop { + let next_deadline = state.lock().next_queued_deadline(); + let event = match next_deadline { + Some(deadline) => { + let wait = deadline.saturating_duration_since(Instant::now()); + match event_rx.recv_timeout(wait) { + Ok(event) => event, + Err(RecvTimeoutError::Timeout) => SchedulerEvent::Wake, + Err(RecvTimeoutError::Disconnected) => break, + } + } + None => match event_rx.recv() { + Ok(event) => event, + Err(_) => break, + }, + }; let shutdown; { let mut state = state.lock(); @@ -2687,11 +2722,15 @@ fn dispatch_runnable_class( JobClass::Maintenance => state.maintenance_inflight += 1, } made_progress = true; + let dispatched_root = run_job.root_id.clone(); if run_tx.send(run_job).is_err() { nonrunnable_dispatches.fetch_add(1, Ordering::AcqRel); *dispatch_failed = true; return made_progress; } + if let Some(actor) = state.actors.get_mut(&dispatched_root) { + actor.deficit = actor.deficit.saturating_sub(JOB_COST); + } } } diff --git a/crates/aft/src/executor/tests.rs b/crates/aft/src/executor/tests.rs index 96cdc2aee..6e1325259 100644 --- a/crates/aft/src/executor/tests.rs +++ b/crates/aft/src/executor/tests.rs @@ -550,6 +550,70 @@ fn drr_fairness() { } } +#[test] +fn drr_charges_only_successful_dispatches() { + let executor = test_executor(1, 1, 1, 1); + let (_dir_a, root_a) = test_root("drr-charge-a"); + let (_dir_b, root_b) = test_root("drr-charge-b"); + executor.register_actor(root_a.clone(), test_ctx()); + executor.register_actor(root_b.clone(), test_ctx()); + + let (started_tx, started_rx) = crossbeam_channel::bounded(2); + let (release_tx, release_rx) = crossbeam_channel::bounded(2); + let first_started = started_tx.clone(); + let first_release = release_rx.clone(); + let first = executor.submit( + root_a.clone(), + Lane::PureRead, + "drr-charge-a-1".to_string(), + Box::new(move |_| { + first_started.send("a1").expect("signal first start"); + first_release.recv().expect("release first"); + ok("a1") + }), + ); + started_rx + .recv_timeout(Duration::from_secs(1)) + .expect("first actor starts"); + + let second_started = started_tx.clone(); + let second = executor.submit( + root_a, + Lane::PureRead, + "drr-charge-a-2".to_string(), + Box::new(move |_| { + second_started.send("a2").expect("signal second start"); + ok("a2") + }), + ); + let third = executor.submit( + root_b, + Lane::PureRead, + "drr-charge-b-1".to_string(), + Box::new(move |_| { + started_tx.send("b1").expect("signal third start"); + ok("b1") + }), + ); + release_tx.send(()).expect("release first"); + + assert_eq!( + started_rx + .recv_timeout(Duration::from_secs(1)) + .expect("next fair dispatch"), + "b1", + "a blocked admission must not consume deficit and earn an extra turn" + ); + for handle in [first, second, third] { + assert!( + handle + .recv_timeout(Duration::from_secs(2)) + .expect("completion") + .success + ); + } +} + #[test] fn heavy_bound() { let executor = test_executor(6, 3, 5, 2); @@ -2560,6 +2624,57 @@ fn queued_deadline_job_is_pruned_and_counted_at_next_turn() { assert_eq!(executed.load(Ordering::Acquire), 0); } +#[test] +fn queued_deadline_wakes_scheduler_without_external_event() { + let executor = test_executor(2, 1, 1, 1); + let (_dir, root) = test_root("deadline-self-wake"); + executor.register_actor(root.clone(), test_ctx()); + + let (started_tx, started_rx) = crossbeam_channel::bounded(1); + let (release_tx, release_rx) = crossbeam_channel::bounded(1); + let blocker = executor.submit( + root.clone(), + Lane::Mutating, + "deadline-self-wake-blocker".to_string(), + Box::new(move |_| { + started_tx.send(()).expect("signal blocker start"); + release_rx.recv().expect("release blocker"); + ok("blocker") + }), + ); + started_rx + .recv_timeout(Duration::from_secs(1)) + .expect("blocker starts"); + + let executed = Arc::new(AtomicUsize::new(0)); + let executed_probe = Arc::clone(&executed); + let (rx, _token) = executor.submit_cancellable_async_with_deadline( + root, + Lane::PureRead, + "deadline-self-wake-victim".to_string(), + Box::new(move |_| { + executed_probe.fetch_add(1, Ordering::AcqRel); + ok("victim") + }), + Some(Instant::now() + Duration::from_millis(75)), + ); + + let response = rx + .blocking_recv() + .expect("deadline completion without another scheduler event"); + assert!(!response.success); + assert_eq!(response.data["code"], "request_deadline_exceeded"); + assert_eq!(executed.load(Ordering::Acquire), 0); + + release_tx.send(()).expect("release blocker"); + assert!( + blocker + .recv_timeout(Duration::from_secs(2)) + .expect("blocker completion") + .success + ); +} + #[test] fn interactive_queue_cap_returns_typed_backpressure_per_actor_and_global() { let executor = Executor::with_config(ExecutorConfig { @@ -2648,8 +2763,18 @@ fn interactive_queue_cap_returns_typed_backpressure_per_actor_and_global() { release_tx.send(()).expect("release blocker a"); release_tx.send(()).expect("release blocker b"); - assert!(blocker_a.recv_timeout(Duration::from_secs(5)).unwrap().success); - assert!(blocker_b.recv_timeout(Duration::from_secs(5)).unwrap().success); + assert!( + blocker_a + .recv_timeout(Duration::from_secs(5)) + .unwrap() + .success + ); + assert!( + blocker_b + .recv_timeout(Duration::from_secs(5)) + .unwrap() + .success + ); assert!(recv_async(queued_a, "queued a").success); assert!(recv_async(queued_b, "queued b").success); } diff --git a/crates/aft/src/resource_policy.rs b/crates/aft/src/resource_policy.rs index 29184fbf3..d915b4996 100644 --- a/crates/aft/src/resource_policy.rs +++ b/crates/aft/src/resource_policy.rs @@ -143,7 +143,11 @@ fn sample_linux_power_at(root: &std::path::Path) -> PowerState { Some("Battery") => { found_battery = true; if let Ok(value) = std::fs::read_to_string(path.join("capacity")) { - battery_capacity = value.trim().parse::().ok().or(battery_capacity); + if let Ok(capacity) = value.trim().parse::() { + battery_capacity = Some( + battery_capacity.map_or(capacity, |current: u8| current.min(capacity)), + ); + } } } _ => {} @@ -367,7 +371,15 @@ mod tests { std::fs::write(battery.join("capacity"), "5\n").unwrap(); assert_eq!(sample_linux_power_at(dir.path()), PowerState::BatterySaving); + let second_battery = dir.path().join("BAT1"); + std::fs::create_dir(&second_battery).unwrap(); + std::fs::write(second_battery.join("type"), "Battery\n").unwrap(); + std::fs::write(second_battery.join("capacity"), "90\n").unwrap(); + assert_eq!(sample_linux_power_at(dir.path()), PowerState::BatterySaving); + std::fs::remove_file(battery.join("capacity")).unwrap(); + assert_eq!(sample_linux_power_at(dir.path()), PowerState::Battery); + std::fs::remove_file(second_battery.join("capacity")).unwrap(); assert_eq!(sample_linux_power_at(dir.path()), PowerState::Unknown); } } diff --git a/crates/aft/src/search_index.rs b/crates/aft/src/search_index.rs index a33ceff1e..07367791f 100644 --- a/crates/aft/src/search_index.rs +++ b/crates/aft/src/search_index.rs @@ -1145,6 +1145,10 @@ impl SearchIndex { .collect(), ), }; + if validate_search_spill_segments(&staging_dir, manifest.spill_seq, files.len()).is_err() { + let _ = fs::remove_dir_all(&staging_dir); + return Ok(SearchBuildSliceOutcome::Yielded); + } let mut sources: Vec> = (0..manifest.spill_seq) .map(|seq| SpillSegmentSource::open(&staging_dir.join(format!("segment.{seq:06}.bin")))) .collect::>>()? @@ -3040,6 +3044,32 @@ impl PostingRecordSource for SpillSegmentSource { } } +fn validate_search_spill_segments( + staging_dir: &Path, + spill_seq: usize, + file_count: usize, +) -> std::io::Result<()> { + use std::io::BufRead; + + for seq in 0..spill_seq { + let path = staging_dir.join(format!("segment.{seq:06}.bin")); + let mut source = SpillSegmentSource::open(&path)?; + while let Some(record) = source.next_record()? { + if usize::try_from(record.file_id).unwrap_or(usize::MAX) >= file_count { + return Err(std::io::Error::other( + "search spill references an invalid file id", + )); + } + } + if source.remaining_in_group != 0 || !source.reader.fill_buf()?.is_empty() { + return Err(std::io::Error::other( + "search spill has inconsistent record framing", + )); + } + } + Ok(()) +} + struct BaseRecordSource { base: Arc, id_map: Arc>, @@ -8740,6 +8770,49 @@ mod tests { assert_eq!(restarted.files.len(), SEARCH_SLICE_FILES); } + #[test] + fn resumable_search_discards_missing_spill_before_publication() { + let dir = tempfile::tempdir().expect("create temp dir"); + let project = dir.path().join("project"); + let cache = dir.path().join("cache"); + fs::create_dir_all(&project).expect("create project"); + for index in 0..40 { + fs::write( + project.join(format!("file_{index:03}.rs")), + format!("fn marker_{index}() {{}}\n"), + ) + .expect("write source"); + } + + loop { + let outcome = + SearchIndex::resume_cold_build_slice(&project, DEFAULT_MAX_FILE_SIZE, &cache) + .expect("prepare slice"); + let manifest = load_search_staging_manifest( + &cache.join(SEARCH_STAGING_DIR).join(SEARCH_STAGING_MANIFEST), + ) + .expect("staging manifest"); + assert_eq!(outcome, SearchBuildSliceOutcome::Yielded); + if manifest.cursor == manifest.paths.len() + && manifest.validation_cursor == manifest.files.len() + { + break; + } + } + + let staging_dir = cache.join(SEARCH_STAGING_DIR); + fs::remove_file(staging_dir.join("segment.000000.bin")).expect("remove spill segment"); + assert_eq!( + SearchIndex::resume_cold_build_slice(&project, DEFAULT_MAX_FILE_SIZE, &cache) + .expect("recover corrupt staging"), + SearchBuildSliceOutcome::Yielded + ); + assert!( + load_search_staging_manifest(&staging_dir.join(SEARCH_STAGING_MANIFEST)).is_none(), + "invalid staging manifest must be discarded" + ); + } + #[test] fn ignore_rule_discovery_respects_gitignore() { let _git_env = crate::test_env::hermetic_git_env_guard(); diff --git a/crates/aft/src/semantic_index.rs b/crates/aft/src/semantic_index.rs index d7876771d..5f0ff3c6e 100644 --- a/crates/aft/src/semantic_index.rs +++ b/crates/aft/src/semantic_index.rs @@ -121,10 +121,10 @@ impl EmbeddingRequestPolicy { } } -const SEMANTIC_STAGING_VERSION: u32 = 2; -const SEMANTIC_STAGING_FILE: &str = "semantic-staging-v2.json"; -const SEMANTIC_STAGING_CHUNKS_FILE: &str = "semantic-staging-chunks-v2.jsonl"; -const SEMANTIC_STAGING_VECTORS_FILE: &str = "semantic-staging-vectors-v2.bin"; +const SEMANTIC_STAGING_VERSION: u32 = 3; +const SEMANTIC_STAGING_FILE: &str = "semantic-staging-v3.json"; +const SEMANTIC_STAGING_CHUNKS_FILE: &str = "semantic-staging-chunks-v3.jsonl"; +const SEMANTIC_STAGING_VECTORS_FILE: &str = "semantic-staging-vectors-v3.bin"; const SEMANTIC_COLLECT_SLICE_FILES: usize = 32; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -143,6 +143,7 @@ struct SemanticStagingManifest { collect_cursor: usize, embed_cursor: usize, chunks_count: usize, + chunk_offsets: Vec, metadata: Vec, vectors_count: usize, } @@ -2091,16 +2092,29 @@ fn write_semantic_staging(path: &Path, manifest: &SemanticStagingManifest) -> Re crate::fs_lock::rename_over(&temporary, path).map_err(|error| error.to_string()) } -fn append_semantic_chunks(path: &Path, chunks: &[SemanticChunk]) -> Result<(), String> { +fn append_semantic_chunks( + path: &Path, + chunks: &[SemanticChunk], + offsets: &mut Vec, +) -> Result<(), String> { use std::io::Write; let mut file = fs::OpenOptions::new() .create(true) .append(true) .open(path) .map_err(|error| error.to_string())?; + let mut offset = file.metadata().map_err(|error| error.to_string())?.len(); + if offsets.is_empty() { + offsets.push(offset); + } for chunk in chunks { - serde_json::to_writer(&mut file, chunk).map_err(|error| error.to_string())?; + let bytes = serde_json::to_vec(chunk).map_err(|error| error.to_string())?; + file.write_all(&bytes).map_err(|error| error.to_string())?; file.write_all(b"\n").map_err(|error| error.to_string())?; + offset = offset + .checked_add(u64::try_from(bytes.len() + 1).map_err(|error| error.to_string())?) + .ok_or_else(|| "semantic staging chunk offset overflow".to_string())?; + offsets.push(offset); } file.sync_data().map_err(|error| error.to_string()) } @@ -2109,37 +2123,37 @@ fn read_semantic_chunk_range( path: &Path, start: usize, end: usize, - total: usize, + offsets: &[u64], ) -> Result, String> { + use std::io::{Read, Seek, SeekFrom}; + let total = offsets.len().saturating_sub(1); if start > end || end > total { return Err("semantic staging chunk range is invalid".to_string()); } - if total == 0 { + if start == end { return Ok(Vec::new()); } - let file = fs::File::open(path).map_err(|error| error.to_string())?; - let mut chunks = Vec::with_capacity(end - start); - let mut count = 0usize; - for line in std::io::BufRead::lines(BufReader::new(file)) { - let line = line.map_err(|error| error.to_string())?; - if line.is_empty() { - continue; - } - if count >= start && count < end { - chunks.push(serde_json::from_str(&line).map_err(|error| error.to_string())?); - } - count = count.saturating_add(1); - } - if count != total { - return Err(format!( - "semantic staging chunk segment count mismatch: expected {total}, got {count}" - )); - } - Ok(chunks) + let start_offset = offsets[start]; + let byte_len = offsets[end] + .checked_sub(start_offset) + .ok_or_else(|| "semantic staging chunk offsets are invalid".to_string())?; + let byte_len = usize::try_from(byte_len) + .map_err(|_| "semantic staging chunk range is too large".to_string())?; + let mut file = fs::File::open(path).map_err(|error| error.to_string())?; + file.seek(SeekFrom::Start(start_offset)) + .map_err(|error| error.to_string())?; + let mut bytes = vec![0; byte_len]; + file.read_exact(&mut bytes) + .map_err(|error| error.to_string())?; + bytes + .split(|byte| *byte == b'\n') + .filter(|line| !line.is_empty()) + .map(|line| serde_json::from_slice(line).map_err(|error| error.to_string())) + .collect() } -fn read_semantic_chunks(path: &Path, count: usize) -> Result, String> { - read_semantic_chunk_range(path, 0, count, count) +fn read_semantic_chunks(path: &Path, offsets: &[u64]) -> Result, String> { + read_semantic_chunk_range(path, 0, offsets.len().saturating_sub(1), offsets) } fn append_semantic_vectors( @@ -2973,11 +2987,15 @@ impl SemanticIndex { fs::create_dir_all(&dir).map_err(|error| error.to_string())?; let valid_segment_lengths = |manifest: &SemanticStagingManifest| { - let chunks_valid = fs::read_to_string(&chunks_path) - .map(|text| { - text.lines().filter(|line| !line.is_empty()).count() == manifest.chunks_count - }) - .unwrap_or(manifest.chunks_count == 0); + let offsets_valid = manifest.chunk_offsets.len() == manifest.chunks_count + 1 + && manifest.chunk_offsets.first() == Some(&0) + && manifest + .chunk_offsets + .windows(2) + .all(|window| window[0] <= window[1]); + let chunks_valid = fs::metadata(&chunks_path) + .map(|metadata| manifest.chunk_offsets.last() == Some(&metadata.len())) + .unwrap_or(manifest.chunks_count == 0 && manifest.chunk_offsets == [0]); let vector_bytes = manifest .vectors_count .saturating_mul(fingerprint.dimension) @@ -2985,7 +3003,7 @@ impl SemanticIndex { let vectors_valid = fs::metadata(&vectors_path) .map(|metadata| metadata.len() == u64::try_from(vector_bytes).unwrap_or(u64::MAX)) .unwrap_or(manifest.vectors_count == 0); - chunks_valid && vectors_valid + offsets_valid && chunks_valid && vectors_valid }; let mut manifest = load_semantic_staging(&staging_path) .filter(|manifest| { @@ -3010,6 +3028,7 @@ impl SemanticIndex { collect_cursor: 0, embed_cursor: 0, chunks_count: 0, + chunk_offsets: vec![0], metadata: Vec::new(), vectors_count: 0, } @@ -3019,7 +3038,7 @@ impl SemanticIndex { let end = (manifest.collect_cursor + SEMANTIC_COLLECT_SLICE_FILES).min(files.len()); let (chunks, metadata) = Self::collect_chunks(&canonical_root, &files[manifest.collect_cursor..end]); - append_semantic_chunks(&chunks_path, &chunks)?; + append_semantic_chunks(&chunks_path, &chunks, &mut manifest.chunk_offsets)?; manifest.chunks_count += chunks.len(); manifest .metadata @@ -3047,7 +3066,7 @@ impl SemanticIndex { &chunks_path, manifest.embed_cursor, end, - manifest.chunks_count, + &manifest.chunk_offsets, )?; let texts = chunks .iter() @@ -3073,7 +3092,7 @@ impl SemanticIndex { return Ok(SemanticBuildSliceOutcome::Yielded); } - let chunks = read_semantic_chunks(&chunks_path, manifest.chunks_count)?; + let chunks = read_semantic_chunks(&chunks_path, &manifest.chunk_offsets)?; let vectors = read_semantic_vectors(&vectors_path, manifest.vectors_count, fingerprint.dimension)?; let file_metadata = manifest @@ -7751,6 +7770,46 @@ public class Greeter { .any(|entry| entry.chunk.name == "old_symbol")); } + #[test] + fn semantic_chunk_range_seeks_without_parsing_preceding_records() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("chunks.jsonl"); + let chunks = vec![ + SemanticChunk { + file: PathBuf::from("first.rs"), + name: "first".to_string(), + qualified_name: None, + kind: SymbolKind::Function, + start_line: 0, + end_line: 1, + exported: false, + embed_text: "first".to_string(), + snippet: "fn first() {}".to_string(), + }, + SemanticChunk { + file: PathBuf::from("second.rs"), + name: "second".to_string(), + qualified_name: None, + kind: SymbolKind::Function, + start_line: 0, + end_line: 1, + exported: false, + embed_text: "second".to_string(), + snippet: "fn second() {}".to_string(), + }, + ]; + let mut offsets = vec![0]; + append_semantic_chunks(&path, &chunks, &mut offsets).expect("append chunks"); + let mut bytes = fs::read(&path).expect("read chunks"); + bytes[0] = b'!'; + fs::write(&path, bytes).expect("corrupt preceding record"); + + let selected = read_semantic_chunk_range(&path, 1, 2, &offsets).expect("read range"); + + assert_eq!(selected.len(), 1); + assert_eq!(selected[0].name, "second"); + } + #[test] fn resumable_semantic_build_yields_rejects_stale_state_and_matches_monolithic_corpus() { let dir = tempfile::tempdir().expect("temp dir"); diff --git a/crates/aft/src/standing_roots.rs b/crates/aft/src/standing_roots.rs index c93e871ca..b599a7f29 100644 --- a/crates/aft/src/standing_roots.rs +++ b/crates/aft/src/standing_roots.rs @@ -1141,6 +1141,33 @@ mod tests { ); } + #[test] + fn session_owned_root_stays_parked_until_unbind_resumes_it() { + let storage = tempdir().unwrap(); + let root_dir = tempdir().unwrap(); + let roots = StandingRoots::default(); + let cfg = config( + storage.path(), + vec![root(root_dir.path(), vec![IndexKind::Search])], + ); + roots.reconcile(&cfg).unwrap(); + let literal = root_dir.path().to_str().unwrap(); + + roots.begin_case_a_bind(literal).unwrap(); + assert!( + roots.admit_build(literal).is_none(), + "a bound session must park standing artifact work" + ); + + roots + .resume_after_session(literal, &[IndexKind::Search]) + .unwrap(); + assert!( + roots.admit_build(literal).is_some(), + "session unbind must reopen standing artifact work" + ); + } + #[test] fn configuration_add_modify_and_remove_mint_boundaries_and_delete_rows() { let storage = tempdir().unwrap(); diff --git a/crates/aft/src/subc/mod.rs b/crates/aft/src/subc/mod.rs index 69bd13210..712af301a 100644 --- a/crates/aft/src/subc/mod.rs +++ b/crates/aft/src/subc/mod.rs @@ -894,8 +894,8 @@ struct PendingBashAsk { grants: Vec, expires_at: Instant, /// The caller's absolute request deadline, captured at ingress before - /// permission elicitation. Checked again on an allowed reply before any - /// spawn bookkeeping or executor submission. + /// permission elicitation. The maintenance tick settles the ask at this + /// deadline, and an allowed reply checks it again before any spawn. request_deadline: Option, } @@ -1803,6 +1803,47 @@ async fn settle_pending_bash_ask_denied( .await } +#[allow(clippy::too_many_arguments)] +async fn settle_pending_bash_ask_deadline_exceeded( + tx: &WriterSender, + pending: PendingBashAsk, + routes: &HashMap, + live_roots: &mut HashMap, + route_bash_cancels: &mut HashMap, + shutdown: &Arc, + metrics: &DispatchPathMetrics, +) -> Result<(), SubcError> { + let response = Response::error_with_data( + pending.request_id.clone(), + "request_deadline_exceeded", + "request deadline elapsed during permission elicitation", + serde_json::json!({ + "retryable": false, + "phase": "queue", + }), + ); + let completion = bash::bash_deadline_exceeded_completion( + pending.route, + pending.tool_corr, + pending.tool_flags, + pending.tool_ver, + pending.root, + pending.request_id, + pending.format_context, + response, + ); + bash::handle_bash_deferred_completion( + tx, + completion, + routes, + live_roots, + route_bash_cancels, + shutdown, + metrics, + ) + .await +} + fn take_pending_bash_asks_for_route( pending_bash_asks: &mut HashMap, route: RouteChannel, @@ -1885,25 +1926,43 @@ async fn expire_pending_bash_asks( let now = Instant::now(); let expired = pending_bash_asks .iter() - .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key)) + .filter_map(|(key, pending)| { + let request_expired = pending + .request_deadline + .is_some_and(|deadline| deadline <= now); + (request_expired || pending.expires_at <= now).then_some((*key, request_expired)) + }) .collect::>(); - for key in expired { + for (key, request_expired) in expired { if let Some(pending) = pending_bash_asks.remove(&key) { log::debug!( "subc attach: bash elicitation request {} on route {} expired fail-closed", key.corr, pending.route ); - settle_pending_bash_ask_denied( - tx, - pending, - routes, - live_roots, - route_bash_cancels, - shutdown, - metrics, - ) - .await?; + if request_expired { + settle_pending_bash_ask_deadline_exceeded( + tx, + pending, + routes, + live_roots, + route_bash_cancels, + shutdown, + metrics, + ) + .await?; + } else { + settle_pending_bash_ask_denied( + tx, + pending, + routes, + live_roots, + route_bash_cancels, + shutdown, + metrics, + ) + .await?; + } } } Ok(()) @@ -1936,39 +1995,21 @@ async fn handle_bash_elicitation_reply( // The request deadline is checked BEFORE bash-wait bookkeeping and // before any executor submission: an expired permission answer must // prove that no bash command started. - if let Some(deadline) = pending.request_deadline { - if Instant::now() >= deadline { - let response = Response::error_with_data( - pending.request_id.clone(), - "request_deadline_exceeded", - "request deadline elapsed during permission elicitation", - serde_json::json!({ - "retryable": false, - "phase": "queue", - }), - ); - let completion = bash::bash_deadline_exceeded_completion( - pending.route, - pending.tool_corr, - pending.tool_flags, - pending.tool_ver, - pending.root, - pending.request_id, - pending.format_context, - response, - ); - bash::handle_bash_deferred_completion( - tx, - completion, - routes, - live_roots, - route_bash_cancels, - shutdown, - metrics, - ) - .await?; - return Ok(()); - } + if pending + .request_deadline + .is_some_and(|deadline| Instant::now() >= deadline) + { + settle_pending_bash_ask_deadline_exceeded( + tx, + pending, + routes, + live_roots, + route_bash_cancels, + shutdown, + metrics, + ) + .await?; + return Ok(()); } if routes.contains_key(&key.route) { bash::submit_deferred_bash( @@ -6890,6 +6931,58 @@ mod tests { assert_eq!(frame.frame.header.ty, FrameType::Request); } + #[tokio::test] + async fn ready_data_frame_dispatches_immediately_after_control_burst() { + let (control_tx, mut control_rx) = mpsc::channel(CONTROL_BURST_LIMIT + 2); + let (data_tx, mut data_rx) = mpsc::channel(1); + for seq in 0..=CONTROL_BURST_LIMIT { + control_tx + .send(Ok(DecodedFrame { + frame: Frame::build( + FrameType::Ping, + control_flags(), + 0, + 0, + seq as u64, + Vec::new(), + ) + .unwrap(), + phase_trace: PhaseTrace::new(Instant::now()), + })) + .await + .unwrap(); + } + + let mut lane = PrioritizedFrameLane::default(); + for _ in 0..CONTROL_BURST_LIMIT { + recv_prioritized_frame(&mut control_rx, &mut data_rx, &mut lane) + .await + .unwrap() + .unwrap(); + } + data_tx + .send(Ok(DecodedFrame { + frame: Frame::build( + FrameType::Request, + control_flags(), + 1, + 1, + 99, + br#"{}"#.to_vec(), + ) + .unwrap(), + phase_trace: PhaseTrace::new(Instant::now()), + })) + .await + .unwrap(); + + let frame = recv_prioritized_frame(&mut control_rx, &mut data_rx, &mut lane) + .await + .unwrap() + .unwrap(); + assert_eq!(frame.frame.header.ty, FrameType::Request); + } + #[tokio::test] async fn control_lane_remains_live_after_burst_when_data_is_idle() { let (control_tx, mut control_rx) = mpsc::channel(CONTROL_BURST_LIMIT + 1); diff --git a/crates/aft/src/subc/standing.rs b/crates/aft/src/subc/standing.rs index 23fb1a053..5e598d45a 100644 --- a/crates/aft/src/subc/standing.rs +++ b/crates/aft/src/subc/standing.rs @@ -334,9 +334,7 @@ impl StandingActor { { schedule.pending.remove(&key); } - schedule - .queue - .complete_generation(key, generation, 1, true); + schedule.queue.complete_generation(key, generation, 1, true); continue; } if schedule @@ -887,8 +885,8 @@ fn build_missing_callgraph_after_strict_check( .get(&entry.literal_path) .is_none_or(|cached| cached.resolved_target != entry.resolved_target); if needs_refresh { - let files = crate::callgraph::walk_project_files(&entry.resolved_target) - .collect::>(); + let files = + crate::callgraph::walk_project_files(&entry.resolved_target).collect::>(); cache.insert( entry.literal_path.clone(), CallgraphBuildCache { @@ -897,7 +895,12 @@ fn build_missing_callgraph_after_strict_check( }, ); } - Arc::clone(&cache.get(&entry.literal_path).expect("cache inserted").files) + Arc::clone( + &cache + .get(&entry.literal_path) + .expect("cache inserted") + .files, + ) }; let cache_dir = storage_dir.join("callgraph").join(&entry.artifact_key); let lease = match crate::root_cache::WriterLease::acquire_shared( diff --git a/crates/aft/tests/integration/bash_pty_test.rs b/crates/aft/tests/integration/bash_pty_test.rs index 21fe937e1..64f4fbb8b 100644 --- a/crates/aft/tests/integration/bash_pty_test.rs +++ b/crates/aft/tests/integration/bash_pty_test.rs @@ -771,8 +771,7 @@ fn pty_kill_terminates_sighup_ignoring_cat() { Duration::from_secs(30), ); - let killed = registry.kill(&task_id, SESSION).unwrap(); - assert_eq!(killed.info.status, BgTaskStatus::Killing); + registry.kill(&task_id, SESSION).unwrap(); wait_for_status(®istry, &task_id, BgTaskStatus::Killed); } diff --git a/crates/aft/tests/integration/subc_bridge_test.rs b/crates/aft/tests/integration/subc_bridge_test.rs index a698bb173..e8ef9b999 100644 --- a/crates/aft/tests/integration/subc_bridge_test.rs +++ b/crates/aft/tests/integration/subc_bridge_test.rs @@ -2365,6 +2365,17 @@ fn subc_bridge_untrusted_bash_elicitation_ttl_denies_and_settles() { ); } +#[test] +fn subc_bridge_untrusted_bash_elicitation_request_deadline_settles() { + run_subc_bridge_test_with_env( + "subc_bridge_untrusted_bash_elicitation_request_deadline_settles", + Duration::from_secs(45), + || vec![set_test_bash_elicitation_ttl_ms(5_000)], + drive_bash_elicitation_request_deadline_daemon, + |_, _, _| {}, + ); +} + #[test] fn subc_bridge_untrusted_bash_elicitation_goodbye_sweeps_pending() { run_subc_bridge_test( @@ -6830,6 +6841,60 @@ async fn drive_bash_elicitation_ttl_daemon(input: FakeDaemonInput) { send_connection_goodbye(&mut stream).await; } +async fn drive_bash_elicitation_request_deadline_daemon(input: FakeDaemonInput) { + let FakeDaemonSession { + mut stream, root1, .. + } = open_fake_daemon_session(input).await; + bind_untrusted_elicitation_route(&mut stream, 1, 101, &root1).await; + + let touched = root1.join("elicitation-request-deadline.txt"); + send_tool_call_with_deadline( + &mut stream, + 1, + 302, + "bash", + json!({ "command": touch_command(&touched), "compressed": false }), + 150, + ) + .await; + let (ask_corr, _) = expect_bash_elicitation_request(&mut stream, 1, "touch").await; + let frame = read_frame_within( + &mut stream, + Duration::from_secs(2), + "request deadline response", + ) + .await + .expect("pending ask should settle at the request deadline"); + assert_eq!(frame.header.channel, 1); + assert_eq!(frame.header.corr, 302); + assert_untrusted_tool_error( + &frame, + "request deadline elapsed during permission elicitation", + "request deadline", + ); + assert!(!touched.exists(), "expired ask must not spawn"); + + send_bash_elicitation_reply( + &mut stream, + 1, + ask_corr, + json!({ "action": "accept", "content": { "decision": "allow" } }), + ) + .await; + assert_no_response_frame_within( + &mut stream, + Duration::from_millis(300), + "late allow after request deadline", + ) + .await; + assert!( + !touched.exists(), + "late allow after deadline must be ignored" + ); + + send_connection_goodbye(&mut stream).await; +} + async fn drive_bash_elicitation_goodbye_daemon(input: FakeDaemonInput) { let FakeDaemonSession { mut stream, root1, .. @@ -8806,6 +8871,34 @@ async fn send_tool_call( send_tool_call_epoch(stream, channel, 1, corr, name, arguments).await; } +async fn send_tool_call_with_deadline( + stream: &mut tokio::net::TcpStream, + channel: u16, + corr: u64, + name: &str, + arguments: Value, + deadline_ms_remaining: u64, +) { + let body = json!({ + "name": name, + "arguments": arguments, + "deadline_ms_remaining": deadline_ms_remaining, + }); + send_frame( + stream, + Frame::build( + FrameType::Request, + Flags::new(false, Priority::Interactive, false), + channel, + 1, + corr, + serde_json::to_vec(&body).expect("tool call body"), + ) + .expect("tool call frame"), + ) + .await; +} + async fn send_registered_tool_call( stream: &mut tokio::net::TcpStream, channel: u16, diff --git a/crates/aft/tests/integration/subc_storm_test.rs b/crates/aft/tests/integration/subc_storm_test.rs index 2d4fac39f..9ba7a9905 100644 --- a/crates/aft/tests/integration/subc_storm_test.rs +++ b/crates/aft/tests/integration/subc_storm_test.rs @@ -1778,8 +1778,8 @@ async fn drive_standing_yield_daemon(input: FakeDaemonInput) { ); } - // A PureRead with a finite deadline finishes well inside its budget while - // standing work keeps cycling; health replies stay fast. + // A PureRead with a finite deadline finishes well inside its budget after + // standing passes yield under saturated cold capacity. let read_root_id = ProjectRootId::from_path(&session.root1).expect("read root id"); let started = Instant::now(); let (read_tx, read_rx) = tokio::sync::oneshot::channel(); @@ -1808,8 +1808,8 @@ async fn drive_standing_yield_daemon(input: FakeDaemonInput) { "read latency stayed inside the request budget" ); - // The nonblocking health mirror must be observable without contention: - // pending depths stay under the configured caps while standing work runs. + // The nonblocking health mirror remains observable without contention, + // and pending depths stay under the configured caps. let liveness = executor .try_dispatch_liveness_snapshot() .expect("nonblocking dispatch liveness under standing saturation"); @@ -1822,7 +1822,7 @@ async fn drive_standing_yield_daemon(input: FakeDaemonInput) { "pending maintenance depth stayed under the process cap" ); - // Release the cold permits; standing passes can acquire again. + // Release the cold permits and confirm nonblocking cold admission recovers. drop(permit_a); drop(permit_b); let resumed = aft::cold_build_limiter::try_acquire(); diff --git a/packages/aft-bridge/src/__tests__/subc-transport.test.ts b/packages/aft-bridge/src/__tests__/subc-transport.test.ts index ee28be466..4ade9a1a6 100644 --- a/packages/aft-bridge/src/__tests__/subc-transport.test.ts +++ b/packages/aft-bridge/src/__tests__/subc-transport.test.ts @@ -1932,6 +1932,40 @@ describe("SubcTransportPool request budget (deadline_ms_remaining)", () => { expect(firstStamp).toBeGreaterThan(retryStamp); }); + test("a not-sent stale-route retry preserves the reopened shared route", async () => { + const client = new FakeClient(async () => envelope({ id: "r", success: true, text: "" })); + let requestAttempts = 0; + const originalRequest = client.request.bind(client); + client.request = async (route, body, options) => { + requestAttempts += 1; + if (requestAttempts === 1) { + client.requests.push({ route, channel: route.channel, body, options }); + throw new SubcError("unknown channel", "unknown_channel"); + } + if (requestAttempts === 2) { + throw new SubcCallError( + "not_sent", + "request deadline elapsed before the request could be sent", + "request_deadline_exceeded_before_send", + ); + } + return originalRequest(route, body, options); + }; + const pool = poolWithDefault(client, 30_000); + + await expect(pool.getBridge(TEST_PROJECT_ROOT).toolCall("s", "read", {})).rejects.toMatchObject( + { + kind: "not_sent", + code: "request_deadline_exceeded_before_send", + }, + ); + const opensAfterExpiredCaller = client.routeOpens.length; + + await pool.getBridge(TEST_PROJECT_ROOT).toolCall("s", "read", {}); + expect(client.routeOpens.length).toBe(opensAfterExpiredCaller); + expect(requestAttempts).toBe(3); + }); + test("the route open race observes late settlement without invalidating the shared route", async () => { const gate = Promise.withResolvers(); const releaseOpen: () => void = () => gate.resolve(); diff --git a/packages/aft-bridge/src/subc-transport.ts b/packages/aft-bridge/src/subc-transport.ts index 2f02a3f8c..c191e81cf 100644 --- a/packages/aft-bridge/src/subc-transport.ts +++ b/packages/aft-bridge/src/subc-transport.ts @@ -1652,6 +1652,9 @@ export class SubcTransportPool implements AftTransportPool { }; const handleRequestFailure = (error: unknown, entry: RouteEntry): void => { + // A caller-scoped pre-send expiry says nothing about shared route + // health. Preserve the freshly reopened route for other callers. + if (error instanceof SubcCallError && error.kind === "not_sent") return; clearRouteEntry(entry); if ( !this.isCurrentSession(key, record) || diff --git a/packages/pi-plugin/src/__tests__/config.test.ts b/packages/pi-plugin/src/__tests__/config.test.ts index d89acb6c6..006f40835 100644 --- a/packages/pi-plugin/src/__tests__/config.test.ts +++ b/packages/pi-plugin/src/__tests__/config.test.ts @@ -66,8 +66,7 @@ afterEach(() => { tempRoots.clear(); }); -test("index resource policy defaults, validates, and remains user-only", () => { - expect(AftConfigSchema.parse({}).index?.resource_policy ?? "balanced").toBe("balanced"); +test("index resource policy validates and remains user-only", () => { expect( AftConfigSchema.parse({ index: { resource_policy: "balanced" } }).index?.resource_policy, ).toBe("balanced"); diff --git a/packages/pi-plugin/src/tools/_shared.ts b/packages/pi-plugin/src/tools/_shared.ts index a9b5aec2e..44c436d62 100644 --- a/packages/pi-plugin/src/tools/_shared.ts +++ b/packages/pi-plugin/src/tools/_shared.ts @@ -5,7 +5,7 @@ import { existsSync } from "node:fs"; import type { AftProjectTransport, - BridgeRequestOptions, + AftTransportOptions, ToolCallOptions, ToolCallResult, } from "@cortexkit/aft-bridge"; @@ -41,8 +41,8 @@ export interface PiToolCallOptions> extends T function piTransportOptions( command: string, - options: BridgeRequestOptions = {}, -): BridgeRequestOptions { + options: AftTransportOptions = {}, +): AftTransportOptions { const { timeoutMs: callerTimeoutMs, ...rest } = options; const requested = rest.transportTimeoutMs ?? callerTimeoutMs ?? bridgeTimeoutForCommand(command); const transportTimeoutMs = Math.min( @@ -167,7 +167,7 @@ export async function callBridge( command: string, params: Record = {}, extCtx?: ExtensionContext, - options?: BridgeRequestOptions, + options?: AftTransportOptions, ): Promise> { const merged: Record = { ...params }; const sessionId = extCtx ? resolveSessionId(extCtx) : undefined; From ee00371f46d69d1a98a256e1313129bf14cda5aa Mon Sep 17 00:00:00 2001 From: Naadir Jeewa Date: Mon, 31 Aug 2026 23:13:20 +0100 Subject: [PATCH 14/14] fix(index): reset replaced root generation Signed-off-by: Naadir Jeewa Co-authored-by: alfonso-aft <289616620+alfonso-aft@users.noreply.github.com> --- crates/aft/src/subc/standing.rs | 48 +++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/crates/aft/src/subc/standing.rs b/crates/aft/src/subc/standing.rs index 5e598d45a..7740341b6 100644 --- a/crates/aft/src/subc/standing.rs +++ b/crates/aft/src/subc/standing.rs @@ -271,11 +271,18 @@ impl StandingActor { .next_kind .retain(|key, _| entries.iter().any(|entry| entry.literal_path == *key)); for entry in entries { - let selection_changed = schedule - .entries - .get(&entry.literal_path) - .is_some_and(|previous| previous.indexes != entry.indexes); - if selection_changed { + let identity_changed = + schedule + .entries + .get(&entry.literal_path) + .is_some_and(|previous| { + previous.resolved_target != entry.resolved_target + || previous.resolved_git_toplevel != entry.resolved_git_toplevel + || previous.scoped_relative_path != entry.scoped_relative_path + || previous.artifact_key != entry.artifact_key + || previous.indexes != entry.indexes + }); + if identity_changed { schedule.queue.reconfigure(&entry.literal_path); schedule.next_kind.insert(entry.literal_path.clone(), 0); } else { @@ -1002,6 +1009,37 @@ mod tests { assert_eq!(schedule.next_kind.get(&entry.literal_path), Some(&0)); } + #[test] + fn identity_change_resets_kind_cursor_and_scheduler_generation() { + let mut schedule = StandingScheduleState::default(); + let mut entry = StandingRootEntry { + literal_path: "/tmp/root".to_string(), + resolved_target: std::path::PathBuf::from("/tmp/old-target"), + resolved_git_toplevel: None, + scoped_relative_path: None, + artifact_key: "old-key".to_string(), + indexes: vec![IndexKind::Search], + config_order: 0, + }; + schedule.queue.reconcile([entry.literal_path.clone()]); + schedule + .entries + .insert(entry.literal_path.clone(), entry.clone()); + schedule.next_kind.insert(entry.literal_path.clone(), 1); + let old_generation = schedule.queue.generation(&entry.literal_path).unwrap(); + + entry.resolved_target = std::path::PathBuf::from("/tmp/new-target"); + entry.artifact_key = "new-key".to_string(); + StandingActor::reconcile_kind_cursors(&mut schedule, std::slice::from_ref(&entry)); + + assert_eq!(schedule.next_kind.get(&entry.literal_path), Some(&0)); + assert_ne!( + schedule.queue.generation(&entry.literal_path), + Some(old_generation), + "a replacement must fence completion from the former target" + ); + } + #[test] fn strict_search_verification_accepts_metadata_only_drift() { let storage = tempfile::tempdir().unwrap();