From cefe8073bdeeb51935a62773b72c76d8578fa4f4 Mon Sep 17 00:00:00 2001 From: Qiiks Date: Thu, 17 Sep 2026 19:47:58 +0530 Subject: [PATCH] fix(cuda): advertise enforced embedding batch limits --- crates/synapse-module/src/lib.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/synapse-module/src/lib.rs b/crates/synapse-module/src/lib.rs index 284647c..6ee0681 100644 --- a/crates/synapse-module/src/lib.rs +++ b/crates/synapse-module/src/lib.rs @@ -1887,7 +1887,7 @@ struct SystemClock; /// for the measurements. fn recommended_batch_for_engine(engine: &str, max_tokens: usize) -> Option { match engine { - "owned-metal" => Some(RecommendedBatch { + "owned-metal" | CUDA_WORKER_ENGINE => Some(RecommendedBatch { rows: MAX_ENGINE_BATCH_ITEMS, token_budget: DEFAULT_ENGINE_BATCH_TOKEN_BUDGET, }), @@ -17196,6 +17196,12 @@ mod tests { assert_eq!(owned.rows, MAX_ENGINE_BATCH_ITEMS); assert_eq!(owned.token_budget, DEFAULT_ENGINE_BATCH_TOKEN_BUDGET); + let cuda = recommended_batch_for_engine(CUDA_WORKER_ENGINE, 2048) + .expect("CUDA clients need usable batch advice"); + let wire = serde_json::to_value(cuda).expect("serialize CUDA batch advice"); + assert_eq!(wire["rows"], MAX_ENGINE_BATCH_ITEMS); + assert_eq!(wire["token_budget"], DEFAULT_ENGINE_BATCH_TOKEN_BUDGET); + let ane = recommended_batch_for_engine("ane", 512).unwrap(); assert_eq!(ane.rows, MAX_ENGINE_BATCH_ITEMS); assert_eq!(ane.token_budget, 512 * MAX_ENGINE_BATCH_ITEMS as u64);