From 4c83c0b341807ded5dc14b423011c721606310ea Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Sun, 23 Aug 2026 01:44:44 +0530 Subject: [PATCH 1/5] feat: overlap CPU cold-expert chain with GPU hot chain (GGML_SCHED_ASYNC_CPU) The dual-path MoE cache ran its two chains serially: the scheduler blocks on CPU splits, and the cold chain's input copies drain the whole CUDA stream. Three changes make them concurrent, gated behind GGML_SCHED_ASYNC_CPU=1: - sched: CPU splits execute on a persistent worker thread; the main loop keeps launching later splits and joins lazily at the first split that is CPU or reads a CPU-resident input - graph: cold chain is built before the hot chain so the hot split follows it and carries no cross-backend inputs (launches while the worker computes) - graph: the merge add is pinned to CPU so it does not pull the hot chain into a split that waits on the cold result --- ggml/src/ggml-backend.cpp | 121 ++++++++++++++++++++++++++++++++++++++ src/llama-graph.cpp | 20 ++++++- 2 files changed, 138 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 5a5d9151e2fe..0fe739e686ed 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -20,6 +20,9 @@ #include #include #include +#include +#include +#include #include #ifdef __APPLE__ @@ -775,6 +778,78 @@ struct ggml_backend_sched_split { struct ggml_cgraph graph; }; +// async execution of CPU splits (GGML_SCHED_ASYNC_CPU): a persistent worker +// computes a CPU split while the main thread keeps launching later splits that +// do not depend on it, so an independent GPU split overlaps the CPU compute +struct ggml_sched_cpu_async { + std::thread worker; + std::mutex mtx; + std::condition_variable cv; + ggml_backend_t job_backend = nullptr; + struct ggml_cgraph * job_graph = nullptr; + enum ggml_status job_status = GGML_STATUS_SUCCESS; + bool job_ready = false; + bool job_done = false; + bool stop = false; + bool pending = false; // main-thread view: a job is queued or running + + ggml_sched_cpu_async() { + worker = std::thread([this]() { + for (;;) { + std::unique_lock lock(mtx); + cv.wait(lock, [this]() { return job_ready || stop; }); + if (stop) { + return; + } + job_ready = false; + ggml_backend_t backend = job_backend; + ggml_cgraph * graph = job_graph; + lock.unlock(); + + enum ggml_status status = ggml_backend_graph_compute_async(backend, graph); + + lock.lock(); + job_status = status; + job_done = true; + cv.notify_all(); + } + }); + } + + ~ggml_sched_cpu_async() { + { + std::lock_guard lock(mtx); + stop = true; + } + cv.notify_all(); + worker.join(); + } + + void launch(ggml_backend_t backend, struct ggml_cgraph * graph) { + { + std::lock_guard lock(mtx); + job_backend = backend; + job_graph = graph; + job_status = GGML_STATUS_SUCCESS; + job_done = false; + job_ready = true; + } + cv.notify_all(); + pending = true; + } + + // wait for the in-flight job (if any); returns its status + enum ggml_status join() { + if (!pending) { + return GGML_STATUS_SUCCESS; + } + std::unique_lock lock(mtx); + cv.wait(lock, [this]() { return job_done; }); + pending = false; + return job_status; + } +}; + struct ggml_backend_sched { bool is_reset; // true if the scheduler has been reset since the last graph split bool is_alloc; @@ -835,6 +910,9 @@ struct ggml_backend_sched { bool prefetch_used[GGML_SCHED_MAX_PREFETCH_SLOTS]; int prefetch_cur; + // async CPU split execution (GGML_SCHED_ASYNC_CPU); NULL when disabled + struct ggml_sched_cpu_async * cpu_async; + int debug; // used for debugging graph reallocations [GGML_SCHED_DEBUG_REALLOC] @@ -1647,6 +1725,11 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s GGML_ASSERT(sched); struct ggml_backend_sched_split * splits = sched->splits; + if (sched->cpu_async) { + // a job left over from an aborted eval references stale split memory - drain it + sched->cpu_async->join(); + } + ggml_tensor * prev_ids_tensor = nullptr; std::vector ids; std::vector used_ids; @@ -1660,6 +1743,22 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s ggml_backend_buffer_t prefetch_saved_buffer = NULL; void * prefetch_saved_data = NULL; + // an async CPU split may still be computing; join before anything that + // depends on it: another CPU split, or a split reading a CPU tensor + if (sched->cpu_async && sched->cpu_async->pending) { + bool must_join = split_backend_id == sched->n_backends - 1; + for (int input_id = 0; !must_join && input_id < split->n_inputs; input_id++) { + ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[input_id]); + must_join = input_backend == sched->backends[sched->n_backends - 1]; + } + if (must_join) { + enum ggml_status ec = sched->cpu_async->join(); + if (ec != GGML_STATUS_SUCCESS) { + return ec; + } + } + } + // copy the input tensors to the split backend for (int input_id = 0; input_id < split->n_inputs; input_id++) { ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[input_id]); @@ -1822,6 +1921,12 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s } if (!sched->callback_eval) { + if (sched->cpu_async && split_backend_id == sched->n_backends - 1 && split_prefetch_slot == -1) { + // run the CPU split on the worker; the loop continues launching + // later splits until one depends on this split's outputs + sched->cpu_async->launch(split_backend, &split->graph); + continue; + } enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &split->graph); if (split_prefetch_slot != -1) { // the kernels have captured the slot address at launch, safe to restore @@ -1875,6 +1980,13 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s } } + if (sched->cpu_async) { + enum ggml_status ec = sched->cpu_async->join(); + if (ec != GGML_STATUS_SUCCESS) { + return ec; + } + } + return GGML_STATUS_SUCCESS; } @@ -1951,6 +2063,11 @@ ggml_backend_sched_t ggml_backend_sched_new( // default of 3 covers the gate/up/down expert tensors of one MoE layer sched->prefetch_n_slots = prefetch_n_slots <= 1 ? 3 : std::min(prefetch_n_slots, GGML_SCHED_MAX_PREFETCH_SLOTS); + const char * GGML_SCHED_ASYNC_CPU = getenv("GGML_SCHED_ASYNC_CPU"); + if (GGML_SCHED_ASYNC_CPU && atoi(GGML_SCHED_ASYNC_CPU) > 0) { + sched->cpu_async = new ggml_sched_cpu_async(); + } + ggml_backend_sched_reset(sched); return sched; @@ -1975,6 +2092,7 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) { } ggml_backend_free(sched->prefetch_backend); } + delete sched->cpu_async; ggml_gallocr_free(sched->galloc); ggml_free(sched->ctx); ggml_hash_set_free(&sched->hash_set); @@ -2076,6 +2194,9 @@ enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sch void ggml_backend_sched_synchronize(ggml_backend_sched_t sched) { GGML_ASSERT(sched); + if (sched->cpu_async) { + sched->cpu_async->join(); + } for (int i = 0; i < sched->n_backends; i++) { ggml_backend_synchronize(sched->backends[i]); } diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 41ddf9673065..d8d0f950cb32 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -2009,14 +2009,28 @@ ggml_tensor * llm_graph_context::build_moe_ffn( return down; }; - ggml_tensor * hot = build_pack_chain(moe_cache->ffn_gate_exps_hot, moe_cache->ffn_up_exps_hot, moe_cache->ffn_down_exps_hot, ids_hot); - cb(hot, "ffn_moe_down_hot", il); + // cold chain is built first so its nodes precede the hot chain in the + // graph: the scheduler then emits [cold split, hot split] and, with + // GGML_SCHED_ASYNC_CPU, computes the cold chain on a worker while the + // hot chain (which has no CPU inputs) runs concurrently on the GPU. + // pinning the merge to CPU keeps it out of the hot split so the hot + // split stays free of cross-backend inputs. + static const bool sched_async_cpu = [] { + const char * v = getenv("GGML_SCHED_ASYNC_CPU"); + return v && atoi(v) > 0; + }(); ggml_tensor * cold = build_pack_chain(gate_exps, up_exps, down_exps, ids_cold); cb(cold, "ffn_moe_down_cold", il); - experts = ggml_add(ctx0, hot, cold); + ggml_tensor * hot = build_pack_chain(moe_cache->ffn_gate_exps_hot, moe_cache->ffn_up_exps_hot, moe_cache->ffn_down_exps_hot, ids_hot); + cb(hot, "ffn_moe_down_hot", il); + + experts = ggml_add(ctx0, cold, hot); cb(experts, "ffn_moe_down", il); + if (sched_async_cpu) { + ggml_backend_sched_set_tensor_backend(sched, experts, backend_cpu); + } } else if (gate_up_exps) { // merged gate_up path: one mul_mat_id, then split into gate and up views ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, selected_experts, up_exps_s); // [n_ff*2, n_expert_used, n_tokens] From d11c8a3807694a43ad2fa103e6085e0160b4daf7 Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Sun, 23 Aug 2026 01:49:23 +0530 Subject: [PATCH 2/5] feat: GGML_SCHED_ASYNC_CPU=2 diagnostic mode (worker without graph restructure) --- src/llama-graph.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index d8d0f950cb32..b26b42344ec1 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -2015,9 +2015,10 @@ ggml_tensor * llm_graph_context::build_moe_ffn( // hot chain (which has no CPU inputs) runs concurrently on the GPU. // pinning the merge to CPU keeps it out of the hot split so the hot // split stays free of cross-backend inputs. + // 1 = worker + graph restructure (overlap); 2 = sched worker only (diagnostic) static const bool sched_async_cpu = [] { const char * v = getenv("GGML_SCHED_ASYNC_CPU"); - return v && atoi(v) > 0; + return v && atoi(v) == 1; }(); ggml_tensor * cold = build_pack_chain(gate_exps, up_exps, down_exps, ids_cold); From 4806d5c4953fab36db94fb8181741f336c4314b2 Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Sun, 23 Aug 2026 01:52:41 +0530 Subject: [PATCH 3/5] fix: mark expert cache pack buffer as weights Without the weights usage flag the scheduler's assignment pass does not pin the pack tensors to their backend, so a CPU-assigned consumer (the pinned merge in async mode) propagates backward and pulls the hot matmuls onto the CPU, copying pack weights over PCIe every layer. --- src/llama-model.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 85e495e23897..2dfe7eefc9fb 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1764,6 +1764,10 @@ void llama_model_base::init_moe_expert_cache() { } return; } + // weights usage pins the pack tensors to their backend during graph + // assignment - without it a CPU-assigned consumer can drag the hot + // matmuls (and a per-layer weight copy) onto the CPU + ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); // fill packs (expert dim is outermost: one contiguous slab per expert) std::vector slab; From a648bba343f6f5ca5c7cbe879633c20c39590c5f Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Sun, 23 Aug 2026 02:02:28 +0530 Subject: [PATCH 4/5] feat: make async CPU splits the default, expose --sched-async-cpu flag Replaces the GGML_SCHED_ASYNC_CPU env var: the overlap is now on by default and controlled through llama_context_params.sched_async_cpu. - ggml: ggml_backend_sched_set_async_cpu() setter, env read removed - llama: context param + cparams plumbing, graph gate reads cparams - common: --sched-async-cpu / --no-sched-async-cpu (server, cli) - llama-bench: sched_async_cpu test dimension (--sched-async-cpu 0,1 benches both in one run, column shown as 'sac' when non-default) --- common/arg.cpp | 8 +++++++ common/common.cpp | 1 + common/common.h | 1 + ggml/include/ggml-backend.h | 4 ++++ ggml/src/ggml-backend.cpp | 15 +++++++++---- include/llama.h | 1 + src/llama-context.cpp | 4 ++++ src/llama-cparams.h | 1 + src/llama-graph.cpp | 10 ++------- tools/llama-bench/llama-bench.cpp | 35 +++++++++++++++++++++++++++++-- 10 files changed, 66 insertions(+), 14 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index c4c1cdbb70e5..69cbbe89e805 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2805,6 +2805,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.no_op_offload = !value; } )); + add_opt(common_arg( + {"--sched-async-cpu"}, + {"--no-sched-async-cpu"}, + string_format("whether to run CPU graph splits on a worker thread so independent GPU splits overlap them (default: %s)", params.sched_async_cpu ? "true" : "false"), + [](common_params & params, bool value) { + params.sched_async_cpu = value; + } + )); add_opt(common_arg( {"--lora"}, "FNAME", "path to LoRA adapter (use comma-separated values to load multiple adapters)", diff --git a/common/common.cpp b/common/common.cpp index 2dff9454bfaf..4313d3fd2a8a 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1619,6 +1619,7 @@ struct llama_context_params common_context_params_to_llama(const common_params & cparams.offload_kqv = !params.no_kv_offload; cparams.no_perf = params.no_perf; cparams.op_offload = !params.no_op_offload; + cparams.sched_async_cpu = params.sched_async_cpu; cparams.swa_full = params.swa_full; cparams.kv_unified = params.kv_unified; diff --git a/common/common.h b/common/common.h index cc05835fd488..fa1adefe5ffb 100644 --- a/common/common.h +++ b/common/common.h @@ -580,6 +580,7 @@ struct common_params { bool warmup = true; // warmup run bool check_tensors = false; // validate tensor data bool no_op_offload = false; // globally disable offload host tensor operations to device + bool sched_async_cpu = true; // run CPU graph splits on a worker thread (overlaps independent GPU splits) bool no_extra_bufts = false; // disable extra buffer types (used for weight repacking) bool no_host = false; // bypass host buffer allowing extra buffers to be used diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h index 2924fdbe9884..3001ba3bd6c4 100644 --- a/ggml/include/ggml-backend.h +++ b/ggml/include/ggml-backend.h @@ -332,6 +332,10 @@ extern "C" { GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend); GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend); + + // when enabled, CPU splits run on a worker thread so that independent splits + // on other backends execute concurrently with them + GGML_API void ggml_backend_sched_set_async_cpu(ggml_backend_sched_t sched, bool enable); GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node); // Split graph without allocating it diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 0fe739e686ed..86b661bd5df1 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -2063,10 +2063,6 @@ ggml_backend_sched_t ggml_backend_sched_new( // default of 3 covers the gate/up/down expert tensors of one MoE layer sched->prefetch_n_slots = prefetch_n_slots <= 1 ? 3 : std::min(prefetch_n_slots, GGML_SCHED_MAX_PREFETCH_SLOTS); - const char * GGML_SCHED_ASYNC_CPU = getenv("GGML_SCHED_ASYNC_CPU"); - if (GGML_SCHED_ASYNC_CPU && atoi(GGML_SCHED_ASYNC_CPU) > 0) { - sched->cpu_async = new ggml_sched_cpu_async(); - } ggml_backend_sched_reset(sched); @@ -2192,6 +2188,17 @@ enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sch return ggml_backend_sched_compute_splits(sched); } +void ggml_backend_sched_set_async_cpu(ggml_backend_sched_t sched, bool enable) { + GGML_ASSERT(sched); + if (enable && sched->cpu_async == NULL) { + sched->cpu_async = new ggml_sched_cpu_async(); + } else if (!enable && sched->cpu_async != NULL) { + sched->cpu_async->join(); + delete sched->cpu_async; + sched->cpu_async = NULL; + } +} + void ggml_backend_sched_synchronize(ggml_backend_sched_t sched) { GGML_ASSERT(sched); if (sched->cpu_async) { diff --git a/include/llama.h b/include/llama.h index 7f494ba01a5e..a234213e49a6 100644 --- a/include/llama.h +++ b/include/llama.h @@ -392,6 +392,7 @@ extern "C" { bool offload_kqv; // offload the KQV ops (including the KV cache) to GPU bool no_perf; // measure performance timings bool op_offload; // offload host tensor operations to device + bool sched_async_cpu; // run CPU graph splits on a worker thread so independent GPU splits overlap them bool swa_full; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) // NOTE: setting to false when n_seq_max > 1 can cause bad performance in some cases // ref: https://github.com/ggml-org/llama.cpp/pull/13845#issuecomment-2924800573 diff --git a/src/llama-context.cpp b/src/llama-context.cpp index eed041eef4e7..e75e3f9564ee 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -117,6 +117,7 @@ llama_context::llama_context( cparams.embeddings_nextn = false; cparams.embeddings_nextn_masked = false; cparams.offload_kqv = params.offload_kqv; + cparams.sched_async_cpu = params.sched_async_cpu; cparams.no_perf = params.no_perf; cparams.warmup = false; @@ -594,6 +595,7 @@ void llama_context::sched_reserve() { gf_res_reserve.reset(new llm_graph_result(max_nodes)); sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, cparams.pipeline_parallel, cparams.op_offload)); + ggml_backend_sched_set_async_cpu(sched.get(), cparams.sched_async_cpu); llama_memory_context_ptr mctx; if (memory) { @@ -629,6 +631,7 @@ void llama_context::sched_reserve() { LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__); cparams.pipeline_parallel = false; sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, false, cparams.op_offload)); + ggml_backend_sched_set_async_cpu(sched.get(), cparams.sched_async_cpu); gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get()); } if (!gf) { @@ -3494,6 +3497,7 @@ llama_context_params llama_context_default_params() { /*.offload_kqv =*/ true, /*.no_perf =*/ true, /*.op_offload =*/ true, + /*.sched_async_cpu =*/ true, /*.swa_full =*/ true, /*.kv_unified =*/ false, /*.sampler =*/ nullptr, diff --git a/src/llama-cparams.h b/src/llama-cparams.h index 5018170ed85e..8ba78eb710ee 100644 --- a/src/llama-cparams.h +++ b/src/llama-cparams.h @@ -36,6 +36,7 @@ struct llama_cparams { bool embeddings_nextn_masked; // extract for only rows where batch.logits != 0 bool causal_attn; bool offload_kqv; + bool sched_async_cpu; bool flash_attn; bool auto_fa; bool fused_gdn_ar; // use fused gated delta net (autoregressive) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index b26b42344ec1..5426812ed13f 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -2011,16 +2011,10 @@ ggml_tensor * llm_graph_context::build_moe_ffn( // cold chain is built first so its nodes precede the hot chain in the // graph: the scheduler then emits [cold split, hot split] and, with - // GGML_SCHED_ASYNC_CPU, computes the cold chain on a worker while the + // async CPU splits, computes the cold chain on a worker while the // hot chain (which has no CPU inputs) runs concurrently on the GPU. // pinning the merge to CPU keeps it out of the hot split so the hot // split stays free of cross-backend inputs. - // 1 = worker + graph restructure (overlap); 2 = sched worker only (diagnostic) - static const bool sched_async_cpu = [] { - const char * v = getenv("GGML_SCHED_ASYNC_CPU"); - return v && atoi(v) == 1; - }(); - ggml_tensor * cold = build_pack_chain(gate_exps, up_exps, down_exps, ids_cold); cb(cold, "ffn_moe_down_cold", il); @@ -2029,7 +2023,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn( experts = ggml_add(ctx0, cold, hot); cb(experts, "ffn_moe_down", il); - if (sched_async_cpu) { + if (cparams.sched_async_cpu) { ggml_backend_sched_set_tensor_backend(sched, experts, backend_cpu); } } else if (gate_up_exps) { diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cpp index 8e1facd80a94..d7321158375c 100644 --- a/tools/llama-bench/llama-bench.cpp +++ b/tools/llama-bench/llama-bench.cpp @@ -349,6 +349,7 @@ struct cmd_params { std::vector> tensor_buft_overrides; std::vector embeddings; std::vector no_op_offload; + std::vector sched_async_cpu; std::vector no_host; std::vector fit_params_target; std::vector fit_params_min_ctx; @@ -393,6 +394,7 @@ static const cmd_params cmd_params_defaults = { /* tensor_buft_overrides*/ { std::vector{ { nullptr, nullptr } } }, /* embeddings */ { false }, /* no_op_offload */ { false }, + /* sched_async_cpu */ { true }, /* no_host */ { false }, /* fit_params_target */ { 0 }, /* fit_params_min_ctx */ { 0 }, @@ -467,6 +469,7 @@ static void print_usage(int /* argc */, char ** argv) { printf(" -ot --override-tensor =;...\n"); printf(" (default: disabled)\n"); printf(" -nopo, --no-op-offload <0|1> (default: 0)\n"); + printf(" --sched-async-cpu <0|1> (default: 1)\n"); printf(" --no-host <0|1> (default: %s)\n", join(cmd_params_defaults.no_host, ",").c_str()); printf("\n"); printf( @@ -913,6 +916,13 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { } auto p = string_split(argv[i], split_delim); params.no_op_offload.insert(params.no_op_offload.end(), p.begin(), p.end()); + } else if (arg == "--sched-async-cpu") { + if (++i >= argc) { + invalid_param = true; + break; + } + auto p = string_split(argv[i], split_delim); + params.sched_async_cpu.insert(params.sched_async_cpu.end(), p.begin(), p.end()); } else if (arg == "--no-host") { if (++i >= argc) { invalid_param = true; @@ -1181,6 +1191,9 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { if (params.no_op_offload.empty()) { params.no_op_offload = cmd_params_defaults.no_op_offload; } + if (params.sched_async_cpu.empty()) { + params.sched_async_cpu = cmd_params_defaults.sched_async_cpu; + } if (params.no_host.empty()) { params.no_host = cmd_params_defaults.no_host; } @@ -1231,6 +1244,7 @@ struct cmd_params_instance { std::vector tensor_buft_overrides; bool embeddings; bool no_op_offload; + bool sched_async_cpu; bool no_host; size_t fit_target; uint32_t fit_min_ctx; @@ -1307,6 +1321,7 @@ struct cmd_params_instance { cparams.flash_attn_type = flash_attn; cparams.embeddings = embeddings; cparams.op_offload = !no_op_offload; + cparams.sched_async_cpu = sched_async_cpu; cparams.swa_full = false; return cparams; @@ -1332,6 +1347,7 @@ static std::vector get_cmd_params_instances(const cmd_param for (const auto & noh : params.no_host) for (const auto & embd : params.embeddings) for (const auto & nopo : params.no_op_offload) + for (const auto & sac : params.sched_async_cpu) for (const auto & nb : params.n_batch) for (const auto & nub : params.n_ubatch) for (const auto & tk : params.type_k) @@ -1372,6 +1388,7 @@ static std::vector get_cmd_params_instances(const cmd_param /* .tensor_buft_overrides = */ ot, /* .embeddings = */ embd, /* .no_op_offload = */ nopo, + /* .sched_async_cpu = */ sac, /* .no_host = */ noh, /* .fit_target = */ fpt, /* .fit_min_ctx = */ fpc, @@ -1408,6 +1425,7 @@ static std::vector get_cmd_params_instances(const cmd_param /* .tensor_buft_overrides = */ ot, /* .embeddings = */ embd, /* .no_op_offload = */ nopo, + /* .sched_async_cpu = */ sac, /* .no_host = */ noh, /* .fit_target = */ fpt, /* .fit_min_ctx = */ fpc, @@ -1444,6 +1462,7 @@ static std::vector get_cmd_params_instances(const cmd_param /* .tensor_buft_overrides = */ ot, /* .embeddings = */ embd, /* .no_op_offload = */ nopo, + /* .sched_async_cpu = */ sac, /* .no_host = */ noh, /* .fit_target = */ fpt, /* .fit_min_ctx = */ fpc, @@ -1485,6 +1504,7 @@ struct test { std::vector tensor_buft_overrides; bool embeddings; bool no_op_offload; + bool sched_async_cpu; bool no_host; size_t fit_target; uint32_t fit_min_ctx; @@ -1524,6 +1544,7 @@ struct test { tensor_buft_overrides = inst.tensor_buft_overrides; embeddings = inst.embeddings; no_op_offload = inst.no_op_offload; + sched_async_cpu = inst.sched_async_cpu; no_host = inst.no_host; fit_target = inst.fit_target; fit_min_ctx = inst.fit_min_ctx; @@ -1584,7 +1605,7 @@ struct test { "type_k", "type_v", "n_gpu_layers", "n_cpu_moe", "split_mode", "main_gpu", "no_kv_offload", "flash_attn", "devices", "tensor_split", "tensor_buft_overrides", "load_mode", "embeddings", - "no_op_offload", "no_host", "fit_target", "fit_min_ctx", + "no_op_offload", "sched_async_cpu", "no_host", "fit_target", "fit_min_ctx", "n_prompt", "n_gen", "n_depth", "test_time", "avg_ns", "stddev_ns", "avg_ts", "stddev_ts" }; @@ -1602,7 +1623,7 @@ struct test { return INT; } if (field == "f16_kv" || field == "no_kv_offload" || field == "cpu_strict" || - field == "embeddings" || field == "no_host") { + field == "embeddings" || field == "no_host" || field == "sched_async_cpu") { return BOOL; } if (field == "avg_ts" || field == "stddev_ts") { @@ -1680,6 +1701,7 @@ struct test { llama_load_mode_name(load_mode), std::to_string(embeddings), std::to_string(no_op_offload), + std::to_string(sched_async_cpu), std::to_string(no_host), std::to_string(fit_target), std::to_string(fit_min_ctx), @@ -1874,6 +1896,9 @@ struct markdown_printer : public printer { if (field == "no_host") { return 4; } + if (field == "sched_async_cpu") { + return 4; + } int width = std::max((int) field.length(), 10); @@ -1908,6 +1933,9 @@ struct markdown_printer : public printer { if (field == "no_op_offload") { return "nopo"; } + if (field == "sched_async_cpu") { + return "sac"; + } if (field == "no_host") { return "noh"; } @@ -1998,6 +2026,9 @@ struct markdown_printer : public printer { if (params.no_op_offload.size() > 1 || params.no_op_offload != cmd_params_defaults.no_op_offload) { fields.emplace_back("no_op_offload"); } + if (params.sched_async_cpu.size() > 1 || params.sched_async_cpu != cmd_params_defaults.sched_async_cpu) { + fields.emplace_back("sched_async_cpu"); + } if (params.no_host.size() > 1 || params.no_host != cmd_params_defaults.no_host) { fields.emplace_back("no_host"); } From 1fcb8313b7b915f68357679ea646c6242a21d54e Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Sun, 23 Aug 2026 02:10:40 +0530 Subject: [PATCH 5/5] fix: restrict CPU-pinned merge to decode-size batches For prefill batches the CPU merge and its per-layer activation copies cost more than the overlap hides (pp256 regressed 307 -> 257 t/s). Decode and MTP verify batches (n_tokens <= 8) keep the overlap. --- src/llama-graph.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 5426812ed13f..b50ef57bc455 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -2023,7 +2023,9 @@ ggml_tensor * llm_graph_context::build_moe_ffn( experts = ggml_add(ctx0, cold, hot); cb(experts, "ffn_moe_down", il); - if (cparams.sched_async_cpu) { + // decode-size batches only: for large (prefill) batches the CPU merge + // and its per-layer activation copies cost more than the overlap hides + if (cparams.sched_async_cpu && n_tokens <= 8) { ggml_backend_sched_set_tensor_backend(sched, experts, backend_cpu); } } else if (gate_up_exps) {