From 81ffe67cce26c451e6cd9e0f09fa68b566cf23c7 Mon Sep 17 00:00:00 2001 From: Evan Reichard Date: Tue, 19 May 2026 06:22:08 -0400 Subject: [PATCH] refactor(llama-swap): replace --parallel with -np and add -kvu flag Switch llama-server invocations from --parallel to -np with -kvu (kv-cache unified) across Qwen3.6 model configs. Also reduce context for qwen3.6-27b-cuda0 from 150k to 140k. --- modules/nixos/services/llama-swap/config.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/nixos/services/llama-swap/config.nix b/modules/nixos/services/llama-swap/config.nix index a90d393..081234e 100644 --- a/modules/nixos/services/llama-swap/config.nix +++ b/modules/nixos/services/llama-swap/config.nix @@ -41,7 +41,7 @@ in --port ''${PORT} \ -m /mnt/ssd/Models/Qwen3.6/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf \ -c ''${ctx} \ - --parallel 1 \ + -np 2 -kvu \ --temp 0.6 \ --top-p 0.95 \ --top-k 20 \ @@ -66,13 +66,13 @@ in # https://huggingface.co/unsloth/Qwen3.6-27B-GGUF-MTP/tree/main "qwen3.6-27b-cuda0" = { name = "Qwen3.6 27B (CUDA0, UD-Q4)"; - macros.ctx = "150000"; + macros.ctx = "140000"; cmd = '' ${llama-cpp}/bin/llama-server \ --port ''${PORT} \ -m /mnt/ssd/Models/Qwen3.6/Qwen3.6-27B-UD-Q4_K_XL.gguf \ -c ''${ctx} \ - --parallel 1 \ + -np 2 -kvu \ --temp 0.6 \ --top-p 0.95 \ --top-k 20 \ @@ -634,7 +634,7 @@ in --port ''${PORT} \ -m /mnt/ssd/Models/Qwen3.6/Qwen3.6-27B-UD-Q6_K_XL.gguf \ -c ''${ctx} \ - --parallel 1 \ + -np 4 -kvu \ --temp 0.6 \ --top-p 0.95 \ --top-k 20 \ @@ -666,7 +666,7 @@ in --port ''${PORT} \ -m /mnt/ssd/Models/Qwen3.6/Qwen3.6-35B-A3B-UD-Q6_K.gguf \ -c ''${ctx} \ - --parallel 2 \ + -np 4 -kvu \ --temp 0.6 \ --top-p 0.95 \ --top-k 20 \