evan/nix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(llama-swap): replace --parallel with -np and add -kvu flag

Browse Source 
Switch llama-server invocations from --parallel to -np with -kvu
(kv-cache unified) across Qwen3.6 model configs. Also reduce
context for qwen3.6-27b-cuda0 from 150k to 140k.
This commit is contained in:
Evan Reichard
2026-05-19 06:22:08 -04:00
parent 5b2111c7e8
commit 81ffe67cce
1 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
modules/nixos/services/llama-swap/config.nix
View File

@@ -41,7 +41,7 @@ in
--port ''${PORT} \ --port ''${PORT} \
-m /mnt/ssd/Models/Qwen3.6/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf \ -m /mnt/ssd/Models/Qwen3.6/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf \
-c ''${ctx} \ -c ''${ctx} \
--parallel 1 \ -np 2 -kvu \
--temp 0.6 \ --temp 0.6 \
--top-p 0.95 \ --top-p 0.95 \
--top-k 20 \ --top-k 20 \
@@ -66,13 +66,13 @@ in
# https://huggingface.co/unsloth/Qwen3.6-27B-GGUF-MTP/tree/main # https://huggingface.co/unsloth/Qwen3.6-27B-GGUF-MTP/tree/main
"qwen3.6-27b-cuda0" = { "qwen3.6-27b-cuda0" = {
name = "Qwen3.6 27B (CUDA0, UD-Q4)"; name = "Qwen3.6 27B (CUDA0, UD-Q4)";
macros.ctx = "150000"; macros.ctx = "140000";
cmd = '' cmd = ''
${llama-cpp}/bin/llama-server \ ${llama-cpp}/bin/llama-server \
--port ''${PORT} \ --port ''${PORT} \
-m /mnt/ssd/Models/Qwen3.6/Qwen3.6-27B-UD-Q4_K_XL.gguf \ -m /mnt/ssd/Models/Qwen3.6/Qwen3.6-27B-UD-Q4_K_XL.gguf \
-c ''${ctx} \ -c ''${ctx} \
--parallel 1 \ -np 2 -kvu \
--temp 0.6 \ --temp 0.6 \
--top-p 0.95 \ --top-p 0.95 \
--top-k 20 \ --top-k 20 \
@@ -634,7 +634,7 @@ in
--port ''${PORT} \ --port ''${PORT} \
-m /mnt/ssd/Models/Qwen3.6/Qwen3.6-27B-UD-Q6_K_XL.gguf \ -m /mnt/ssd/Models/Qwen3.6/Qwen3.6-27B-UD-Q6_K_XL.gguf \
-c ''${ctx} \ -c ''${ctx} \
--parallel 1 \ -np 4 -kvu \
--temp 0.6 \ --temp 0.6 \
--top-p 0.95 \ --top-p 0.95 \
--top-k 20 \ --top-k 20 \
@@ -666,7 +666,7 @@ in
--port ''${PORT} \ --port ''${PORT} \
-m /mnt/ssd/Models/Qwen3.6/Qwen3.6-35B-A3B-UD-Q6_K.gguf \ -m /mnt/ssd/Models/Qwen3.6/Qwen3.6-35B-A3B-UD-Q6_K.gguf \
-c ''${ctx} \ -c ''${ctx} \
--parallel 2 \ -np 4 -kvu \
--temp 0.6 \ --temp 0.6 \
--top-p 0.95 \ --top-p 0.95 \
--top-k 20 \ --top-k 20 \