feat: add coding model filtering and CUDA acceleration

- claude: filter model completion to coding/synthetic models only
- llama-swap: update model to IQ4_XS and add CUDA device selection
This commit is contained in:
2026-02-26 15:47:25 -05:00
parent 8e1a2ad85a
commit bf7cc81a61
2 changed files with 13 additions and 4 deletions

View File

@@ -165,19 +165,20 @@ in
};
};
# https://huggingface.co/unsloth/Qwen3.5-35B-A3B-GGUF/tree/main
# https://huggingface.co/AesSedai/Qwen3.5-35B-A3B-GGUF/tree/main
"qwen3.5-35b-thinking" = {
name = "Qwen3.5 (35B) - Thinking";
macros.ctx = "262144";
cmd = ''
${llama-cpp}/bin/llama-server \
--port ''${PORT} \
-m /mnt/ssd/Models/Qwen3.5/Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf \
-m /mnt/ssd/Models/Qwen3.5/Qwen3.5-35B-A3B-IQ4_XS-00001-of-00002.gguf \
-c ''${ctx} \
--temp 0.6 \
--top-p 0.95 \
--top-k 20 \
--min-p 0.00 \
-dev CUDA0 \
-fit off
'';
# --chat-template-kwargs "{\"enable_thinking\": false}"