evan/nix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

config(llama-swap): enable preserve_thinking in chat template kwargs

Browse Source
This commit is contained in:
Evan Reichard
2026-04-30 07:45:57 -04:00
parent d1d3f3c1a3
commit 976edab339
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
modules/nixos/services/llama-swap/config.nix
View File

@@ -140,7 +140,8 @@ in
--presence-penalty 0.0 \ --presence-penalty 0.0 \
--repeat-penalty 1.0 \ --repeat-penalty 1.0 \
-dev CUDA0 \ -dev CUDA0 \
-fit off -fit off \
--chat-template-kwargs "{\"preserve_thinking\": true}"
''; '';
metadata = { metadata = {
type = [ type = [
@@ -168,7 +169,6 @@ in
-dev CUDA0 \ -dev CUDA0 \
-fit off -fit off
''; '';
# --chat-template-kwargs "{\"enable_thinking\": false}"
metadata = { metadata = {
type = [ type = [
"text-generation" "text-generation"
@@ -194,7 +194,8 @@ in
-ctk q8_0 \ -ctk q8_0 \
-ctv q8_0 \ -ctv q8_0 \
-dev CUDA0 \ -dev CUDA0 \
-fit off -fit off \
--chat-template-kwargs "{\"preserve_thinking\": true}"
''; '';
# --chat-template-kwargs "{\"enable_thinking\": false}" # --chat-template-kwargs "{\"enable_thinking\": false}"
metadata = { metadata = {