feat: add coding model filtering and CUDA acceleration
- claude: filter model completion to coding/synthetic models only - llama-swap: update model to IQ4_XS and add CUDA device selection
This commit is contained in:
@@ -69,13 +69,21 @@ in
|
|||||||
local prev=''${COMP_WORDS[COMP_CWORD-1]}
|
local prev=''${COMP_WORDS[COMP_CWORD-1]}
|
||||||
|
|
||||||
if [[ "$prev" == "-m" || "$prev" == "--model" ]]; then
|
if [[ "$prev" == "-m" || "$prev" == "--model" ]]; then
|
||||||
local models=( $(${pkgs.curl}/bin/curl -s -H "Authorization: Bearer ${authToken}" "${baseUrl}/v1/models" | ${pkgs.jq}/bin/jq -r '.data[].id' 2>/dev/null) )
|
local models=( $(${pkgs.curl}/bin/curl -s -H "Authorization: Bearer ${authToken}" "${baseUrl}/v1/models" | ${pkgs.jq}/bin/jq -r '
|
||||||
|
.data[] |
|
||||||
|
select(
|
||||||
|
(try (.meta.llamaswap.type[] | contains("coding")) catch false) or
|
||||||
|
(.name | startswith("synthetic:"))
|
||||||
|
) |
|
||||||
|
.id
|
||||||
|
' 2>/dev/null) )
|
||||||
|
|
||||||
COMPREPLY=( $(compgen -W "''${models[*]}" -- "$cur") )
|
COMPREPLY=( $(compgen -W "''${models[*]}" -- "$cur") )
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# Register Completion
|
# Register Completion
|
||||||
complete -F _complete_claude_custom claude
|
complete -F _complete_claude_custom claude-custom
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -165,19 +165,20 @@ in
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
# https://huggingface.co/unsloth/Qwen3.5-35B-A3B-GGUF/tree/main
|
# https://huggingface.co/AesSedai/Qwen3.5-35B-A3B-GGUF/tree/main
|
||||||
"qwen3.5-35b-thinking" = {
|
"qwen3.5-35b-thinking" = {
|
||||||
name = "Qwen3.5 (35B) - Thinking";
|
name = "Qwen3.5 (35B) - Thinking";
|
||||||
macros.ctx = "262144";
|
macros.ctx = "262144";
|
||||||
cmd = ''
|
cmd = ''
|
||||||
${llama-cpp}/bin/llama-server \
|
${llama-cpp}/bin/llama-server \
|
||||||
--port ''${PORT} \
|
--port ''${PORT} \
|
||||||
-m /mnt/ssd/Models/Qwen3.5/Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf \
|
-m /mnt/ssd/Models/Qwen3.5/Qwen3.5-35B-A3B-IQ4_XS-00001-of-00002.gguf \
|
||||||
-c ''${ctx} \
|
-c ''${ctx} \
|
||||||
--temp 0.6 \
|
--temp 0.6 \
|
||||||
--top-p 0.95 \
|
--top-p 0.95 \
|
||||||
--top-k 20 \
|
--top-k 20 \
|
||||||
--min-p 0.00 \
|
--min-p 0.00 \
|
||||||
|
-dev CUDA0 \
|
||||||
-fit off
|
-fit off
|
||||||
'';
|
'';
|
||||||
# --chat-template-kwargs "{\"enable_thinking\": false}"
|
# --chat-template-kwargs "{\"enable_thinking\": false}"
|
||||||
|
|||||||
Reference in New Issue
Block a user