evan/nix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add fim 4b + 9b

Browse Source
This commit is contained in:
Evan Reichard
2026-05-01 21:42:50 -04:00
parent 09fdff4908
commit 1283b7cdef
2 changed files with 50 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
modules/home/programs/terminal/nvim/config/lua/llm-config.lua
View File

@@ -1,7 +1,7 @@
local llm_endpoint = "https://llm-api.va.reichard.io" local llm_endpoint = "https://llm-api.va.reichard.io"
local llm_assistant_model = "qwen3.5-27b-thinking" local llm_assistant_model = "vllm-qwen3.6-27b-tools-text "
local llm_infill_model = llm_assistant_model local llm_infill_model = "qwen3.5-4b-thinking"
local current_fim = "copilot" local current_fim = "llama"
-- Copilot Configuration -- Copilot Configuration
vim.g.copilot_no_tab_map = true vim.g.copilot_no_tab_map = true

65
modules/nixos/services/llama-swap/config.nix
View File

@@ -12,6 +12,25 @@ in
# -------------- RTX 3090 --------------- # -------------- RTX 3090 ---------------
# --------------------------------------- # ---------------------------------------
# https://huggingface.co/mradermacher/gpt-oss-20b-heretic-v2-i1-GGUF/tree/main
"gpt-oss-20b-thinking" = {
name = "GPT OSS (20B) - Thinking";
macros.ctx = "131072";
cmd = ''
${llama-cpp}/bin/llama-server \
--port ''${PORT} \
-m /mnt/ssd/Models/GPT-OSS/gpt-oss-20b-heretic-v2.i1-MXFP4_MOE.gguf \
-c ''${ctx} \
--temp 1.0 \
--top-p 1.0 \
--top-k 40 \
-dev CUDA0
'';
metadata = {
type = [ "text-generation" ];
};
};
# https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF/tree/main # https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF/tree/main
"qwen3.6-35b-thinking" = { "qwen3.6-35b-thinking" = {
name = "Qwen3.6 (35B) - Thinking"; name = "Qwen3.6 (35B) - Thinking";
@@ -369,42 +388,51 @@ in
}; };
}; };
# https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/tree/main # https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/tree/main
"qwen3-4b-2507-instruct" = { "qwen3.5-9b-thinking" = {
name = "Qwen3 2507 (4B) - Instruct"; name = "Qwen3.5 (9B) - Thinking";
macros.ctx = "98304"; macros.ctx = "131072";
env = [ "CUDA_VISIBLE_DEVICES=1" ]; env = [ "CUDA_VISIBLE_DEVICES=1" ];
cmd = '' cmd = ''
${llama-cpp}/bin/llama-server \ ${llama-cpp}/bin/llama-server \
--port ''${PORT} \ --port ''${PORT} \
-m /mnt/ssd/Models/Qwen3/Qwen3-4B-Instruct-2507-Q4_K_M.gguf \ -m /mnt/ssd/Models/Qwen3.5/Qwen3.5-9B-IQ4_XS.gguf \
-c ''${ctx} \ -c ''${ctx} \
--temp 0.6 \
--top-p 0.95 \
--top-k 20 \
--min-p 0.0 \
-fit off \ -fit off \
-ctk q8_0 \
-ctv q8_0 \
-dev CUDA0 -dev CUDA0
''; '';
metadata = { metadata = {
type = [ "text-generation" ]; type = [
"text-generation"
];
}; };
}; };
# https://huggingface.co/mradermacher/gpt-oss-20b-heretic-v2-i1-GGUF/tree/main # https://huggingface.co/unsloth/Qwen3.5-4B-GGUF/tree/main
"gpt-oss-20b-thinking" = { "qwen3.5-4b-thinking" = {
name = "GPT OSS (20B) - Thinking"; name = "Qwen3.5 (4B) - Thinking";
macros.ctx = "131072"; macros.ctx = "131072";
env = [ "CUDA_VISIBLE_DEVICES=1" ];
cmd = '' cmd = ''
${llama-cpp}/bin/llama-server \ ${llama-cpp}/bin/llama-server \
--port ''${PORT} \ --port ''${PORT} \
-m /mnt/ssd/Models/GPT-OSS/gpt-oss-20b-heretic-v2.i1-MXFP4_MOE.gguf \ -m /mnt/ssd/Models/Qwen3.5/Qwen3.5-4B-IQ4_XS.gguf \
-c ''${ctx} \ -c ''${ctx} \
--temp 1.0 \ --temp 0.6 \
--top-p 1.0 \ --top-p 0.95 \
--top-k 40 \ --top-k 20 \
--min-p 0.0 \
-fit off \
-dev CUDA0 -dev CUDA0
''; '';
metadata = { metadata = {
type = [ "text-generation" ]; type = [
"text-generation"
];
}; };
}; };
@@ -549,7 +577,8 @@ in
# --- GTX 1080 Ti Models --- # --- GTX 1080 Ti Models ---
qv = "qwen3-8b-vision"; qv = "qwen3-8b-vision";
q4 = "qwen3-4b-2507-instruct"; q4 = "qwen3.5-4b-thinking";
q9 = "qwen3.5-9b-thinking";
}; };
evict_costs = { evict_costs = {
@@ -559,7 +588,7 @@ in
}; };
sets = { sets = {
concurrent = "(go | q36a | q36b | vlt | vtt | vlv | zi | qie | qi | cr) & (qv | q4)"; concurrent = "(go | q36a | q36b | vlt | vtt | vlv | zi | qie | qi | cr) & (qv | q4 | q9)";
}; };
}; };
} }