diff --git a/modules/home/programs/terminal/bash/config/hey-intern.sh b/modules/home/programs/terminal/bash/config/hey-intern.sh index c4b7c6f..34c0508 100644 --- a/modules/home/programs/terminal/bash/config/hey-intern.sh +++ b/modules/home/programs/terminal/bash/config/hey-intern.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -MODEL="vllm-qwen3.6-27b-long-text" +MODEL="qwen3.6-27b-vllm-180k-cuda0" SYSTEM_PROMPT="You are a shell command expert. Given a natural language query, generate a single shell command that accomplishes the task." # Colors diff --git a/modules/home/programs/terminal/git/config/.gitignore b/modules/home/programs/terminal/git/config/.gitignore index 223a9d0..beb95cc 100755 --- a/modules/home/programs/terminal/git/config/.gitignore +++ b/modules/home/programs/terminal/git/config/.gitignore @@ -2,3 +2,4 @@ _scratch .direnv .envrc .agents +.pi diff --git a/modules/home/programs/terminal/nvim/config/lua/llm-config.lua b/modules/home/programs/terminal/nvim/config/lua/llm-config.lua index 6fd4a01..e35bf34 100755 --- a/modules/home/programs/terminal/nvim/config/lua/llm-config.lua +++ b/modules/home/programs/terminal/nvim/config/lua/llm-config.lua @@ -1,6 +1,6 @@ local llm_endpoint = "https://llm-api.va.reichard.io" -local llm_assistant_model = "vllm-qwen3.6-27b-tools-text " -local llm_infill_model = "qwen3.5-4b-thinking" +local llm_assistant_model = "qwen3.6-27b-vllm-75k-cuda0" +local llm_infill_model = "qwen3.5-4b-cuda1" local current_fim = "llama" -- Copilot Configuration diff --git a/modules/nixos/services/llama-swap/AGENTS.md b/modules/nixos/services/llama-swap/AGENTS.md index 6ef5f8a..161eebc 100644 --- a/modules/nixos/services/llama-swap/AGENTS.md +++ b/modules/nixos/services/llama-swap/AGENTS.md @@ -1,8 +1,12 @@ # llama-swap Module — Agent Guide +## Model ID Convention + +Use `-[-backend/variant][-context][-vl]-`. Omit `thinking` from IDs, use `vl` for vision-language models, and keep placement as the final suffix (`cuda0`, `cuda1`, or `dual`). Keep quantization and richer behavior details in the display `name` unless they are needed to distinguish two active configs for the same family/placement. + ## Syncing vLLM Configs from club-3090 -The three vLLM model configs in `config.nix` (`vllm-qwen3.6-27b-long-text`, `vllm-qwen3.6-27b-long-vision`, `vllm-qwen3.6-27b-tools-text`) are derived from the club-3090 repo's Docker Compose files. Each config block has a `Synced from:` comment with the commit hash it was last aligned to. +The three vLLM model configs in `config.nix` (`qwen3.6-27b-vllm-180k-cuda0`, `qwen3.6-27b-vllm-145k-vl-cuda0`, `qwen3.6-27b-vllm-75k-cuda0`) are derived from the club-3090 repo's Docker Compose files. Each config block has a `Synced from:` comment with the commit hash it was last aligned to. ### Source Files @@ -10,9 +14,9 @@ The upstream compose files live at https://github.com/noonghunna/club-3090 under | config.nix model ID | Compose file | |------------------------------------|-------------------------------------| -| `vllm-qwen3.6-27b-long-text` | `docker-compose.long-text.yml` | -| `vllm-qwen3.6-27b-long-vision` | `docker-compose.long-vision.yml` | -| `vllm-qwen3.6-27b-tools-text` | `docker-compose.tools-text.yml` | +| `qwen3.6-27b-vllm-180k-cuda0` | `docker-compose.long-text.yml` | +| `qwen3.6-27b-vllm-145k-vl-cuda0` | `docker-compose.long-vision.yml` | +| `qwen3.6-27b-vllm-75k-cuda0` | `docker-compose.tools-text.yml` | ### Sync Process diff --git a/modules/nixos/services/llama-swap/config.nix b/modules/nixos/services/llama-swap/config.nix index 8319755..a90d393 100644 --- a/modules/nixos/services/llama-swap/config.nix +++ b/modules/nixos/services/llama-swap/config.nix @@ -14,8 +14,8 @@ in # --------------------------------------- # https://huggingface.co/mradermacher/gpt-oss-20b-heretic-v2-i1-GGUF/tree/main - "gpt-oss-20b-thinking" = { - name = "GPT OSS (20B) - Thinking"; + "gpt-oss-20b-cuda0" = { + name = "GPT OSS 20B (CUDA0)"; macros.ctx = "131072"; cmd = '' ${llama-cpp}/bin/llama-server \ @@ -33,19 +33,24 @@ in }; # https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF/tree/main - "qwen3.6-35b-thinking" = { - name = "Qwen3.6 (35B) - Thinking"; - macros.ctx = "262144"; + "qwen3.6-35b-cuda0" = { + name = "Qwen3.6 35B (CUDA0, UD-Q4)"; + macros.ctx = "100000"; cmd = '' ${llama-cpp}/bin/llama-server \ --port ''${PORT} \ - -m /mnt/ssd/Models/Qwen3.6/Qwen3.6-35B-A3B-UD-IQ4_XS.gguf \ + -m /mnt/ssd/Models/Qwen3.6/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf \ -c ''${ctx} \ + --parallel 1 \ --temp 0.6 \ --top-p 0.95 \ --top-k 20 \ --min-p 0.0 \ - --presence-penalty 1.5 \ + --presence-penalty 0.0 \ + -ctk q8_0 \ + -ctv q8_0 \ + --spec-type draft-mtp \ + --spec-draft-n-max 3 \ -dev CUDA0 \ -fit off \ --chat-template-kwargs "{\"preserve_thinking\": true}" @@ -59,9 +64,9 @@ in }; # https://huggingface.co/unsloth/Qwen3.6-27B-GGUF-MTP/tree/main - "qwen3.6-27b-udq4-thinking" = { - name = "Qwen3.6 (27B) - Thinking (UD-Q4)"; - macros.ctx = "140000"; + "qwen3.6-27b-cuda0" = { + name = "Qwen3.6 27B (CUDA0, UD-Q4)"; + macros.ctx = "150000"; cmd = '' ${llama-cpp}/bin/llama-server \ --port ''${PORT} \ @@ -89,39 +94,9 @@ in }; }; - # https://huggingface.co/ubergarm/Qwen3.6-27B-GGUF/tree/main - "ik-qwen3.6-27b-iq4ks-thinking" = { - name = "Qwen3.6 (27B) - Thinking (ik IQ4_KS)"; - macros.ctx = "131072"; - env = [ "CUDA_VISIBLE_DEVICES=0" ]; - cmd = '' - ${ik-llama-cpp}/bin/llama-server \ - --port ''${PORT} \ - --model /mnt/ssd/Models/Qwen3.6/Qwen3.6-27B-MTP-IQ4_KS.gguf \ - -c ''${ctx} \ - -ctk f16 -ctv q8_0 \ - -mtp --draft-max 4 --draft-p-min 0.70 \ - --merge-qkv \ - -muge \ - -ngl 99 \ - --threads 1 \ - --parallel 1 \ - --jinja \ - --no-mmap \ - --ctx-checkpoints 32 \ - -cram 32768 - ''; - metadata = { - type = [ - "text-generation" - "coding" - ]; - }; - }; - # https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF/tree/main - "gemma-4-26b-vision" = { - name = "Gemma 4 (26B) - Vision"; + "gemma-4-26b-vl-cuda0" = { + name = "Gemma 4 26B (VL, CUDA0)"; macros.ctx = "196608"; cmd = '' ${llama-cpp}/bin/llama-server \ @@ -151,9 +126,8 @@ in }; # https://huggingface.co/Lorbus/Qwen3.6-27B-int4-AutoRound - # Vanilla single-GPU vLLM config pinned to CUDA0 with 50K context. - "qwen3.6-27b-vllm-50k" = { - name = "Qwen 3.6 27B INT4 AutoRound (vLLM - Single GPU - 50K ctx)"; + "qwen3.6-27b-vllm-50k-cuda0" = { + name = "Qwen3.6 27B (vLLM, 50K, CUDA0)"; checkEndpoint = "/v1/models"; macros.ctx = "50000"; proxy = "http://127.0.0.1:\${PORT}"; @@ -196,11 +170,240 @@ in }; # https://github.com/noonghunna/club-3090/tree/master/models/qwen3.6-27b/vllm - # Upstream: club-3090 83bf73d (2026-05-10) - single/long-text.yml - # Long-text variant - 180K context, text-only (no vision) - # TurboQuant 3-bit KV + MTP n=3 + Genesis v7.69 + Cliff 2 closure recipe - "vllm-qwen3.6-27b-long-text" = { - name = "vLLM Qwen3.6 (27B) - Long Text"; + "qwen3.6-27b-vllm-75k-cuda0" = { + name = "Qwen3.6 27B (vLLM, 75K, CUDA0)"; + checkEndpoint = "/v1/models"; + macros.ctx = "75000"; + proxy = "http://127.0.0.1:\${PORT}"; + cmd = + let + vllmCmd = '' + set -e; pip install xxhash pandas scipy -q; + python3 -m vllm._genesis.patches.apply_all; + python3 /patches/patch_timings_1acd67a.py; + exec vllm serve ''${VLLM_ENFORCE_EAGER:+--enforce-eager} + --served-model-name ''${MODEL_ID} + --model /root/.cache/huggingface/qwen3.6-27b-autoround-int4 + --quantization auto_round + --dtype float16 + --tensor-parallel-size 1 + --max-model-len ''${ctx} + --gpu-memory-utilization 0.97 + --max-num-seqs 1 + --max-num-batched-tokens 2048 + --kv-cache-dtype fp8_e5m2 + --language-model-only + --trust-remote-code + --reasoning-parser qwen3 + --enable-auto-tool-choice + --tool-call-parser qwen3_coder + --chat-template /templates/chat_template.jinja + --enable-prefix-caching + --enable-chunked-prefill + --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3}' + --host 0.0.0.0 + --port 8000 + ''; + vllmCmdFlat = builtins.replaceStrings [ "\n" ] [ " " ] vllmCmd; + in + '' + ${pkgs.docker}/bin/docker run --rm --device=nvidia.com/gpu=all \ + --name ''${MODEL_ID} \ + --ipc=host \ + -e CUDA_DEVICE_MAX_CONNECTIONS=8 \ + -e CUDA_DEVICE_ORDER=PCI_BUS_ID \ + -e CUDA_VISIBLE_DEVICES=0 \ + -e GENESIS_ENABLE_P58_ASYNC_PLACEHOLDER_FIX=1 \ + -e GENESIS_ENABLE_P64_QWEN3CODER_MTP_STREAMING=1 \ + -e GENESIS_ENABLE_P66_CUDAGRAPH_SIZE_FILTER=1 \ + -e GENESIS_ENABLE_P68_AUTO_FORCE_TOOL=1 \ + -e GENESIS_ENABLE_P69_LONG_CTX_TOOL_REMINDER=1 \ + -e GENESIS_ENABLE_P72_PROFILE_RUN_CAP=1 \ + -e GENESIS_ENABLE_P74_CHUNK_CLAMP=1 \ + -e GENESIS_ENABLE_P94=1 \ + -e GENESIS_ENABLE_PN13_CUDA_GRAPH_LAMBDA_ARITY=1 \ + -e GENESIS_ENABLE_PN14_TQ_DECODE_OOB_CLAMP=1 \ + -e GENESIS_ENABLE_PN17_FA2_LSE_CLAMP=1 \ + -e GENESIS_ENABLE_PN19_SCOPED_MAX_SPLIT=1 \ + -e GENESIS_ENABLE_PN59_STREAMING_GDN=1 \ + -e GENESIS_ENABLE_PN8_MTP_DRAFT_ONLINE_QUANT=1 \ + -e GENESIS_P68_P69_LONG_CTX_THRESHOLD_CHARS=50000 \ + -e GENESIS_PROFILE_RUN_CAP_M=4128 \ + -e NCCL_CUMEM_ENABLE=0 \ + -e NCCL_P2P_DISABLE=1 \ + -e OMP_NUM_THREADS=1 \ + -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:512 \ + -e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \ + -e VLLM_FLOAT32_MATMUL_PRECISION=high \ + -e VLLM_MARLIN_USE_ATOMIC_ADD=1 \ + -e VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 \ + -e VLLM_NO_USAGE_STATS=1 \ + -e VLLM_USE_FLASHINFER_SAMPLER=1 \ + -e VLLM_WORKER_MULTIPROC_METHOD=spawn \ + -e VLLM_ENFORCE_EAGER \ + -v /mnt/ssd/vLLM/Models:/root/.cache/huggingface \ + -v /mnt/ssd/vLLM/Patches/genesis/vllm/_genesis:/usr/local/lib/python3.12/dist-packages/vllm/_genesis:ro \ + -v /mnt/ssd/vLLM/Patches/patch_timings_1acd67a.py:/patches/patch_timings_1acd67a.py:ro \ + -v /mnt/ssd/vLLM/Templates/chat_template-v11.jinja:/templates/chat_template.jinja \ + -p ''${PORT}:8000 \ + --entrypoint /bin/bash \ + vllm/vllm-openai:nightly-1acd67a795ebccdf9b9db7697ae9082058301657 \ + -c "${vllmCmdFlat}" + ''; + + # Cache Bug - On resume from cache, VRAM usage is higher than just generating in real time. + + # -e TRITON_CACHE_DIR=/root/.triton/cache \ + # -v /mnt/ssd/vLLM/Cache/torch_compile:/root/.cache/vllm/torch_compile_cache \ + # -v /mnt/ssd/vLLM/Cache/triton:/root/.triton/cache \ + + cmdStop = "${pkgs.docker}/bin/docker stop \${MODEL_ID}"; + + metadata = { + type = [ + "text-generation" + "coding" + ]; + }; + }; + + # https://github.com/noonghunna/club-3090/tree/master/models/qwen3.6-27b/vllm + "qwen3.6-27b-vllm-145k-vl-cuda0" = { + name = "Qwen3.6 27B (vLLM, 145K, VL, CUDA0)"; + checkEndpoint = "/v1/models"; + macros.ctx = "145000"; + proxy = "http://127.0.0.1:\${PORT}"; + cmd = + let + vllmCmd = '' + set -e; pip install xxhash pandas scipy -q; + python3 -m vllm._genesis.patches.apply_all; + python3 /patches/patch_timings_1acd67a.py; + exec vllm serve ''${VLLM_ENFORCE_EAGER:+--enforce-eager} + --served-model-name ''${MODEL_ID} + --model /root/.cache/huggingface/qwen3.6-27b-autoround-int4 + --quantization auto_round + --dtype float16 + --tensor-parallel-size 1 + --max-model-len ''${ctx} + --gpu-memory-utilization 0.95 + --max-num-seqs 1 + --max-num-batched-tokens 4128 + --kv-cache-dtype turboquant_3bit_nc + --trust-remote-code + --reasoning-parser qwen3 + --enable-auto-tool-choice + --tool-call-parser qwen3_coder + --enable-prefix-caching + --enable-chunked-prefill + --no-scheduler-reserve-full-isl + --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3}' + --host 0.0.0.0 + --port 8000 + ''; + vllmCmdFlat = builtins.replaceStrings [ "\n" ] [ " " ] vllmCmd; + in + '' + ${pkgs.docker}/bin/docker run --rm --device=nvidia.com/gpu=all \ + --name ''${MODEL_ID} \ + --ipc=host \ + -e CUDA_DEVICE_MAX_CONNECTIONS=8 \ + -e CUDA_DEVICE_ORDER=PCI_BUS_ID \ + -e CUDA_VISIBLE_DEVICES=0 \ + -e GENESIS_BUFFER_MODE=shared \ + -e GENESIS_ENABLE_P100=1 \ + -e GENESIS_ENABLE_P101=1 \ + -e GENESIS_ENABLE_P103=1 \ + -e GENESIS_ENABLE_P15B_FA_VARLEN_CLAMP=1 \ + -e GENESIS_ENABLE_P38B_COMPILE_SAFE=1 \ + -e GENESIS_ENABLE_P4=1 \ + -e GENESIS_ENABLE_P58_ASYNC_PLACEHOLDER_FIX=1 \ + -e GENESIS_ENABLE_P60B_TRITON_KERNEL=1 \ + -e GENESIS_ENABLE_P60_GDN_NGRAM_FIX=1 \ + -e GENESIS_ENABLE_P61B_STREAMING_OVERLAP=1 \ + -e GENESIS_ENABLE_P61_QWEN3_MULTI_TOOL=1 \ + -e GENESIS_ENABLE_P62_STRUCT_OUT_SPEC_TIMING=1 \ + -e GENESIS_ENABLE_P64_QWEN3CODER_MTP_STREAMING=1 \ + -e GENESIS_ENABLE_P66_CUDAGRAPH_SIZE_FILTER=1 \ + -e GENESIS_ENABLE_P67_TQ_MULTI_QUERY_KERNEL=1 \ + -e GENESIS_ENABLE_P68_AUTO_FORCE_TOOL=1 \ + -e GENESIS_ENABLE_P69_LONG_CTX_TOOL_REMINDER=1 \ + -e GENESIS_ENABLE_P72_PROFILE_RUN_CAP=1 \ + -e GENESIS_ENABLE_P74_CHUNK_CLAMP=1 \ + -e GENESIS_ENABLE_P78_TOLIST_CAPTURE_GUARD=0 \ + -e GENESIS_ENABLE_P81_FP8_BLOCK_SCALED_M_LE_8=0 \ + -e GENESIS_ENABLE_P82=0 \ + -e GENESIS_ENABLE_P83=1 \ + -e GENESIS_ENABLE_P87=1 \ + -e GENESIS_ENABLE_P91=1 \ + -e GENESIS_ENABLE_P94=1 \ + -e GENESIS_ENABLE_P98=1 \ + -e GENESIS_ENABLE_P99=1 \ + -e GENESIS_ENABLE_PN11_GDN_AB_CONTIGUOUS=1 \ + -e GENESIS_ENABLE_PN12_FFN_INTERMEDIATE_POOL=1 \ + -e GENESIS_ENABLE_PN13_CUDA_GRAPH_LAMBDA_ARITY=1 \ + -e GENESIS_ENABLE_PN14_TQ_DECODE_OOB_CLAMP=1 \ + -e GENESIS_ENABLE_PN17_FA2_LSE_CLAMP=1 \ + -e GENESIS_ENABLE_PN19_SCOPED_MAX_SPLIT=1 \ + -e GENESIS_ENABLE_PN22_LOCAL_ARGMAX_TP=1 \ + -e GENESIS_ENABLE_PN25_SILU_INDUCTOR_SAFE=1 \ + -e GENESIS_ENABLE_PN26_SPARSE_V=1 \ + -e GENESIS_ENABLE_PN30_DS_LAYOUT_SPEC_DECODE=1 \ + -e GENESIS_ENABLE_PN34_WORKSPACE_LOCK_RELAX=1 \ + -e GENESIS_ENABLE_PN59_STREAMING_GDN=1 \ + -e GENESIS_ENABLE_PN8_MTP_DRAFT_ONLINE_QUANT=1 \ + -e GENESIS_ENABLE_PN9_INDEPENDENT_DRAFTER_ATTN=1 \ + -e GENESIS_P68_P69_LONG_CTX_THRESHOLD_CHARS=50000 \ + -e GENESIS_P82_THRESHOLD_SINGLE=0.3 \ + -e GENESIS_PN26_SPARSE_V_BLOCK_KV=8 \ + -e GENESIS_PN26_SPARSE_V_NUM_WARPS=4 \ + -e GENESIS_PN26_SPARSE_V_THRESHOLD=0.01 \ + -e GENESIS_PREALLOC_TOKEN_BUDGET=4128 \ + -e GENESIS_PROFILE_RUN_CAP_M=4128 \ + -e NCCL_CUMEM_ENABLE=0 \ + -e NCCL_P2P_DISABLE=1 \ + -e OMP_NUM_THREADS=1 \ + -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:512 \ + -e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \ + -e VLLM_FLOAT32_MATMUL_PRECISION=high \ + -e VLLM_MARLIN_USE_ATOMIC_ADD=1 \ + -e VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 \ + -e VLLM_NO_USAGE_STATS=1 \ + -e VLLM_SSM_CONV_STATE_LAYOUT=DS \ + -e VLLM_USE_FLASHINFER_SAMPLER=1 \ + -e VLLM_USE_FUSED_MOE_GROUPED_TOPK=1 \ + -e VLLM_WORKER_MULTIPROC_METHOD=spawn \ + -e VLLM_ENFORCE_EAGER \ + -v /mnt/ssd/vLLM/Models:/root/.cache/huggingface \ + -v /mnt/ssd/vLLM/Patches/genesis/vllm/_genesis:/usr/local/lib/python3.12/dist-packages/vllm/_genesis:ro \ + -v /mnt/ssd/vLLM/Patches/patch_timings_1acd67a.py:/patches/patch_timings_1acd67a.py:ro \ + -p ''${PORT}:8000 \ + --entrypoint /bin/bash \ + vllm/vllm-openai:nightly-1acd67a795ebccdf9b9db7697ae9082058301657 \ + -c "${vllmCmdFlat}" + ''; + + # Cache Bug - On resume from cache, VRAM usage is higher than just generating in real time. + + # -e TRITON_CACHE_DIR=/root/.triton/cache \ + # -v /mnt/ssd/vLLM/Cache/torch_compile:/root/.cache/vllm/torch_compile_cache \ + # -v /mnt/ssd/vLLM/Cache/triton:/root/.triton/cache \ + + cmdStop = "${pkgs.docker}/bin/docker stop \${MODEL_ID}"; + + metadata = { + type = [ + "text-generation" + "coding" + "vision" + ]; + }; + }; + + # https://github.com/noonghunna/club-3090/tree/master/models/qwen3.6-27b/vllm + "qwen3.6-27b-vllm-180k-cuda0" = { + name = "Qwen3.6 27B (vLLM, 180K, CUDA0)"; + checkEndpoint = "/v1/models"; macros.ctx = "180000"; proxy = "http://127.0.0.1:\${PORT}"; cmd = @@ -336,276 +539,13 @@ in }; }; - # https://github.com/noonghunna/club-3090/tree/master/models/qwen3.6-27b/vllm - # Upstream: club-3090 83bf73d (2026-05-10) - single/long-vision.yml - # Long-vision variant - 145K context with vision tower active - # TurboQuant 3-bit KV + MTP n=3 + Genesis v7.69 + Cliff 2 env vars (mem-util kept at 0.95) - "vllm-qwen3.6-27b-long-vision" = { - name = "vLLM Qwen3.6 (27B) - Long Vision"; - macros.ctx = "145000"; - proxy = "http://127.0.0.1:\${PORT}"; - cmd = - let - vllmCmd = '' - set -e; pip install xxhash pandas scipy -q; - python3 -m vllm._genesis.patches.apply_all; - python3 /patches/patch_timings_1acd67a.py; - exec vllm serve ''${VLLM_ENFORCE_EAGER:+--enforce-eager} - --served-model-name ''${MODEL_ID} - --model /root/.cache/huggingface/qwen3.6-27b-autoround-int4 - --quantization auto_round - --dtype float16 - --tensor-parallel-size 1 - --max-model-len ''${ctx} - --gpu-memory-utilization 0.95 - --max-num-seqs 1 - --max-num-batched-tokens 4128 - --kv-cache-dtype turboquant_3bit_nc - --trust-remote-code - --reasoning-parser qwen3 - --enable-auto-tool-choice - --tool-call-parser qwen3_coder - --enable-prefix-caching - --enable-chunked-prefill - --no-scheduler-reserve-full-isl - --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3}' - --host 0.0.0.0 - --port 8000 - ''; - vllmCmdFlat = builtins.replaceStrings [ "\n" ] [ " " ] vllmCmd; - in - '' - ${pkgs.docker}/bin/docker run --rm --device=nvidia.com/gpu=all \ - --name ''${MODEL_ID} \ - --ipc=host \ - -e CUDA_DEVICE_MAX_CONNECTIONS=8 \ - -e CUDA_DEVICE_ORDER=PCI_BUS_ID \ - -e CUDA_VISIBLE_DEVICES=0 \ - -e GENESIS_BUFFER_MODE=shared \ - -e GENESIS_ENABLE_P100=1 \ - -e GENESIS_ENABLE_P101=1 \ - -e GENESIS_ENABLE_P103=1 \ - -e GENESIS_ENABLE_P15B_FA_VARLEN_CLAMP=1 \ - -e GENESIS_ENABLE_P38B_COMPILE_SAFE=1 \ - -e GENESIS_ENABLE_P4=1 \ - -e GENESIS_ENABLE_P58_ASYNC_PLACEHOLDER_FIX=1 \ - -e GENESIS_ENABLE_P60B_TRITON_KERNEL=1 \ - -e GENESIS_ENABLE_P60_GDN_NGRAM_FIX=1 \ - -e GENESIS_ENABLE_P61B_STREAMING_OVERLAP=1 \ - -e GENESIS_ENABLE_P61_QWEN3_MULTI_TOOL=1 \ - -e GENESIS_ENABLE_P62_STRUCT_OUT_SPEC_TIMING=1 \ - -e GENESIS_ENABLE_P64_QWEN3CODER_MTP_STREAMING=1 \ - -e GENESIS_ENABLE_P66_CUDAGRAPH_SIZE_FILTER=1 \ - -e GENESIS_ENABLE_P67_TQ_MULTI_QUERY_KERNEL=1 \ - -e GENESIS_ENABLE_P68_AUTO_FORCE_TOOL=1 \ - -e GENESIS_ENABLE_P69_LONG_CTX_TOOL_REMINDER=1 \ - -e GENESIS_ENABLE_P72_PROFILE_RUN_CAP=1 \ - -e GENESIS_ENABLE_P74_CHUNK_CLAMP=1 \ - -e GENESIS_ENABLE_P78_TOLIST_CAPTURE_GUARD=0 \ - -e GENESIS_ENABLE_P81_FP8_BLOCK_SCALED_M_LE_8=0 \ - -e GENESIS_ENABLE_P82=0 \ - -e GENESIS_ENABLE_P83=1 \ - -e GENESIS_ENABLE_P87=1 \ - -e GENESIS_ENABLE_P91=1 \ - -e GENESIS_ENABLE_P94=1 \ - -e GENESIS_ENABLE_P98=1 \ - -e GENESIS_ENABLE_P99=1 \ - -e GENESIS_ENABLE_PN11_GDN_AB_CONTIGUOUS=1 \ - -e GENESIS_ENABLE_PN12_FFN_INTERMEDIATE_POOL=1 \ - -e GENESIS_ENABLE_PN13_CUDA_GRAPH_LAMBDA_ARITY=1 \ - -e GENESIS_ENABLE_PN14_TQ_DECODE_OOB_CLAMP=1 \ - -e GENESIS_ENABLE_PN17_FA2_LSE_CLAMP=1 \ - -e GENESIS_ENABLE_PN19_SCOPED_MAX_SPLIT=1 \ - -e GENESIS_ENABLE_PN22_LOCAL_ARGMAX_TP=1 \ - -e GENESIS_ENABLE_PN25_SILU_INDUCTOR_SAFE=1 \ - -e GENESIS_ENABLE_PN26_SPARSE_V=1 \ - -e GENESIS_ENABLE_PN30_DS_LAYOUT_SPEC_DECODE=1 \ - -e GENESIS_ENABLE_PN34_WORKSPACE_LOCK_RELAX=1 \ - -e GENESIS_ENABLE_PN59_STREAMING_GDN=1 \ - -e GENESIS_ENABLE_PN8_MTP_DRAFT_ONLINE_QUANT=1 \ - -e GENESIS_ENABLE_PN9_INDEPENDENT_DRAFTER_ATTN=1 \ - -e GENESIS_P68_P69_LONG_CTX_THRESHOLD_CHARS=50000 \ - -e GENESIS_P82_THRESHOLD_SINGLE=0.3 \ - -e GENESIS_PN26_SPARSE_V_BLOCK_KV=8 \ - -e GENESIS_PN26_SPARSE_V_NUM_WARPS=4 \ - -e GENESIS_PN26_SPARSE_V_THRESHOLD=0.01 \ - -e GENESIS_PREALLOC_TOKEN_BUDGET=4128 \ - -e GENESIS_PROFILE_RUN_CAP_M=4128 \ - -e NCCL_CUMEM_ENABLE=0 \ - -e NCCL_P2P_DISABLE=1 \ - -e OMP_NUM_THREADS=1 \ - -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:512 \ - -e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \ - -e VLLM_FLOAT32_MATMUL_PRECISION=high \ - -e VLLM_MARLIN_USE_ATOMIC_ADD=1 \ - -e VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 \ - -e VLLM_NO_USAGE_STATS=1 \ - -e VLLM_SSM_CONV_STATE_LAYOUT=DS \ - -e VLLM_USE_FLASHINFER_SAMPLER=1 \ - -e VLLM_USE_FUSED_MOE_GROUPED_TOPK=1 \ - -e VLLM_WORKER_MULTIPROC_METHOD=spawn \ - -e VLLM_ENFORCE_EAGER \ - -v /mnt/ssd/vLLM/Models:/root/.cache/huggingface \ - -v /mnt/ssd/vLLM/Patches/genesis/vllm/_genesis:/usr/local/lib/python3.12/dist-packages/vllm/_genesis:ro \ - -v /mnt/ssd/vLLM/Patches/patch_timings_1acd67a.py:/patches/patch_timings_1acd67a.py:ro \ - -p ''${PORT}:8000 \ - --entrypoint /bin/bash \ - vllm/vllm-openai:nightly-1acd67a795ebccdf9b9db7697ae9082058301657 \ - -c "${vllmCmdFlat}" - ''; - - # Cache Bug - On resume from cache, VRAM usage is higher than just generating in real time. - - # -e TRITON_CACHE_DIR=/root/.triton/cache \ - # -v /mnt/ssd/vLLM/Cache/torch_compile:/root/.cache/vllm/torch_compile_cache \ - # -v /mnt/ssd/vLLM/Cache/triton:/root/.triton/cache \ - - cmdStop = "${pkgs.docker}/bin/docker stop \${MODEL_ID}"; - - metadata = { - type = [ - "text-generation" - "coding" - "vision" - ]; - }; - }; - - # https://github.com/noonghunna/club-3090/tree/master/models/qwen3.6-27b/vllm - # Upstream: club-3090 83bf73d (2026-05-10) - single/tools-text.yml - # Tools-text variant - 75K context, text-only (no vision) - # fp8_e5m2 KV + MTP n=3. IDE agents (Cline, Cursor, OpenCode, etc.) - "vllm-qwen3.6-27b-tools-text" = { - name = "vLLM Qwen3.6 (27B) - Tools Text"; - macros.ctx = "75000"; - proxy = "http://127.0.0.1:\${PORT}"; - cmd = - let - vllmCmd = '' - set -e; pip install xxhash pandas scipy -q; - python3 -m vllm._genesis.patches.apply_all; - python3 /patches/patch_timings_1acd67a.py; - exec vllm serve ''${VLLM_ENFORCE_EAGER:+--enforce-eager} - --served-model-name ''${MODEL_ID} - --model /root/.cache/huggingface/qwen3.6-27b-autoround-int4 - --quantization auto_round - --dtype float16 - --tensor-parallel-size 1 - --max-model-len ''${ctx} - --gpu-memory-utilization 0.97 - --max-num-seqs 1 - --max-num-batched-tokens 2048 - --kv-cache-dtype fp8_e5m2 - --language-model-only - --trust-remote-code - --reasoning-parser qwen3 - --enable-auto-tool-choice - --tool-call-parser qwen3_coder - --chat-template /templates/chat_template.jinja - --enable-prefix-caching - --enable-chunked-prefill - --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3}' - --host 0.0.0.0 - --port 8000 - ''; - vllmCmdFlat = builtins.replaceStrings [ "\n" ] [ " " ] vllmCmd; - in - '' - ${pkgs.docker}/bin/docker run --rm --device=nvidia.com/gpu=all \ - --name ''${MODEL_ID} \ - --ipc=host \ - -e CUDA_DEVICE_MAX_CONNECTIONS=8 \ - -e CUDA_DEVICE_ORDER=PCI_BUS_ID \ - -e CUDA_VISIBLE_DEVICES=0 \ - -e GENESIS_ENABLE_P58_ASYNC_PLACEHOLDER_FIX=1 \ - -e GENESIS_ENABLE_P64_QWEN3CODER_MTP_STREAMING=1 \ - -e GENESIS_ENABLE_P66_CUDAGRAPH_SIZE_FILTER=1 \ - -e GENESIS_ENABLE_P68_AUTO_FORCE_TOOL=1 \ - -e GENESIS_ENABLE_P69_LONG_CTX_TOOL_REMINDER=1 \ - -e GENESIS_ENABLE_P72_PROFILE_RUN_CAP=1 \ - -e GENESIS_ENABLE_P74_CHUNK_CLAMP=1 \ - -e GENESIS_ENABLE_P94=1 \ - -e GENESIS_ENABLE_PN13_CUDA_GRAPH_LAMBDA_ARITY=1 \ - -e GENESIS_ENABLE_PN14_TQ_DECODE_OOB_CLAMP=1 \ - -e GENESIS_ENABLE_PN17_FA2_LSE_CLAMP=1 \ - -e GENESIS_ENABLE_PN19_SCOPED_MAX_SPLIT=1 \ - -e GENESIS_ENABLE_PN59_STREAMING_GDN=1 \ - -e GENESIS_ENABLE_PN8_MTP_DRAFT_ONLINE_QUANT=1 \ - -e GENESIS_P68_P69_LONG_CTX_THRESHOLD_CHARS=50000 \ - -e GENESIS_PROFILE_RUN_CAP_M=4128 \ - -e NCCL_CUMEM_ENABLE=0 \ - -e NCCL_P2P_DISABLE=1 \ - -e OMP_NUM_THREADS=1 \ - -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:512 \ - -e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \ - -e VLLM_FLOAT32_MATMUL_PRECISION=high \ - -e VLLM_MARLIN_USE_ATOMIC_ADD=1 \ - -e VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 \ - -e VLLM_NO_USAGE_STATS=1 \ - -e VLLM_USE_FLASHINFER_SAMPLER=1 \ - -e VLLM_WORKER_MULTIPROC_METHOD=spawn \ - -e VLLM_ENFORCE_EAGER \ - -v /mnt/ssd/vLLM/Models:/root/.cache/huggingface \ - -v /mnt/ssd/vLLM/Patches/genesis/vllm/_genesis:/usr/local/lib/python3.12/dist-packages/vllm/_genesis:ro \ - -v /mnt/ssd/vLLM/Patches/patch_timings_1acd67a.py:/patches/patch_timings_1acd67a.py:ro \ - -v /mnt/ssd/vLLM/Templates/chat_template-v11.jinja:/templates/chat_template.jinja \ - -p ''${PORT}:8000 \ - --entrypoint /bin/bash \ - vllm/vllm-openai:nightly-1acd67a795ebccdf9b9db7697ae9082058301657 \ - -c "${vllmCmdFlat}" - ''; - - # Cache Bug - On resume from cache, VRAM usage is higher than just generating in real time. - - # -e TRITON_CACHE_DIR=/root/.triton/cache \ - # -v /mnt/ssd/vLLM/Cache/torch_compile:/root/.cache/vllm/torch_compile_cache \ - # -v /mnt/ssd/vLLM/Cache/triton:/root/.triton/cache \ - - cmdStop = "${pkgs.docker}/bin/docker stop \${MODEL_ID}"; - - metadata = { - type = [ - "text-generation" - "coding" - ]; - }; - }; - # --------------------------------------- # ------------- GTX 1080 Ti ------------- # --------------------------------------- - # https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/tree/main - "qwen3-8b-vision" = { - name = "Qwen3 Vision (8B) - Thinking"; - macros.ctx = "60000"; - env = [ "CUDA_VISIBLE_DEVICES=1" ]; - cmd = '' - ${llama-cpp}/bin/llama-server \ - --port ''${PORT} \ - -m /mnt/ssd/Models/Qwen3/Qwen3-VL-8B-Instruct-UD-Q4_K_XL.gguf \ - --mmproj /mnt/ssd/Models/Qwen3/Qwen3-VL-8B-Instruct-UD-Q4_K_XL_mmproj-F16.gguf \ - -c ''${ctx} \ - --temp 0.7 \ - --min-p 0.0 \ - --top-p 0.8 \ - --top-k 20 \ - -ctk q8_0 \ - -ctv q8_0 \ - -fit off \ - -dev CUDA0 - ''; - metadata = { - type = [ - "text-generation" - "vision" - ]; - }; - }; - # https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/tree/main - "qwen3.5-9b-thinking" = { - name = "Qwen3.5 (9B) - Thinking"; + "qwen3.5-9b-vl-cuda1" = { + name = "Qwen3.5 9B (VL, CUDA1)"; macros.ctx = "131072"; env = [ "CUDA_VISIBLE_DEVICES=1" ]; cmd = '' @@ -631,8 +571,8 @@ in }; # https://huggingface.co/unsloth/Qwen3.5-4B-GGUF/tree/main - "qwen3.5-4b-thinking" = { - name = "Qwen3.5 (4B) - Thinking"; + "qwen3.5-4b-cuda1" = { + name = "Qwen3.5 4B (CUDA1)"; macros.ctx = "131072"; env = [ "CUDA_VISIBLE_DEVICES=1" ]; cmd = '' @@ -650,6 +590,7 @@ in metadata = { type = [ "text-generation" + "coding" ]; }; }; @@ -659,8 +600,8 @@ in # --------------------------------------- # https://huggingface.co/unsloth/Qwen3-Coder-Next-GGUF/tree/main - "qwen3-coder-next-80b-instruct" = { - name = "Qwen3 Coder Next (80B) - Instruct"; + "qwen3-coder-next-80b-dual" = { + name = "Qwen3 Coder Next 80B (Dual GPU)"; macros.ctx = "131072"; cmd = '' ${llama-cpp}/bin/llama-server \ @@ -685,8 +626,8 @@ in }; # https://huggingface.co/unsloth/Qwen3.6-27B-GGUF-MTP/tree/main - "qwen3.6-27b-udq6-thinking" = { - name = "Qwen3.6 (27B) - Thinking (UD-Q6)"; + "qwen3.6-27b-dual" = { + name = "Qwen3.6 27B (Dual GPU, UD-Q6)"; macros.ctx = "196608"; cmd = '' ${llama-cpp}/bin/llama-server \ @@ -704,7 +645,7 @@ in --spec-type draft-mtp \ --spec-draft-n-max 3 \ -dev CUDA0,CUDA1 \ - -ts 75,25 \ + -ts 73,27 \ -fit off \ --chat-template-kwargs "{\"preserve_thinking\": true}" ''; @@ -716,13 +657,46 @@ in }; }; + # https://huggingface.co/unsloth/Qwen3.6-35B-A3B-MTP-GGUF/tree/main + "qwen3.6-35b-dual" = { + name = "Qwen3.6 35B (Dual GPU, UD-Q6)"; + macros.ctx = "262144"; + cmd = '' + ${llama-cpp}/bin/llama-server \ + --port ''${PORT} \ + -m /mnt/ssd/Models/Qwen3.6/Qwen3.6-35B-A3B-UD-Q6_K.gguf \ + -c ''${ctx} \ + --parallel 2 \ + --temp 0.6 \ + --top-p 0.95 \ + --top-k 20 \ + --min-p 0.00 \ + --presence-penalty 0.0 \ + -ctk q8_0 \ + -ctv q8_0 \ + --spec-type draft-mtp \ + --spec-draft-n-max 3 \ + -dev CUDA0,CUDA1 \ + -fit off \ + -ts 7,3 \ + --chat-template-kwargs "{\"preserve_thinking\": true}" + ''; + metadata = { + type = [ + "text-generation" + "coding" + ]; + }; + }; + # --------------------------------------- - # ---------- Stable Diffussion ---------- + # ---------- Stable Diffusion ---------- # --------------------------------------- - "z-image-turbo" = { + "z-image-turbo-cuda0" = { name = "Z-Image-Turbo"; checkEndpoint = "/"; + env = [ "CUDA_VISIBLE_DEVICES=0" ]; cmd = '' ${stable-diffusion-cpp}/bin/sd-server \ --listen-port ''${PORT} \ @@ -739,9 +713,10 @@ in }; }; - "qwen-image-edit-2511" = { + "qwen-image-edit-2511-cuda0" = { name = "Qwen Image Edit 2511"; checkEndpoint = "/"; + env = [ "CUDA_VISIBLE_DEVICES=0" ]; cmd = '' ${stable-diffusion-cpp}/bin/sd-server \ --listen-port ''${PORT} \ @@ -765,9 +740,10 @@ in }; }; - "qwen-image-2512" = { + "qwen-image-2512-cuda0" = { name = "Qwen Image 2512"; checkEndpoint = "/"; + env = [ "CUDA_VISIBLE_DEVICES=0" ]; cmd = '' ${stable-diffusion-cpp}/bin/sd-server \ --listen-port ''${PORT} \ @@ -787,9 +763,10 @@ in }; }; - "chroma-radiance" = { + "chroma-radiance-cuda0" = { name = "Chroma Radiance"; checkEndpoint = "/"; + env = [ "CUDA_VISIBLE_DEVICES=0" ]; cmd = '' ${stable-diffusion-cpp}/bin/sd-server \ --listen-port ''${PORT} \ @@ -809,33 +786,30 @@ in # Concurrent Model Matrix # # CUDA0 models can run alongside CUDA1 models (one each). Models not - # listed in any set (dual-GPU models using -ts) run alone and evict - # everything. + # listed in any set (dual-GPU models) run alone and evict everything. matrix = { vars = { # --- RTX 3090 Models --- - vlt = "vllm-qwen3.6-27b-long-text"; - vtt = "vllm-qwen3.6-27b-tools-text"; - vlv = "vllm-qwen3.6-27b-long-vision"; - v50 = "qwen3.6-27b-vllm-50k"; - go = "gpt-oss-20b-thinking"; - g4 = "gemma-4-26b-vision"; - q36a = "qwen3.6-35b-thinking"; - q36b = "qwen3.6-27b-udq4-thinking"; - iq36 = "ik-qwen3.6-27b-iq4ks-thinking"; - zi = "z-image-turbo"; - qie = "qwen-image-edit-2511"; - qi = "qwen-image-2512"; - cr = "chroma-radiance"; + v180 = "qwen3.6-27b-vllm-180k-cuda0"; + v145 = "qwen3.6-27b-vllm-145k-vl-cuda0"; + v75 = "qwen3.6-27b-vllm-75k-cuda0"; + v50 = "qwen3.6-27b-vllm-50k-cuda0"; + go = "gpt-oss-20b-cuda0"; + g4 = "gemma-4-26b-vl-cuda0"; + q36a = "qwen3.6-35b-cuda0"; + q36b = "qwen3.6-27b-cuda0"; + zi = "z-image-turbo-cuda0"; + qie = "qwen-image-edit-2511-cuda0"; + qi = "qwen-image-2512-cuda0"; + cr = "chroma-radiance-cuda0"; # --- GTX 1080 Ti Models --- - qv = "qwen3-8b-vision"; - q4 = "qwen3.5-4b-thinking"; - q9 = "qwen3.5-9b-thinking"; + q4 = "qwen3.5-4b-cuda1"; + q9 = "qwen3.5-9b-vl-cuda1"; }; sets = { - concurrent = "(go | g4 | q36a | q36b | iq36 | vlt | vtt | vlv | v50 | zi | qie | qi | cr) & (qv | q4 | q9)"; + concurrent = "(go | g4 | q36a | q36b | v180 | v145 | v75 | v50 | zi | qie | qi | cr) & (q4 | q9)"; }; }; }