diff --git a/modules/nixos/services/llama-swap/AGENTS.md b/modules/nixos/services/llama-swap/AGENTS.md new file mode 100644 index 0000000..1ef8ab0 --- /dev/null +++ b/modules/nixos/services/llama-swap/AGENTS.md @@ -0,0 +1,41 @@ +# llama-swap Module — Agent Guide + +## Syncing vLLM Configs from club-3090 + +The three vLLM model configs in `config.nix` (`vllm-qwen3.6-27b-long-text`, `vllm-qwen3.6-27b-long-vision`, `vllm-qwen3.6-27b-tools-text`) are derived from the club-3090 repo's Docker Compose files. Each config block has a `Synced from:` comment with the commit hash it was last aligned to. + +### Source Files + +The upstream compose files live at https://github.com/noonghunna/club-3090 under `models/qwen3.6-27b/vllm/compose/`: + +| config.nix model ID | Compose file | +|------------------------------------|-------------------------------------| +| `vllm-qwen3.6-27b-long-text` | `docker-compose.long-text.yml` | +| `vllm-qwen3.6-27b-long-vision` | `docker-compose.long-vision.yml` | +| `vllm-qwen3.6-27b-tools-text` | `docker-compose.tools-text.yml` | + +### Sync Process + +1. **Fetch the latest compose files** from https://github.com/noonghunna/club-3090 (master branch) and note the HEAD commit hash. +2. **Diff each compose file** against the current config.nix block. The mapping is: + - Compose `command:` args → Nix `vllmCmd` string (the `exec vllm serve ...` block) + - Compose `environment:` → Nix docker `-e` flags + - Compose `volumes:` → Nix docker `-v` flags + - Compose `image:` → Nix docker image tag at the end of the `docker run` command + - Compose `entrypoint:` → Nix `vllmCmd` preamble (the `set -e; pip install ...; python3 ...` lines before `exec vllm serve`) +3. **Apply changes** to `config.nix`. Key things to watch: + - `--max-model-len` and `--gpu-memory-utilization` — these change across versions + - Genesis env vars — the full set grows frequently; add new ones, remove deprecated ones + - Sidecar patches — old patches get absorbed into Genesis; drop them from entrypoint + volume mounts + - Docker image tag — update when the compose files move to a new nightly +4. **Keep `patch_timings_07351e088.py`** — this is our own patch, not from club-3090. Always retain it in the entrypoint and volume mounts. +5. **Update the `Synced from:` comment** on each config block with the new commit hash and date. +6. **Update `setup-qwen36-vllm.sh`** if the upstream `patches/` directory changed (new patches added, old ones removed). The setup script downloads sidecar patches and creates cache directories. +7. **Verify syntax**: `nix-instantiate --parse config.nix` + +### Structural Notes + +- `config.nix` uses Nix string interpolation. Newlines in `vllmCmd` are flattened to spaces via `builtins.replaceStrings` before passing to `docker run -c`. +- We pin `CUDA_VISIBLE_DEVICES=0` and `CUDA_DEVICE_ORDER=PCI_BUS_ID` (not in compose files) because the host has multiple GPUs and llama-swap's concurrency matrix manages GPU assignment. +- Volume mounts use `/mnt/ssd/vLLM/` paths (Models, Patches, Cache) — these match what `setup-qwen36-vllm.sh` creates. +- The `patches/` subdirectory in this module contains our custom timings patch and its source `.patch` file — unrelated to club-3090's `patches/` dir. diff --git a/modules/nixos/services/llama-swap/config.nix b/modules/nixos/services/llama-swap/config.nix index d47fc27..c1fcd16 100644 --- a/modules/nixos/services/llama-swap/config.nix +++ b/modules/nixos/services/llama-swap/config.nix @@ -97,19 +97,19 @@ in }; }; - # https://github.com/noonghunna/club-3090/tree/v0.20-experimental/models/qwen3.6-27b/vllm - # Long-text variant - v0.20 experimental single-3090 profile, text-only (no vision) - # TurboQuant 3-bit KV + MTP n=3 + workspace-lock sidecar. + # https://github.com/noonghunna/club-3090/tree/master/models/qwen3.6-27b/vllm + # Synced from: club-3090 ae4846f (2026-05-02) — docker-compose.long-text.yml + # Long-text variant - 180K context, text-only (no vision) + # TurboQuant 3-bit KV + MTP n=3 + Genesis v7.65 full PROD env set "vllm-qwen3.6-27b-long-text" = { name = "vLLM Qwen3.6 (27B) - Long Text"; - macros.ctx = "214000"; + macros.ctx = "180000"; proxy = "http://127.0.0.1:\${PORT}"; cmd = let vllmCmd = '' set -e; pip install xxhash pandas scipy -q; python3 -m vllm._genesis.patches.apply_all; - python3 /patches/patch_pn12_compile_safe_custom_op.py; python3 /patches/patch_workspace_lock_disable.py; python3 /patches/patch_tolist_cudagraph.py; python3 /patches/patch_timings_07351e088.py; @@ -120,7 +120,7 @@ in --dtype float16 --tensor-parallel-size 1 --max-model-len ''${ctx} - --gpu-memory-utilization 0.985 + --gpu-memory-utilization 0.95 --max-num-seqs 1 --max-num-batched-tokens 4128 --kv-cache-dtype turboquant_3bit_nc @@ -156,20 +156,64 @@ in -e CUDA_DEVICE_ORDER=PCI_BUS_ID \ -e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \ -e VLLM_MARLIN_USE_ATOMIC_ADD=1 \ + -e TRITON_CACHE_DIR=/root/.triton/cache \ + -e GENESIS_ENABLE_P4=1 \ + -e GENESIS_ENABLE_P58_ASYNC_PLACEHOLDER_FIX=1 \ + -e GENESIS_ENABLE_P60_GDN_NGRAM_FIX=1 \ + -e GENESIS_ENABLE_P60B_TRITON_KERNEL=1 \ + -e GENESIS_ENABLE_P61_QWEN3_MULTI_TOOL=1 \ + -e GENESIS_ENABLE_P61B_STREAMING_OVERLAP=1 \ + -e GENESIS_ENABLE_P62_STRUCT_OUT_SPEC_TIMING=1 \ + -e GENESIS_ENABLE_P64_QWEN3CODER_MTP_STREAMING=1 \ -e GENESIS_ENABLE_P65_TURBOQUANT_SPEC_CG_DOWNGRADE=1 \ -e GENESIS_ENABLE_P66_CUDAGRAPH_SIZE_FILTER=1 \ - -e GENESIS_ENABLE_P64_QWEN3CODER_MTP_STREAMING=1 \ + -e GENESIS_ENABLE_P67_TQ_MULTI_QUERY_KERNEL=1 \ + -e GENESIS_ENABLE_P68_AUTO_FORCE_TOOL=1 \ + -e GENESIS_ENABLE_P69_LONG_CTX_TOOL_REMINDER=1 \ + -e GENESIS_P68_P69_LONG_CTX_THRESHOLD_CHARS=50000 \ + -e GENESIS_ENABLE_P72_PROFILE_RUN_CAP=1 \ + -e GENESIS_PROFILE_RUN_CAP_M=4128 \ + -e GENESIS_ENABLE_P74_CHUNK_CLAMP=1 \ + -e GENESIS_ENABLE_P83=1 \ + -e GENESIS_ENABLE_P85=1 \ + -e GENESIS_ENABLE_P87=1 \ + -e GENESIS_ENABLE_P91=1 \ + -e GENESIS_ENABLE_P94=1 \ + -e GENESIS_ENABLE_P98=1 \ + -e GENESIS_ENABLE_P99=1 \ + -e GENESIS_ENABLE_P100=1 \ -e GENESIS_ENABLE_P101=1 \ -e GENESIS_ENABLE_P103=1 \ + -e GENESIS_ENABLE_PN8_MTP_DRAFT_ONLINE_QUANT=1 \ + -e GENESIS_ENABLE_PN9_INDEPENDENT_DRAFTER_ATTN=1 \ + -e GENESIS_ENABLE_PN11_GDN_AB_CONTIGUOUS=1 \ -e GENESIS_ENABLE_PN12_FFN_INTERMEDIATE_POOL=1 \ -e GENESIS_ENABLE_PN13_CUDA_GRAPH_LAMBDA_ARITY=1 \ + -e GENESIS_ENABLE_PN14_TQ_DECODE_OOB_CLAMP=1 \ -e GENESIS_ENABLE_PN17_FA2_LSE_CLAMP=1 \ + -e GENESIS_ENABLE_PN25_SILU_INDUCTOR_SAFE=1 \ + -e GENESIS_ENABLE_PN30_DS_LAYOUT_SPEC_DECODE=1 \ + -e GENESIS_PREALLOC_TOKEN_BUDGET=4128 \ + -e GENESIS_BUFFER_MODE=shared \ + -e GENESIS_ENABLE_P78_TOLIST_CAPTURE_GUARD=0 \ + -e GENESIS_ENABLE_P81_FP8_BLOCK_SCALED_M_LE_8=0 \ + -e GENESIS_ENABLE_P82=0 \ + -e GENESIS_P82_THRESHOLD_SINGLE=0.3 \ -e GENESIS_ENABLE_PN19_SCOPED_MAX_SPLIT=1 \ - -e GENESIS_ENABLE_P98=1 \ + -e GENESIS_ENABLE_PN22_LOCAL_ARGMAX_TP=1 \ + -e GENESIS_ENABLE_PN26_SPARSE_V=1 \ + -e GENESIS_PN26_SPARSE_V_BLOCK_KV=8 \ + -e GENESIS_PN26_SPARSE_V_NUM_WARPS=4 \ + -e GENESIS_PN26_SPARSE_V_THRESHOLD=0.01 \ + -e GENESIS_ENABLE_P38B_COMPILE_SAFE=1 \ + -e GENESIS_ENABLE_P15B_FA_VARLEN_CLAMP=1 \ + -e VLLM_SSM_CONV_STATE_LAYOUT=DS \ + -e VLLM_USE_FUSED_MOE_GROUPED_TOPK=1 \ -v /mnt/ssd/vLLM/Models:/root/.cache/huggingface \ + -v /mnt/ssd/vLLM/Cache/torch_compile:/root/.cache/vllm/torch_compile_cache \ + -v /mnt/ssd/vLLM/Cache/triton:/root/.triton/cache \ -v /mnt/ssd/vLLM/Patches/genesis/vllm/_genesis:/usr/local/lib/python3.12/dist-packages/vllm/_genesis:ro \ -v /mnt/ssd/vLLM/Patches/patch_tolist_cudagraph.py:/patches/patch_tolist_cudagraph.py:ro \ - -v /mnt/ssd/vLLM/Patches/patch_pn12_compile_safe_custom_op.py:/patches/patch_pn12_compile_safe_custom_op.py:ro \ -v /mnt/ssd/vLLM/Patches/patch_workspace_lock_disable.py:/patches/patch_workspace_lock_disable.py:ro \ -v /mnt/ssd/vLLM/Patches/patch_timings_07351e088.py:/patches/patch_timings_07351e088.py:ro \ -p ''${PORT}:8000 \ @@ -188,20 +232,19 @@ in }; # https://github.com/noonghunna/club-3090/tree/master/models/qwen3.6-27b/vllm - # Long-vision variant - 140K context with vision tower active - # TurboQuant 3-bit KV + MTP n=3 + PN12/P104 cliff-closure stack + # Synced from: club-3090 ae4846f (2026-05-02) — docker-compose.long-vision.yml + # Long-vision variant - 145K context with vision tower active + # TurboQuant 3-bit KV + MTP n=3 + Genesis v7.65 full PROD env set "vllm-qwen3.6-27b-long-vision" = { name = "vLLM Qwen3.6 (27B) - Long Vision"; - macros.ctx = "140000"; + macros.ctx = "145000"; proxy = "http://127.0.0.1:\${PORT}"; cmd = let vllmCmd = '' set -e; pip install xxhash pandas scipy -q; python3 -m vllm._genesis.patches.apply_all; - python3 /patches/patch_pn12_ffn_pool_anchor.py; - python3 /patches/patch_pn12_compile_safe_custom_op.py; - python3 /patches/patch_fa_max_seqlen_clamp.py; + python3 /patches/patch_workspace_lock_disable.py; python3 /patches/patch_tolist_cudagraph.py; python3 /patches/patch_timings_07351e088.py; exec vllm serve @@ -235,7 +278,7 @@ in -e VLLM_WORKER_MULTIPROC_METHOD=spawn \ -e NCCL_CUMEM_ENABLE=0 \ -e NCCL_P2P_DISABLE=1 \ - -e VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1 \ + -e VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 \ -e VLLM_NO_USAGE_STATS=1 \ -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:512 \ -e VLLM_FLOAT32_MATMUL_PRECISION=high \ @@ -246,25 +289,69 @@ in -e CUDA_DEVICE_ORDER=PCI_BUS_ID \ -e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \ -e VLLM_MARLIN_USE_ATOMIC_ADD=1 \ + -e TRITON_CACHE_DIR=/root/.triton/cache \ + -e GENESIS_ENABLE_P4=1 \ + -e GENESIS_ENABLE_P58_ASYNC_PLACEHOLDER_FIX=1 \ + -e GENESIS_ENABLE_P60_GDN_NGRAM_FIX=1 \ + -e GENESIS_ENABLE_P60B_TRITON_KERNEL=1 \ + -e GENESIS_ENABLE_P61_QWEN3_MULTI_TOOL=1 \ + -e GENESIS_ENABLE_P61B_STREAMING_OVERLAP=1 \ + -e GENESIS_ENABLE_P62_STRUCT_OUT_SPEC_TIMING=1 \ + -e GENESIS_ENABLE_P64_QWEN3CODER_MTP_STREAMING=1 \ -e GENESIS_ENABLE_P65_TURBOQUANT_SPEC_CG_DOWNGRADE=1 \ -e GENESIS_ENABLE_P66_CUDAGRAPH_SIZE_FILTER=1 \ - -e GENESIS_ENABLE_P64_QWEN3CODER_MTP_STREAMING=1 \ + -e GENESIS_ENABLE_P67_TQ_MULTI_QUERY_KERNEL=1 \ + -e GENESIS_ENABLE_P68_AUTO_FORCE_TOOL=1 \ + -e GENESIS_ENABLE_P69_LONG_CTX_TOOL_REMINDER=1 \ + -e GENESIS_P68_P69_LONG_CTX_THRESHOLD_CHARS=50000 \ + -e GENESIS_ENABLE_P72_PROFILE_RUN_CAP=1 \ + -e GENESIS_PROFILE_RUN_CAP_M=4128 \ + -e GENESIS_ENABLE_P74_CHUNK_CLAMP=1 \ + -e GENESIS_ENABLE_P83=1 \ + -e GENESIS_ENABLE_P85=1 \ + -e GENESIS_ENABLE_P87=1 \ + -e GENESIS_ENABLE_P91=1 \ + -e GENESIS_ENABLE_P94=1 \ + -e GENESIS_ENABLE_P98=1 \ + -e GENESIS_ENABLE_P99=1 \ + -e GENESIS_ENABLE_P100=1 \ -e GENESIS_ENABLE_P101=1 \ -e GENESIS_ENABLE_P103=1 \ + -e GENESIS_ENABLE_PN8_MTP_DRAFT_ONLINE_QUANT=1 \ + -e GENESIS_ENABLE_PN9_INDEPENDENT_DRAFTER_ATTN=1 \ + -e GENESIS_ENABLE_PN11_GDN_AB_CONTIGUOUS=1 \ -e GENESIS_ENABLE_PN12_FFN_INTERMEDIATE_POOL=1 \ -e GENESIS_ENABLE_PN13_CUDA_GRAPH_LAMBDA_ARITY=1 \ - -e GENESIS_ENABLE_FA_MAX_SEQLEN_CLAMP=1 \ + -e GENESIS_ENABLE_PN14_TQ_DECODE_OOB_CLAMP=1 \ -e GENESIS_ENABLE_PN17_FA2_LSE_CLAMP=1 \ + -e GENESIS_ENABLE_PN19_SCOPED_MAX_SPLIT=1 \ + -e GENESIS_ENABLE_PN22_LOCAL_ARGMAX_TP=1 \ + -e GENESIS_ENABLE_PN26_SPARSE_V=1 \ + -e GENESIS_PN26_SPARSE_V_BLOCK_KV=8 \ + -e GENESIS_PN26_SPARSE_V_NUM_WARPS=4 \ + -e GENESIS_PN26_SPARSE_V_THRESHOLD=0.01 \ + -e GENESIS_ENABLE_P38B_COMPILE_SAFE=1 \ + -e GENESIS_ENABLE_P15B_FA_VARLEN_CLAMP=1 \ + -e GENESIS_ENABLE_PN25_SILU_INDUCTOR_SAFE=1 \ + -e GENESIS_ENABLE_PN30_DS_LAYOUT_SPEC_DECODE=1 \ + -e GENESIS_PREALLOC_TOKEN_BUDGET=4128 \ + -e GENESIS_BUFFER_MODE=shared \ + -e GENESIS_ENABLE_P78_TOLIST_CAPTURE_GUARD=0 \ + -e GENESIS_ENABLE_P81_FP8_BLOCK_SCALED_M_LE_8=0 \ + -e GENESIS_ENABLE_P82=0 \ + -e GENESIS_P82_THRESHOLD_SINGLE=0.3 \ + -e VLLM_SSM_CONV_STATE_LAYOUT=DS \ + -e VLLM_USE_FUSED_MOE_GROUPED_TOPK=1 \ -v /mnt/ssd/vLLM/Models:/root/.cache/huggingface \ + -v /mnt/ssd/vLLM/Cache/torch_compile:/root/.cache/vllm/torch_compile_cache \ + -v /mnt/ssd/vLLM/Cache/triton:/root/.triton/cache \ -v /mnt/ssd/vLLM/Patches/genesis/vllm/_genesis:/usr/local/lib/python3.12/dist-packages/vllm/_genesis:ro \ -v /mnt/ssd/vLLM/Patches/patch_tolist_cudagraph.py:/patches/patch_tolist_cudagraph.py:ro \ - -v /mnt/ssd/vLLM/Patches/patch_pn12_ffn_pool_anchor.py:/patches/patch_pn12_ffn_pool_anchor.py:ro \ - -v /mnt/ssd/vLLM/Patches/patch_pn12_compile_safe_custom_op.py:/patches/patch_pn12_compile_safe_custom_op.py:ro \ - -v /mnt/ssd/vLLM/Patches/patch_fa_max_seqlen_clamp.py:/patches/patch_fa_max_seqlen_clamp.py:ro \ + -v /mnt/ssd/vLLM/Patches/patch_workspace_lock_disable.py:/patches/patch_workspace_lock_disable.py:ro \ -v /mnt/ssd/vLLM/Patches/patch_timings_07351e088.py:/patches/patch_timings_07351e088.py:ro \ -p ''${PORT}:8000 \ --entrypoint /bin/bash \ - vllm/vllm-openai:nightly-07351e0883470724dd5a7e9730ed10e01fc99d08 \ + vllm/vllm-openai:nightly-7a1eb8ac2ec4ea69338c51dc7afd4b15010abfa8 \ -c "${vllmCmdFlat}" ''; cmdStop = "${pkgs.docker}/bin/docker stop \${MODEL_ID}"; @@ -279,9 +366,9 @@ in }; # https://github.com/noonghunna/club-3090/tree/master/models/qwen3.6-27b/vllm + # Synced from: club-3090 ae4846f (2026-05-02) — docker-compose.tools-text.yml # Tools-text variant - 75K context, text-only (no vision) - # fp8_e5m2 KV + MTP n=3. This is the repo's validated long-context - # tool-calling profile and should be more stable than TurboQuant 128K. + # fp8_e5m2 KV + MTP n=3. IDE agents (Cline, Cursor, OpenCode, etc.) "vllm-qwen3.6-27b-tools-text" = { name = "vLLM Qwen3.6 (27B) - Tools Text"; macros.ctx = "75000"; @@ -324,7 +411,7 @@ in -e VLLM_WORKER_MULTIPROC_METHOD=spawn \ -e NCCL_CUMEM_ENABLE=0 \ -e NCCL_P2P_DISABLE=1 \ - -e VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1 \ + -e VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 \ -e VLLM_NO_USAGE_STATS=1 \ -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:512 \ -e VLLM_FLOAT32_MATMUL_PRECISION=high \ @@ -335,15 +422,31 @@ in -e CUDA_DEVICE_ORDER=PCI_BUS_ID \ -e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \ -e VLLM_MARLIN_USE_ATOMIC_ADD=1 \ + -e TRITON_CACHE_DIR=/root/.triton/cache \ + -e GENESIS_ENABLE_P58_ASYNC_PLACEHOLDER_FIX=1 \ -e GENESIS_ENABLE_P64_QWEN3CODER_MTP_STREAMING=1 \ + -e GENESIS_ENABLE_P66_CUDAGRAPH_SIZE_FILTER=1 \ + -e GENESIS_ENABLE_P68_AUTO_FORCE_TOOL=1 \ + -e GENESIS_ENABLE_P69_LONG_CTX_TOOL_REMINDER=1 \ + -e GENESIS_P68_P69_LONG_CTX_THRESHOLD_CHARS=50000 \ + -e GENESIS_ENABLE_P72_PROFILE_RUN_CAP=1 \ + -e GENESIS_PROFILE_RUN_CAP_M=4128 \ + -e GENESIS_ENABLE_P74_CHUNK_CLAMP=1 \ + -e GENESIS_ENABLE_P94=1 \ -e GENESIS_ENABLE_PN8_MTP_DRAFT_ONLINE_QUANT=1 \ + -e GENESIS_ENABLE_PN13_CUDA_GRAPH_LAMBDA_ARITY=1 \ + -e GENESIS_ENABLE_PN14_TQ_DECODE_OOB_CLAMP=1 \ + -e GENESIS_ENABLE_PN17_FA2_LSE_CLAMP=1 \ + -e GENESIS_ENABLE_PN19_SCOPED_MAX_SPLIT=1 \ -v /mnt/ssd/vLLM/Models:/root/.cache/huggingface \ + -v /mnt/ssd/vLLM/Cache/torch_compile:/root/.cache/vllm/torch_compile_cache \ + -v /mnt/ssd/vLLM/Cache/triton:/root/.triton/cache \ -v /mnt/ssd/vLLM/Patches/genesis/vllm/_genesis:/usr/local/lib/python3.12/dist-packages/vllm/_genesis:ro \ -v /mnt/ssd/vLLM/Patches/patch_tolist_cudagraph.py:/patches/patch_tolist_cudagraph.py:ro \ -v /mnt/ssd/vLLM/Patches/patch_timings_07351e088.py:/patches/patch_timings_07351e088.py:ro \ -p ''${PORT}:8000 \ --entrypoint /bin/bash \ - vllm/vllm-openai:nightly-07351e0883470724dd5a7e9730ed10e01fc99d08 \ + vllm/vllm-openai:nightly-7a1eb8ac2ec4ea69338c51dc7afd4b15010abfa8 \ -c "${vllmCmdFlat}" ''; cmdStop = "${pkgs.docker}/bin/docker stop \${MODEL_ID}"; diff --git a/modules/nixos/services/llama-swap/default.nix b/modules/nixos/services/llama-swap/default.nix index 91581f7..9ebf267 100644 --- a/modules/nixos/services/llama-swap/default.nix +++ b/modules/nixos/services/llama-swap/default.nix @@ -88,6 +88,11 @@ in # Create Config sops = { + secrets = { + "llama_swap_api_keys/evan" = { + sopsFile = lib.snowfall.fs.get-file "secrets/common/llama-swap.yaml"; + }; + }; secrets = { "llama_swap_api_keys/pi" = { sopsFile = lib.snowfall.fs.get-file "secrets/common/llama-swap.yaml"; @@ -101,6 +106,7 @@ in recursiveUpdate cfg.config { apiKeys = [ config.sops.placeholder."llama_swap_api_keys/pi" + config.sops.placeholder."llama_swap_api_keys/evan" ]; } ); diff --git a/modules/nixos/services/llama-swap/setup-qwen36-vllm.sh b/modules/nixos/services/llama-swap/setup-qwen36-vllm.sh index dcfb9fd..714c2a7 100755 --- a/modules/nixos/services/llama-swap/setup-qwen36-vllm.sh +++ b/modules/nixos/services/llama-swap/setup-qwen36-vllm.sh @@ -1,33 +1,38 @@ #!/usr/bin/env bash # Setup script for vLLM Qwen3.6-27B on a single 3090. # -# Downloads the model, clones Genesis patches, and fetches all sidecar +# Downloads the model, clones Genesis patches (pinned), applies setup-time +# source patches to the Genesis tree, and fetches all boot-time sidecar # patches into place under /mnt/ssd/vLLM/. # # Idempotent — safe to re-run; skips steps already completed. # -# Prerequisites: git (with git-lfs), docker +# Prerequisites: git (with git-lfs), docker, python3 set -euo pipefail MODEL_DIR="/mnt/ssd/vLLM/Models" MODEL_SUBDIR="qwen3.6-27b-autoround-int4" PATCHES_DIR="/mnt/ssd/vLLM/Patches" +CACHE_DIR="/mnt/ssd/vLLM/Cache" GENESIS_DIR="${PATCHES_DIR}/genesis" + +# Pin Genesis to the validated commit (bump requires re-testing all composes) +GENESIS_PIN="${GENESIS_PIN:-fc89395}" + TOLIST_PATCH="${PATCHES_DIR}/patch_tolist_cudagraph.py" -PN12_FFN_PATCH="${PATCHES_DIR}/patch_pn12_ffn_pool_anchor.py" -PN12_COMPILE_PATCH="${PATCHES_DIR}/patch_pn12_compile_safe_custom_op.py" -FA_CLAMP_PATCH="${PATCHES_DIR}/patch_fa_max_seqlen_clamp.py" WORKSPACE_LOCK_PATCH="${PATCHES_DIR}/patch_workspace_lock_disable.py" +PN25_REGISTER_PATCH="${PATCHES_DIR}/patch_pn25_genesis_register_fix.py" +PN30_DST_PATCH="${PATCHES_DIR}/patch_pn30_dst_shaped_temp_fix.py" +PR40798_PATCH="${PATCHES_DIR}/patch_pr40798_workspace.py" TIMINGS_PATCH="${PATCHES_DIR}/patch_timings_07351e088.py" TIMINGS_PATCH_URL="${TIMINGS_PATCH_URL:-https://gitea.va.reichard.io/evan/nix/raw/branch/master/modules/nixos/services/llama-swap/patches/patch_timings_07351e088.py}" -# Base URLs for sidecar patches (club-3090 repo) +# Base URL for sidecar patches (club-3090 repo) PATCH_BASE_URL="https://raw.githubusercontent.com/noonghunna/club-3090/master/models/qwen3.6-27b/vllm/patches" -PATCH_EXPERIMENTAL_BASE_URL="https://raw.githubusercontent.com/noonghunna/club-3090/v0.20-experimental/models/qwen3.6-27b/vllm/patches" # ---------- Preflight Checks ---------- -for cmd in git git-lfs curl; do +for cmd in git git-lfs curl python3; do if ! command -v "$cmd" &>/dev/null; then echo "ERROR: '$cmd' not found in PATH." >&2 exit 1 @@ -36,7 +41,7 @@ done # ---------- Create Directories ---------- echo "Creating directories..." -mkdir -p "${MODEL_DIR}" "${PATCHES_DIR}" +mkdir -p "${MODEL_DIR}" "${PATCHES_DIR}" "${CACHE_DIR}/torch_compile" "${CACHE_DIR}/triton" # ---------- Download Model ---------- if [ -d "${MODEL_DIR}/${MODEL_SUBDIR}/.git" ]; then @@ -48,16 +53,24 @@ else echo "Model cloned." fi -# ---------- Clone Genesis Patches ---------- +# ---------- Clone / Pin Genesis Patches ---------- if [ -d "${GENESIS_DIR}/.git" ]; then - echo "Genesis patches already cloned at ${GENESIS_DIR}, pulling latest..." - git -C "${GENESIS_DIR}" pull --ff-only || echo "Pull failed (non-fatal), using existing." + echo "Genesis already cloned — fetching + checking out ${GENESIS_PIN} ..." + (cd "${GENESIS_DIR}" && git fetch origin && git checkout "${GENESIS_PIN}" 2>&1 | tail -3) else - echo "Cloning Genesis patches..." + echo "Cloning Genesis patches at ${GENESIS_PIN} ..." git clone https://github.com/Sandermage/genesis-vllm-patches "${GENESIS_DIR}" - echo "Genesis patches cloned." + (cd "${GENESIS_DIR}" && git checkout "${GENESIS_PIN}") fi +# Sanity Check — v7.14+ layout +if [[ ! -d "${GENESIS_DIR}/vllm/_genesis" ]]; then + echo "ERROR: genesis tree at ${GENESIS_PIN} missing vllm/_genesis package." >&2 + echo " Re-run with GENESIS_PIN= to try a different version." >&2 + exit 1 +fi +echo "Genesis pinned to ${GENESIS_PIN} ($(cd "${GENESIS_DIR}" && git rev-parse --short HEAD))" + # ---------- Download Sidecar Patches ---------- # Fetched from club-3090 repo so this script is self-contained. download_patch() { @@ -74,21 +87,47 @@ download_patch() { } download_patch "${TOLIST_PATCH}" -download_patch "${PN12_FFN_PATCH}" -download_patch "${PN12_COMPILE_PATCH}" -download_patch "${FA_CLAMP_PATCH}" +download_patch "${WORKSPACE_LOCK_PATCH}" +download_patch "${PN25_REGISTER_PATCH}" +download_patch "${PN30_DST_PATCH}" +download_patch "${PR40798_PATCH}" -# ---------- Download v0.20 Workspace Patch ---------- -if [ -f "${WORKSPACE_LOCK_PATCH}" ]; then - echo "Patch $(basename "${WORKSPACE_LOCK_PATCH}") already present, skipping." -else - echo "Downloading $(basename "${WORKSPACE_LOCK_PATCH}") from v0.20-experimental..." - curl -fsSL \ - "${PATCH_EXPERIMENTAL_BASE_URL}/$(basename "${WORKSPACE_LOCK_PATCH}")" \ - -o "${WORKSPACE_LOCK_PATCH}" - echo "Patch $(basename "${WORKSPACE_LOCK_PATCH}") written." +# ---------- Apply Setup-Time Genesis Source Patches ---------- +# These modify the Genesis checkout in-place. The Genesis tree is mounted +# read-only into the container, so these MUST run at setup time, not boot. + +# The setup-time patches use hardcoded relative paths like +# "models/qwen3.6-27b/vllm/patches/genesis/vllm/_genesis/..." +# rooted at the club-3090 repo root. Our Genesis clone lives at +# ${PATCHES_DIR}/genesis, so we create a temporary symlink tree +# so the relative paths resolve correctly. +PATCH_WORKDIR="$(mktemp -d)" +mkdir -p "${PATCH_WORKDIR}/models/qwen3.6-27b/vllm/patches" +ln -sfn "${GENESIS_DIR}" "${PATCH_WORKDIR}/models/qwen3.6-27b/vllm/patches/genesis" + +# PN25 Worker-Spawn Registration Fix +# Registers the PN25 opaque op at module import time instead of inside +# the compiled forward path. Required for TP=1 spawned workers. +if [ -f "${PN25_REGISTER_PATCH}" ]; then + echo "Applying PN25 genesis register fix to Genesis tree..." + (cd "${PATCH_WORKDIR}" && python3 "${PN25_REGISTER_PATCH}") || { + echo "WARN: PN25 register fix did not apply cleanly. PN25 may not work in workers." >&2 + } fi +# PN30 DS Conv-State Dst-Shaped Temp Fix +# Corrects DS layout corruption in PN30's speculative decode path by +# building a destination-shaped temp instead of compacting source tail. +if [ -f "${PN30_DST_PATCH}" ]; then + echo "Applying PN30 dst-shaped temp fix to Genesis tree..." + (cd "${PATCH_WORKDIR}" && python3 "${PN30_DST_PATCH}") || { + echo "WARN: PN30 dst-shaped temp fix did not apply cleanly. DS layout may not work." >&2 + } +fi + +# Clean Up Symlink Workdir +rm -rf "${PATCH_WORKDIR}" + # ---------- Download Timing Patch ---------- tmp_timings_patch="$(mktemp)" trap 'rm -f "${tmp_timings_patch}"' EXIT @@ -108,21 +147,22 @@ fi echo "" echo "=== Setup Complete ===" echo " Model: ${MODEL_DIR}/${MODEL_SUBDIR}" -echo " Genesis: ${GENESIS_DIR}" -echo " Patch: ${TOLIST_PATCH}" -echo " Workspace: ${WORKSPACE_LOCK_PATCH}" -echo " Timings: ${TIMINGS_PATCH}" +echo " Genesis: ${GENESIS_DIR} (pinned: ${GENESIS_PIN})" +echo " Cache: ${CACHE_DIR}/{torch_compile,triton}" echo "" echo "Expected layout:" echo " /mnt/ssd/vLLM/" echo " ├── Models/" echo " │ └── qwen3.6-27b-autoround-int4/ (model weights)" +echo " ├── Cache/" +echo " │ ├── torch_compile/ (torch.compile cache)" +echo " │ └── triton/ (Triton kernel cache)" echo " └── Patches/" -echo " ├── genesis/ (Genesis v7.14+ repo)" -echo " │ └── vllm/_genesis/ (mounted into container)" -echo " ├── patch_tolist_cudagraph.py (cudagraph capture fix)" -echo " ├── patch_pn12_ffn_pool_anchor.py (PN12 FFN pool anchor fix)" -echo " ├── patch_pn12_compile_safe_custom_op.py (PN12 compile-safe custom op)" -echo " ├── patch_fa_max_seqlen_clamp.py (FA softmax_lse clamp — P104)" -echo " ├── patch_workspace_lock_disable.py (v0.20 WorkspaceManager lock workaround)" -echo " └── patch_timings_07351e088.py (llama.cpp-compatible timings)" +echo " ├── genesis/ (Genesis v7.65 @ ${GENESIS_PIN})" +echo " │ └── vllm/_genesis/ (mounted into container, PN25+PN30 applied)" +echo " ├── patch_tolist_cudagraph.py (boot-time: cudagraph capture fix)" +echo " ├── patch_workspace_lock_disable.py (boot-time: v0.20 WorkspaceManager lock workaround)" +echo " ├── patch_pn25_genesis_register_fix.py (setup-time: applied to Genesis tree)" +echo " ├── patch_pn30_dst_shaped_temp_fix.py (setup-time: applied to Genesis tree)" +echo " ├── patch_pr40798_workspace.py (PR40798 workspace fix)" +echo " └── patch_timings_07351e088.py (boot-time: llama.cpp-compatible timings)" diff --git a/secrets/common/llama-swap.yaml b/secrets/common/llama-swap.yaml index 13396fd..f179c04 100644 --- a/secrets/common/llama-swap.yaml +++ b/secrets/common/llama-swap.yaml @@ -1,6 +1,7 @@ #ENC[AES256_GCM,data:GdmmcWLHlE3LJvl9VfzbuEgZyGGqlKcrtNa+78/FFKO5coPf0n27eKwfo6UGuhf3ln++ePv37Eg=,iv:M+DWl7AZeQXJ0z4l6LHJBYrI/jW5NFY6b2tW9QnL9jM=,tag:fdy4feWIvKPCHbAcNZ6mmQ==,type:comment] llama_swap_api_keys: pi: ENC[AES256_GCM,data:7Cw7RPQemcf5/zO7uazjA+dzpQu2MQo/Nbe3K3/CJ+OeQR90SJx4Z0TZudFugZoIHWR+sPEGQxUk8ne5xcfY6GSHJA==,iv:B5fX93BtSNwIDUdWTXr3ZhBQ4AuUqDHjeeVbkcCk7HI=,tag:6RMyFEF5872waHzxUCUh0Q==,type:str] + evan: ENC[AES256_GCM,data:QKoFxv0gnDd1TZn9a+hFxu/J,iv:rje8Pk4ko8kjt1za/LOiLkoid4mmR5NtHCk0QX6rakg=,tag:MYTTE3KfWvfv2i98rTZUhQ==,type:str] sops: age: - recipient: age1sac93wpnjcv62s7583jv6a4yspndh6k0r25g3qx3k7gq748uvafst6nz4w @@ -75,7 +76,7 @@ sops: SmpMYnNBTWVYTENWSUQvWXMrZXVqbncK6KtP4pOEBDM8gK26uYp3a/WRP4TrkyWV 4ugL2Y7sGkVrWz0Cvr3Jp9QDuPh3xs4jZyEvB8RbxQDMFJzdOEBv2A== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-05-02T02:07:49Z" - mac: ENC[AES256_GCM,data:EroMrSyIJnQEP7Koo+195xmkloWDRQ21WuM0L+bTXzOLT1aQrUa15dxpYbQ+XUUVxtRfFHKSSXkyAG3UR/Ez7qhhxysQwL8ijUK7O+l7yn8EB3FbkQrYUIVASP6HzYe7XMd5Z+Zz/D6n172TxANLmaUxouYd4mlwcZzHSffP0AA=,iv:x5LKYpLb/Rkq6erH0wHCyZxbA7fr1NpW2JWCe3y2fnw=,tag:+ITvd9XqR5Pc1nj8snb2yA==,type:str] + lastmodified: "2026-05-02T10:31:38Z" + mac: ENC[AES256_GCM,data:ZF8NYaDKP42HHkfQ5Nr9uazFwEVZzzahq6mqybf16fQ7Rq9CXd+gCdD7Ie6Dq6gtEpNcCnKDWZwAgUYw5WSl1qzLFK3G5EMfvYvPdcggKuH7Tfxw8Ar6QA3Il/sEnQZgyuW77shXP0ma2XAFGaEXp5WuMIg1ZD8T0TKWeEr9L+Q=,iv:HaKd6MxlJvVYI9wMzuG7Dd656SUDl3moR/L65xQpmX4=,tag:Bo575jGyZ9UabUki9Yvtvg==,type:str] unencrypted_suffix: _unencrypted version: 3.12.1