diff --git a/modules/nixos/services/llama-swap/config.nix b/modules/nixos/services/llama-swap/config.nix index c1fcd16..5a88a15 100644 --- a/modules/nixos/services/llama-swap/config.nix +++ b/modules/nixos/services/llama-swap/config.nix @@ -98,9 +98,9 @@ in }; # https://github.com/noonghunna/club-3090/tree/master/models/qwen3.6-27b/vllm - # Synced from: club-3090 ae4846f (2026-05-02) — docker-compose.long-text.yml + # Synced from: club-3090 f6613c8 (2026-05-02) — docker-compose.long-text.yml # Long-text variant - 180K context, text-only (no vision) - # TurboQuant 3-bit KV + MTP n=3 + Genesis v7.65 full PROD env set + # TurboQuant 3-bit KV + MTP n=3 + Genesis v7.69 + Cliff 2 closure recipe "vllm-qwen3.6-27b-long-text" = { name = "vLLM Qwen3.6 (27B) - Long Text"; macros.ctx = "180000"; @@ -110,8 +110,8 @@ in vllmCmd = '' set -e; pip install xxhash pandas scipy -q; python3 -m vllm._genesis.patches.apply_all; - python3 /patches/patch_workspace_lock_disable.py; python3 /patches/patch_tolist_cudagraph.py; + python3 /patches/patch_inputs_embeds_optional.py; python3 /patches/patch_timings_07351e088.py; exec vllm serve --served-model-name ''${MODEL_ID} @@ -120,7 +120,7 @@ in --dtype float16 --tensor-parallel-size 1 --max-model-len ''${ctx} - --gpu-memory-utilization 0.95 + --gpu-memory-utilization 0.93 --max-num-seqs 1 --max-num-batched-tokens 4128 --kv-cache-dtype turboquant_3bit_nc @@ -184,6 +184,11 @@ in -e GENESIS_ENABLE_P100=1 \ -e GENESIS_ENABLE_P101=1 \ -e GENESIS_ENABLE_P103=1 \ + -e GENESIS_ENABLE_PN32_GDN_CHUNKED_PREFILL=1 \ + -e GENESIS_PN32_GDN_CHUNK_SIZE=8192 \ + -e GENESIS_PN32_GDN_CHUNK_THRESHOLD=16384 \ + -e GENESIS_FLA_FWD_H_MAX_T=16384 \ + -e GENESIS_ENABLE_PN34_WORKSPACE_LOCK_RELAX=1 \ -e GENESIS_ENABLE_PN8_MTP_DRAFT_ONLINE_QUANT=1 \ -e GENESIS_ENABLE_PN9_INDEPENDENT_DRAFTER_ATTN=1 \ -e GENESIS_ENABLE_PN11_GDN_AB_CONTIGUOUS=1 \ @@ -214,7 +219,7 @@ in -v /mnt/ssd/vLLM/Cache/triton:/root/.triton/cache \ -v /mnt/ssd/vLLM/Patches/genesis/vllm/_genesis:/usr/local/lib/python3.12/dist-packages/vllm/_genesis:ro \ -v /mnt/ssd/vLLM/Patches/patch_tolist_cudagraph.py:/patches/patch_tolist_cudagraph.py:ro \ - -v /mnt/ssd/vLLM/Patches/patch_workspace_lock_disable.py:/patches/patch_workspace_lock_disable.py:ro \ + -v /mnt/ssd/vLLM/Patches/patch_inputs_embeds_optional.py:/patches/patch_inputs_embeds_optional.py:ro \ -v /mnt/ssd/vLLM/Patches/patch_timings_07351e088.py:/patches/patch_timings_07351e088.py:ro \ -p ''${PORT}:8000 \ --entrypoint /bin/bash \ @@ -232,9 +237,9 @@ in }; # https://github.com/noonghunna/club-3090/tree/master/models/qwen3.6-27b/vllm - # Synced from: club-3090 ae4846f (2026-05-02) — docker-compose.long-vision.yml + # Synced from: club-3090 f6613c8 (2026-05-02) — docker-compose.long-vision.yml # Long-vision variant - 145K context with vision tower active - # TurboQuant 3-bit KV + MTP n=3 + Genesis v7.65 full PROD env set + # TurboQuant 3-bit KV + MTP n=3 + Genesis v7.69 + Cliff 2 env vars (mem-util kept at 0.95) "vllm-qwen3.6-27b-long-vision" = { name = "vLLM Qwen3.6 (27B) - Long Vision"; macros.ctx = "145000"; @@ -244,7 +249,6 @@ in vllmCmd = '' set -e; pip install xxhash pandas scipy -q; python3 -m vllm._genesis.patches.apply_all; - python3 /patches/patch_workspace_lock_disable.py; python3 /patches/patch_tolist_cudagraph.py; python3 /patches/patch_timings_07351e088.py; exec vllm serve @@ -317,6 +321,11 @@ in -e GENESIS_ENABLE_P100=1 \ -e GENESIS_ENABLE_P101=1 \ -e GENESIS_ENABLE_P103=1 \ + -e GENESIS_ENABLE_PN32_GDN_CHUNKED_PREFILL=1 \ + -e GENESIS_PN32_GDN_CHUNK_SIZE=8192 \ + -e GENESIS_PN32_GDN_CHUNK_THRESHOLD=16384 \ + -e GENESIS_FLA_FWD_H_MAX_T=16384 \ + -e GENESIS_ENABLE_PN34_WORKSPACE_LOCK_RELAX=1 \ -e GENESIS_ENABLE_PN8_MTP_DRAFT_ONLINE_QUANT=1 \ -e GENESIS_ENABLE_PN9_INDEPENDENT_DRAFTER_ATTN=1 \ -e GENESIS_ENABLE_PN11_GDN_AB_CONTIGUOUS=1 \ @@ -347,7 +356,6 @@ in -v /mnt/ssd/vLLM/Cache/triton:/root/.triton/cache \ -v /mnt/ssd/vLLM/Patches/genesis/vllm/_genesis:/usr/local/lib/python3.12/dist-packages/vllm/_genesis:ro \ -v /mnt/ssd/vLLM/Patches/patch_tolist_cudagraph.py:/patches/patch_tolist_cudagraph.py:ro \ - -v /mnt/ssd/vLLM/Patches/patch_workspace_lock_disable.py:/patches/patch_workspace_lock_disable.py:ro \ -v /mnt/ssd/vLLM/Patches/patch_timings_07351e088.py:/patches/patch_timings_07351e088.py:ro \ -p ''${PORT}:8000 \ --entrypoint /bin/bash \ diff --git a/modules/nixos/services/llama-swap/setup-qwen36-vllm.sh b/modules/nixos/services/llama-swap/setup-qwen36-vllm.sh index 714c2a7..2f8a324 100755 --- a/modules/nixos/services/llama-swap/setup-qwen36-vllm.sh +++ b/modules/nixos/services/llama-swap/setup-qwen36-vllm.sh @@ -7,7 +7,7 @@ # # Idempotent — safe to re-run; skips steps already completed. # -# Prerequisites: git (with git-lfs), docker, python3 +# Prerequisites: git (with git-lfs), docker set -euo pipefail @@ -18,9 +18,10 @@ CACHE_DIR="/mnt/ssd/vLLM/Cache" GENESIS_DIR="${PATCHES_DIR}/genesis" # Pin Genesis to the validated commit (bump requires re-testing all composes) -GENESIS_PIN="${GENESIS_PIN:-fc89395}" +GENESIS_PIN="${GENESIS_PIN:-2db18df}" TOLIST_PATCH="${PATCHES_DIR}/patch_tolist_cudagraph.py" +INPUTS_EMBEDS_PATCH="${PATCHES_DIR}/patch_inputs_embeds_optional.py" WORKSPACE_LOCK_PATCH="${PATCHES_DIR}/patch_workspace_lock_disable.py" PN25_REGISTER_PATCH="${PATCHES_DIR}/patch_pn25_genesis_register_fix.py" PN30_DST_PATCH="${PATCHES_DIR}/patch_pn30_dst_shaped_temp_fix.py" @@ -28,11 +29,11 @@ PR40798_PATCH="${PATCHES_DIR}/patch_pr40798_workspace.py" TIMINGS_PATCH="${PATCHES_DIR}/patch_timings_07351e088.py" TIMINGS_PATCH_URL="${TIMINGS_PATCH_URL:-https://gitea.va.reichard.io/evan/nix/raw/branch/master/modules/nixos/services/llama-swap/patches/patch_timings_07351e088.py}" -# Base URL for sidecar patches (club-3090 repo) -PATCH_BASE_URL="https://raw.githubusercontent.com/noonghunna/club-3090/master/models/qwen3.6-27b/vllm/patches" +# Base URL for sidecar patches (club-3090 repo, v7.69-cliff2-test branch) +PATCH_BASE_URL="https://raw.githubusercontent.com/noonghunna/club-3090/v7.69-cliff2-test/models/qwen3.6-27b/vllm/patches" # ---------- Preflight Checks ---------- -for cmd in git git-lfs curl python3; do +for cmd in git git-lfs curl; do if ! command -v "$cmd" &>/dev/null; then echo "ERROR: '$cmd' not found in PATH." >&2 exit 1 @@ -87,47 +88,12 @@ download_patch() { } download_patch "${TOLIST_PATCH}" +download_patch "${INPUTS_EMBEDS_PATCH}" download_patch "${WORKSPACE_LOCK_PATCH}" download_patch "${PN25_REGISTER_PATCH}" download_patch "${PN30_DST_PATCH}" download_patch "${PR40798_PATCH}" -# ---------- Apply Setup-Time Genesis Source Patches ---------- -# These modify the Genesis checkout in-place. The Genesis tree is mounted -# read-only into the container, so these MUST run at setup time, not boot. - -# The setup-time patches use hardcoded relative paths like -# "models/qwen3.6-27b/vllm/patches/genesis/vllm/_genesis/..." -# rooted at the club-3090 repo root. Our Genesis clone lives at -# ${PATCHES_DIR}/genesis, so we create a temporary symlink tree -# so the relative paths resolve correctly. -PATCH_WORKDIR="$(mktemp -d)" -mkdir -p "${PATCH_WORKDIR}/models/qwen3.6-27b/vllm/patches" -ln -sfn "${GENESIS_DIR}" "${PATCH_WORKDIR}/models/qwen3.6-27b/vllm/patches/genesis" - -# PN25 Worker-Spawn Registration Fix -# Registers the PN25 opaque op at module import time instead of inside -# the compiled forward path. Required for TP=1 spawned workers. -if [ -f "${PN25_REGISTER_PATCH}" ]; then - echo "Applying PN25 genesis register fix to Genesis tree..." - (cd "${PATCH_WORKDIR}" && python3 "${PN25_REGISTER_PATCH}") || { - echo "WARN: PN25 register fix did not apply cleanly. PN25 may not work in workers." >&2 - } -fi - -# PN30 DS Conv-State Dst-Shaped Temp Fix -# Corrects DS layout corruption in PN30's speculative decode path by -# building a destination-shaped temp instead of compacting source tail. -if [ -f "${PN30_DST_PATCH}" ]; then - echo "Applying PN30 dst-shaped temp fix to Genesis tree..." - (cd "${PATCH_WORKDIR}" && python3 "${PN30_DST_PATCH}") || { - echo "WARN: PN30 dst-shaped temp fix did not apply cleanly. DS layout may not work." >&2 - } -fi - -# Clean Up Symlink Workdir -rm -rf "${PATCH_WORKDIR}" - # ---------- Download Timing Patch ---------- tmp_timings_patch="$(mktemp)" trap 'rm -f "${tmp_timings_patch}"' EXIT @@ -158,11 +124,12 @@ echo " ├── Cache/" echo " │ ├── torch_compile/ (torch.compile cache)" echo " │ └── triton/ (Triton kernel cache)" echo " └── Patches/" -echo " ├── genesis/ (Genesis v7.65 @ ${GENESIS_PIN})" -echo " │ └── vllm/_genesis/ (mounted into container, PN25+PN30 applied)" +echo " ├── genesis/ (Genesis v7.69 @ ${GENESIS_PIN})" +echo " │ └── vllm/_genesis/ (mounted into container; PN25+PN30+PN34 native)" echo " ├── patch_tolist_cudagraph.py (boot-time: cudagraph capture fix)" -echo " ├── patch_workspace_lock_disable.py (boot-time: v0.20 WorkspaceManager lock workaround)" -echo " ├── patch_pn25_genesis_register_fix.py (setup-time: applied to Genesis tree)" -echo " ├── patch_pn30_dst_shaped_temp_fix.py (setup-time: applied to Genesis tree)" +echo " ├── patch_inputs_embeds_optional.py (boot-time: vllm#35975 backport, text-only models)" +echo " ├── patch_workspace_lock_disable.py (rollback: superseded by PN34 in v7.69)" +echo " ├── patch_pn25_genesis_register_fix.py (rollback: folded into v7.69 natively)" +echo " ├── patch_pn30_dst_shaped_temp_fix.py (rollback: folded into v7.69 natively)" echo " ├── patch_pr40798_workspace.py (PR40798 workspace fix)" echo " └── patch_timings_07351e088.py (boot-time: llama.cpp-compatible timings)" diff --git a/systems/x86_64-linux/lin-va-thinkpad/default.nix b/systems/x86_64-linux/lin-va-thinkpad/default.nix index 04ece21..9cf3fe8 100755 --- a/systems/x86_64-linux/lin-va-thinkpad/default.nix +++ b/systems/x86_64-linux/lin-va-thinkpad/default.nix @@ -57,6 +57,7 @@ in }; services = { + printing = enabled; openssh = enabled; tailscale = enabled; avahi = enabled;