diff --git a/modules/nixos/services/llama-swap/config.nix b/modules/nixos/services/llama-swap/config.nix index 7725b43..7c05487 100644 --- a/modules/nixos/services/llama-swap/config.nix +++ b/modules/nixos/services/llama-swap/config.nix @@ -129,7 +129,7 @@ in }; # https://github.com/noonghunna/club-3090/tree/master/models/qwen3.6-27b/vllm - # Synced from: club-3090 f6613c8 (2026-05-02) - docker-compose.long-text.yml + # Synced from: club-3090 e1137d6 (2026-05-09) - single/long-text.yml # Long-text variant - 180K context, text-only (no vision) # TurboQuant 3-bit KV + MTP n=3 + Genesis v7.69 + Cliff 2 closure recipe "vllm-qwen3.6-27b-long-text" = { @@ -141,8 +141,9 @@ in vllmCmd = '' set -e; pip install xxhash pandas scipy -q; python3 -m vllm._genesis.patches.apply_all; + python3 /patches/qwen3coder_tool_parser_deferred_commit.py; python3 /patches/patch_timings_07351e088.py; - exec vllm serve + exec vllm serve ''${VLLM_ENFORCE_EAGER:+--enforce-eager} --served-model-name ''${MODEL_ID} --model /root/.cache/huggingface/qwen3.6-27b-autoround-int4 --quantization auto_round @@ -188,7 +189,6 @@ in -e GENESIS_ENABLE_P61_QWEN3_MULTI_TOOL=1 \ -e GENESIS_ENABLE_P62_STRUCT_OUT_SPEC_TIMING=1 \ -e GENESIS_ENABLE_P64_QWEN3CODER_MTP_STREAMING=1 \ - -e GENESIS_ENABLE_P65_TURBOQUANT_SPEC_CG_DOWNGRADE=1 \ -e GENESIS_ENABLE_P66_CUDAGRAPH_SIZE_FILTER=1 \ -e GENESIS_ENABLE_P67_TQ_MULTI_QUERY_KERNEL=1 \ -e GENESIS_ENABLE_P68_AUTO_FORCE_TOOL=1 \ @@ -214,6 +214,7 @@ in -e GENESIS_ENABLE_PN25_SILU_INDUCTOR_SAFE=1 \ -e GENESIS_ENABLE_PN26_SPARSE_V=1 \ -e GENESIS_ENABLE_PN30_DS_LAYOUT_SPEC_DECODE=1 \ + -e GENESIS_ENABLE_PN31_FA_VARLEN_PERSISTENT_OUT=1 \ -e GENESIS_ENABLE_PN32_GDN_CHUNKED_PREFILL=1 \ -e GENESIS_ENABLE_PN34_WORKSPACE_LOCK_RELAX=1 \ -e GENESIS_ENABLE_PN59_STREAMING_GDN=1 \ @@ -244,10 +245,12 @@ in -e VLLM_USE_FLASHINFER_SAMPLER=1 \ -e VLLM_USE_FUSED_MOE_GROUPED_TOPK=1 \ -e VLLM_WORKER_MULTIPROC_METHOD=spawn \ + -e VLLM_ENFORCE_EAGER \ -v /mnt/ssd/vLLM/Models:/root/.cache/huggingface \ -v /mnt/ssd/vLLM/Cache/torch_compile:/root/.cache/vllm/torch_compile_cache \ -v /mnt/ssd/vLLM/Cache/triton:/root/.triton/cache \ -v /mnt/ssd/vLLM/Patches/genesis/vllm/_genesis:/usr/local/lib/python3.12/dist-packages/vllm/_genesis:ro \ + -v /mnt/ssd/vLLM/Patches/qwen3coder_tool_parser_deferred_commit.py:/patches/qwen3coder_tool_parser_deferred_commit.py:ro \ -v /mnt/ssd/vLLM/Patches/patch_timings_07351e088.py:/patches/patch_timings_07351e088.py:ro \ -p ''${PORT}:8000 \ --entrypoint /bin/bash \ @@ -265,7 +268,7 @@ in }; # https://github.com/noonghunna/club-3090/tree/master/models/qwen3.6-27b/vllm - # Synced from: club-3090 f6613c8 (2026-05-02) - docker-compose.long-vision.yml + # Synced from: club-3090 e1137d6 (2026-05-09) - single/long-vision.yml # Long-vision variant - 145K context with vision tower active # TurboQuant 3-bit KV + MTP n=3 + Genesis v7.69 + Cliff 2 env vars (mem-util kept at 0.95) "vllm-qwen3.6-27b-long-vision" = { @@ -277,8 +280,9 @@ in vllmCmd = '' set -e; pip install xxhash pandas scipy -q; python3 -m vllm._genesis.patches.apply_all; + python3 /patches/qwen3coder_tool_parser_deferred_commit.py; python3 /patches/patch_timings_07351e088.py; - exec vllm serve + exec vllm serve ''${VLLM_ENFORCE_EAGER:+--enforce-eager} --served-model-name ''${MODEL_ID} --model /root/.cache/huggingface/qwen3.6-27b-autoround-int4 --quantization auto_round @@ -323,7 +327,6 @@ in -e GENESIS_ENABLE_P61_QWEN3_MULTI_TOOL=1 \ -e GENESIS_ENABLE_P62_STRUCT_OUT_SPEC_TIMING=1 \ -e GENESIS_ENABLE_P64_QWEN3CODER_MTP_STREAMING=1 \ - -e GENESIS_ENABLE_P65_TURBOQUANT_SPEC_CG_DOWNGRADE=1 \ -e GENESIS_ENABLE_P66_CUDAGRAPH_SIZE_FILTER=1 \ -e GENESIS_ENABLE_P67_TQ_MULTI_QUERY_KERNEL=1 \ -e GENESIS_ENABLE_P68_AUTO_FORCE_TOOL=1 \ @@ -349,19 +352,15 @@ in -e GENESIS_ENABLE_PN25_SILU_INDUCTOR_SAFE=1 \ -e GENESIS_ENABLE_PN26_SPARSE_V=1 \ -e GENESIS_ENABLE_PN30_DS_LAYOUT_SPEC_DECODE=1 \ - -e GENESIS_ENABLE_PN32_GDN_CHUNKED_PREFILL=1 \ -e GENESIS_ENABLE_PN34_WORKSPACE_LOCK_RELAX=1 \ -e GENESIS_ENABLE_PN59_STREAMING_GDN=1 \ -e GENESIS_ENABLE_PN8_MTP_DRAFT_ONLINE_QUANT=1 \ -e GENESIS_ENABLE_PN9_INDEPENDENT_DRAFTER_ATTN=1 \ - -e GENESIS_FLA_FWD_H_MAX_T=16384 \ -e GENESIS_P68_P69_LONG_CTX_THRESHOLD_CHARS=50000 \ -e GENESIS_P82_THRESHOLD_SINGLE=0.3 \ -e GENESIS_PN26_SPARSE_V_BLOCK_KV=8 \ -e GENESIS_PN26_SPARSE_V_NUM_WARPS=4 \ -e GENESIS_PN26_SPARSE_V_THRESHOLD=0.01 \ - -e GENESIS_PN32_GDN_CHUNK_SIZE=8192 \ - -e GENESIS_PN32_GDN_CHUNK_THRESHOLD=16384 \ -e GENESIS_PREALLOC_TOKEN_BUDGET=4128 \ -e GENESIS_PROFILE_RUN_CAP_M=4128 \ -e NCCL_CUMEM_ENABLE=0 \ @@ -378,10 +377,12 @@ in -e VLLM_USE_FLASHINFER_SAMPLER=1 \ -e VLLM_USE_FUSED_MOE_GROUPED_TOPK=1 \ -e VLLM_WORKER_MULTIPROC_METHOD=spawn \ + -e VLLM_ENFORCE_EAGER \ -v /mnt/ssd/vLLM/Models:/root/.cache/huggingface \ -v /mnt/ssd/vLLM/Cache/torch_compile:/root/.cache/vllm/torch_compile_cache \ -v /mnt/ssd/vLLM/Cache/triton:/root/.triton/cache \ -v /mnt/ssd/vLLM/Patches/genesis/vllm/_genesis:/usr/local/lib/python3.12/dist-packages/vllm/_genesis:ro \ + -v /mnt/ssd/vLLM/Patches/qwen3coder_tool_parser_deferred_commit.py:/patches/qwen3coder_tool_parser_deferred_commit.py:ro \ -v /mnt/ssd/vLLM/Patches/patch_timings_07351e088.py:/patches/patch_timings_07351e088.py:ro \ -p ''${PORT}:8000 \ --entrypoint /bin/bash \ @@ -412,8 +413,9 @@ in vllmCmd = '' set -e; pip install xxhash pandas scipy -q; python3 -m vllm._genesis.patches.apply_all; + python3 /patches/qwen3coder_tool_parser_deferred_commit.py; python3 /patches/patch_timings_07351e088.py; - exec vllm serve + exec vllm serve ''${VLLM_ENFORCE_EAGER:+--enforce-eager} --served-model-name ''${MODEL_ID} --model /root/.cache/huggingface/qwen3.6-27b-autoround-int4 --quantization auto_round @@ -472,10 +474,12 @@ in -e VLLM_NO_USAGE_STATS=1 \ -e VLLM_USE_FLASHINFER_SAMPLER=1 \ -e VLLM_WORKER_MULTIPROC_METHOD=spawn \ + -e VLLM_ENFORCE_EAGER \ -v /mnt/ssd/vLLM/Models:/root/.cache/huggingface \ -v /mnt/ssd/vLLM/Cache/torch_compile:/root/.cache/vllm/torch_compile_cache \ -v /mnt/ssd/vLLM/Cache/triton:/root/.triton/cache \ -v /mnt/ssd/vLLM/Patches/genesis/vllm/_genesis:/usr/local/lib/python3.12/dist-packages/vllm/_genesis:ro \ + -v /mnt/ssd/vLLM/Patches/qwen3coder_tool_parser_deferred_commit.py:/patches/qwen3coder_tool_parser_deferred_commit.py:ro \ -v /mnt/ssd/vLLM/Patches/patch_timings_07351e088.py:/patches/patch_timings_07351e088.py:ro \ -p ''${PORT}:8000 \ --entrypoint /bin/bash \ diff --git a/modules/nixos/services/llama-swap/setup-qwen36-vllm.sh b/modules/nixos/services/llama-swap/setup-qwen36-vllm.sh index 67f4687..1afc2eb 100755 --- a/modules/nixos/services/llama-swap/setup-qwen36-vllm.sh +++ b/modules/nixos/services/llama-swap/setup-qwen36-vllm.sh @@ -23,6 +23,10 @@ GENESIS_PIN="${GENESIS_PIN:-7b9fd319}" BASE_3090_PATCH_URL="https://raw.githubusercontent.com/noonghunna/club-3090/v7.69-cliff2-test/models/qwen3.6-27b/vllm/patches" INPUTS_EMBEDS_PATCH="${PATCHES_DIR}/patch_inputs_embeds_optional.py" +# Tool Parser Patch +TOOL_PARSER_PATCH="${PATCHES_DIR}/qwen3coder_tool_parser_deferred_commit.py" +TOOL_PARSER_PATCH_URL="${TOOL_PARSER_PATCH_URL:-https://raw.githubusercontent.com/noonghunna/club-3090/refs/heads/master/models/qwen3.6-27b/vllm/patches/local/qwen3coder_tool_parser_deferred_commit.py}" + # Timings Patch TIMINGS_PATCH="${PATCHES_DIR}/patch_timings_07351e088.py" TIMINGS_PATCH_URL="${TIMINGS_PATCH_URL:-https://gitea.va.reichard.io/evan/nix/raw/branch/master/modules/nixos/services/llama-swap/patches/patch_timings_07351e088.py}" @@ -83,20 +87,31 @@ download_patch() { download_patch "${INPUTS_EMBEDS_PATCH}" -# ---------- Download Timing Patch ---------- -tmp_timings_patch="$(mktemp)" -trap 'rm -f "${tmp_timings_patch}"' EXIT +# ---------- Download URL Patch ---------- +install_url_patch() { + local name="$1" + local url="$2" + local dest="$3" + local tmp_patch + tmp_patch="$(mktemp)" -echo "Downloading patch_timings_07351e088.py from this repo..." -curl -fsSL "${TIMINGS_PATCH_URL}" -o "${tmp_timings_patch}" + echo "Downloading ${name}..." + curl -fsSL "${url}" -o "${tmp_patch}" -if [ -f "${TIMINGS_PATCH}" ] && cmp -s "${tmp_timings_patch}" "${TIMINGS_PATCH}"; then - echo "Timing patch already current at ${TIMINGS_PATCH}, skipping." -else - echo "Installing timing patch to ${TIMINGS_PATCH}..." - install -m 0644 "${tmp_timings_patch}" "${TIMINGS_PATCH}" - echo "Timing patch installed." -fi + if [ -f "${dest}" ] && cmp -s "${tmp_patch}" "${dest}"; then + echo "${name} already current at ${dest}, skipping." + else + echo "Installing ${name} to ${dest}..." + install -m 0644 "${tmp_patch}" "${dest}" + echo "${name} installed." + fi + + rm -f "${tmp_patch}" +} + +# ---------- Download Boot-Time Patches ---------- +install_url_patch "qwen3coder_tool_parser_deferred_commit.py" "${TOOL_PARSER_PATCH_URL}" "${TOOL_PARSER_PATCH}" +install_url_patch "patch_timings_07351e088.py" "${TIMINGS_PATCH_URL}" "${TIMINGS_PATCH}" # ---------- Summary ---------- echo "" @@ -116,4 +131,5 @@ echo " └── Patches/" echo " ├── genesis/ (Genesis @ ${GENESIS_PIN})" echo " │ └── vllm/_genesis/ (mounted into container)" echo " ├── patch_inputs_embeds_optional.py (boot-time: vllm#35975 backport, text-only models)" +echo " ├── qwen3coder_tool_parser_deferred_commit.py (boot-time: qwen3coder SSE deferred commit fix)" echo " └── patch_timings_07351e088.py (boot-time: llama.cpp-compatible timings)"