evan/nix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(llama-swap): update Genesis to v7.69 and upgrade vLLM nightly image

Browse Source 
- Bump Genesis pin from 2db18df to 7b9fd319
- Upgrade vllm/vllm-openai nightly from 7a1eb8ac to 01d4d1ad
- Remove standalone boot-time patches now folded into Genesis (patch_tolist_cudagraph,
  patch_inputs_embeds_optional, patch_workspace_lock_disable, patch_pn25_genesis_register_fix,
  patch_pn30_dst_shaped_temp_fix, patch_pr40798_workspace)
- Reorganize environment variables across all vLLM compose configs
- Add new Genesis optimizations: P100, P101, P103, P15B, P38B, PN59 streaming GDN
This commit is contained in:
Evan Reichard
2026-05-05 15:50:37 -04:00
parent 3bccdc6382
commit d3ccbda958
2 changed files with 109 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

196
modules/nixos/services/llama-swap/config.nix
View File

@@ -110,8 +110,6 @@ in
vllmCmd = '' vllmCmd = ''
set -e; pip install xxhash pandas scipy -q; set -e; pip install xxhash pandas scipy -q;
python3 -m vllm._genesis.patches.apply_all; python3 -m vllm._genesis.patches.apply_all;
python3 /patches/patch_tolist_cudagraph.py;
python3 /patches/patch_inputs_embeds_optional.py;
python3 /patches/patch_timings_07351e088.py; python3 /patches/patch_timings_07351e088.py;
exec vllm serve exec vllm serve
--served-model-name ''${MODEL_ID} --served-model-name ''${MODEL_ID}
@@ -142,27 +140,21 @@ in
${pkgs.docker}/bin/docker run --rm --device=nvidia.com/gpu=all \ ${pkgs.docker}/bin/docker run --rm --device=nvidia.com/gpu=all \
--name ''${MODEL_ID} \ --name ''${MODEL_ID} \
--ipc=host \ --ipc=host \
-e VLLM_WORKER_MULTIPROC_METHOD=spawn \
-e NCCL_CUMEM_ENABLE=0 \
-e NCCL_P2P_DISABLE=1 \
-e VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 \
-e VLLM_NO_USAGE_STATS=1 \
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:512 \
-e VLLM_FLOAT32_MATMUL_PRECISION=high \
-e VLLM_USE_FLASHINFER_SAMPLER=1 \
-e OMP_NUM_THREADS=1 \
-e CUDA_DEVICE_MAX_CONNECTIONS=8 \ -e CUDA_DEVICE_MAX_CONNECTIONS=8 \
-e CUDA_VISIBLE_DEVICES=0 \
-e CUDA_DEVICE_ORDER=PCI_BUS_ID \ -e CUDA_DEVICE_ORDER=PCI_BUS_ID \
-e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \ -e CUDA_VISIBLE_DEVICES=0 \
-e VLLM_MARLIN_USE_ATOMIC_ADD=1 \ -e GENESIS_BUFFER_MODE=shared \
-e TRITON_CACHE_DIR=/root/.triton/cache \ -e GENESIS_ENABLE_P100=1 \
-e GENESIS_ENABLE_P101=1 \
-e GENESIS_ENABLE_P103=1 \
-e GENESIS_ENABLE_P15B_FA_VARLEN_CLAMP=1 \
-e GENESIS_ENABLE_P38B_COMPILE_SAFE=1 \
-e GENESIS_ENABLE_P4=1 \ -e GENESIS_ENABLE_P4=1 \
-e GENESIS_ENABLE_P58_ASYNC_PLACEHOLDER_FIX=1 \ -e GENESIS_ENABLE_P58_ASYNC_PLACEHOLDER_FIX=1 \
-e GENESIS_ENABLE_P60_GDN_NGRAM_FIX=1 \
-e GENESIS_ENABLE_P60B_TRITON_KERNEL=1 \ -e GENESIS_ENABLE_P60B_TRITON_KERNEL=1 \
-e GENESIS_ENABLE_P61_QWEN3_MULTI_TOOL=1 \ -e GENESIS_ENABLE_P60_GDN_NGRAM_FIX=1 \
-e GENESIS_ENABLE_P61B_STREAMING_OVERLAP=1 \ -e GENESIS_ENABLE_P61B_STREAMING_OVERLAP=1 \
-e GENESIS_ENABLE_P61_QWEN3_MULTI_TOOL=1 \
-e GENESIS_ENABLE_P62_STRUCT_OUT_SPEC_TIMING=1 \ -e GENESIS_ENABLE_P62_STRUCT_OUT_SPEC_TIMING=1 \
-e GENESIS_ENABLE_P64_QWEN3CODER_MTP_STREAMING=1 \ -e GENESIS_ENABLE_P64_QWEN3CODER_MTP_STREAMING=1 \
-e GENESIS_ENABLE_P65_TURBOQUANT_SPEC_CG_DOWNGRADE=1 \ -e GENESIS_ENABLE_P65_TURBOQUANT_SPEC_CG_DOWNGRADE=1 \
@@ -170,59 +162,65 @@ in
-e GENESIS_ENABLE_P67_TQ_MULTI_QUERY_KERNEL=1 \ -e GENESIS_ENABLE_P67_TQ_MULTI_QUERY_KERNEL=1 \
-e GENESIS_ENABLE_P68_AUTO_FORCE_TOOL=1 \ -e GENESIS_ENABLE_P68_AUTO_FORCE_TOOL=1 \
-e GENESIS_ENABLE_P69_LONG_CTX_TOOL_REMINDER=1 \ -e GENESIS_ENABLE_P69_LONG_CTX_TOOL_REMINDER=1 \
-e GENESIS_P68_P69_LONG_CTX_THRESHOLD_CHARS=50000 \
-e GENESIS_ENABLE_P72_PROFILE_RUN_CAP=1 \ -e GENESIS_ENABLE_P72_PROFILE_RUN_CAP=1 \
-e GENESIS_PROFILE_RUN_CAP_M=4128 \
-e GENESIS_ENABLE_P74_CHUNK_CLAMP=1 \ -e GENESIS_ENABLE_P74_CHUNK_CLAMP=1 \
-e GENESIS_ENABLE_P78_TOLIST_CAPTURE_GUARD=0 \
-e GENESIS_ENABLE_P81_FP8_BLOCK_SCALED_M_LE_8=0 \
-e GENESIS_ENABLE_P82=0 \
-e GENESIS_ENABLE_P83=1 \ -e GENESIS_ENABLE_P83=1 \
-e GENESIS_ENABLE_P87=1 \ -e GENESIS_ENABLE_P87=1 \
-e GENESIS_ENABLE_P91=1 \ -e GENESIS_ENABLE_P91=1 \
-e GENESIS_ENABLE_P94=1 \ -e GENESIS_ENABLE_P94=1 \
-e GENESIS_ENABLE_P98=1 \ -e GENESIS_ENABLE_P98=1 \
-e GENESIS_ENABLE_P99=1 \ -e GENESIS_ENABLE_P99=1 \
-e GENESIS_ENABLE_P100=1 \
-e GENESIS_ENABLE_P101=1 \
-e GENESIS_ENABLE_P103=1 \
-e GENESIS_ENABLE_PN32_GDN_CHUNKED_PREFILL=1 \
-e GENESIS_PN32_GDN_CHUNK_SIZE=8192 \
-e GENESIS_PN32_GDN_CHUNK_THRESHOLD=16384 \
-e GENESIS_FLA_FWD_H_MAX_T=16384 \
-e GENESIS_ENABLE_PN34_WORKSPACE_LOCK_RELAX=1 \
-e GENESIS_ENABLE_PN8_MTP_DRAFT_ONLINE_QUANT=1 \
-e GENESIS_ENABLE_PN9_INDEPENDENT_DRAFTER_ATTN=1 \
-e GENESIS_ENABLE_PN11_GDN_AB_CONTIGUOUS=1 \ -e GENESIS_ENABLE_PN11_GDN_AB_CONTIGUOUS=1 \
-e GENESIS_ENABLE_PN12_FFN_INTERMEDIATE_POOL=1 \ -e GENESIS_ENABLE_PN12_FFN_INTERMEDIATE_POOL=1 \
-e GENESIS_ENABLE_PN13_CUDA_GRAPH_LAMBDA_ARITY=1 \ -e GENESIS_ENABLE_PN13_CUDA_GRAPH_LAMBDA_ARITY=1 \
-e GENESIS_ENABLE_PN14_TQ_DECODE_OOB_CLAMP=1 \ -e GENESIS_ENABLE_PN14_TQ_DECODE_OOB_CLAMP=1 \
-e GENESIS_ENABLE_PN17_FA2_LSE_CLAMP=1 \ -e GENESIS_ENABLE_PN17_FA2_LSE_CLAMP=1 \
-e GENESIS_ENABLE_PN25_SILU_INDUCTOR_SAFE=1 \
-e GENESIS_ENABLE_PN30_DS_LAYOUT_SPEC_DECODE=1 \
-e GENESIS_PREALLOC_TOKEN_BUDGET=4128 \
-e GENESIS_BUFFER_MODE=shared \
-e GENESIS_ENABLE_P78_TOLIST_CAPTURE_GUARD=0 \
-e GENESIS_ENABLE_P81_FP8_BLOCK_SCALED_M_LE_8=0 \
-e GENESIS_ENABLE_P82=0 \
-e GENESIS_P82_THRESHOLD_SINGLE=0.3 \
-e GENESIS_ENABLE_PN19_SCOPED_MAX_SPLIT=1 \ -e GENESIS_ENABLE_PN19_SCOPED_MAX_SPLIT=1 \
-e GENESIS_ENABLE_PN22_LOCAL_ARGMAX_TP=1 \ -e GENESIS_ENABLE_PN22_LOCAL_ARGMAX_TP=1 \
-e GENESIS_ENABLE_PN25_SILU_INDUCTOR_SAFE=1 \
-e GENESIS_ENABLE_PN26_SPARSE_V=1 \ -e GENESIS_ENABLE_PN26_SPARSE_V=1 \
-e GENESIS_ENABLE_PN30_DS_LAYOUT_SPEC_DECODE=1 \
-e GENESIS_ENABLE_PN32_GDN_CHUNKED_PREFILL=1 \
-e GENESIS_ENABLE_PN34_WORKSPACE_LOCK_RELAX=1 \
-e GENESIS_ENABLE_PN59_STREAMING_GDN=1 \
-e GENESIS_ENABLE_PN8_MTP_DRAFT_ONLINE_QUANT=1 \
-e GENESIS_ENABLE_PN9_INDEPENDENT_DRAFTER_ATTN=1 \
-e GENESIS_FLA_FWD_H_MAX_T=16384 \
-e GENESIS_P68_P69_LONG_CTX_THRESHOLD_CHARS=50000 \
-e GENESIS_P82_THRESHOLD_SINGLE=0.3 \
-e GENESIS_PN26_SPARSE_V_BLOCK_KV=8 \ -e GENESIS_PN26_SPARSE_V_BLOCK_KV=8 \
-e GENESIS_PN26_SPARSE_V_NUM_WARPS=4 \ -e GENESIS_PN26_SPARSE_V_NUM_WARPS=4 \
-e GENESIS_PN26_SPARSE_V_THRESHOLD=0.01 \ -e GENESIS_PN26_SPARSE_V_THRESHOLD=0.01 \
-e GENESIS_ENABLE_P38B_COMPILE_SAFE=1 \ -e GENESIS_PN32_GDN_CHUNK_SIZE=8192 \
-e GENESIS_ENABLE_P15B_FA_VARLEN_CLAMP=1 \ -e GENESIS_PN32_GDN_CHUNK_THRESHOLD=16384 \
-e GENESIS_PN59_DEBUG=1 \
-e GENESIS_PREALLOC_TOKEN_BUDGET=4128 \
-e GENESIS_PROFILE_RUN_CAP_M=4128 \
-e NCCL_CUMEM_ENABLE=0 \
-e NCCL_P2P_DISABLE=1 \
-e OMP_NUM_THREADS=1 \
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:512 \
-e TRITON_CACHE_DIR=/root/.triton/cache \
-e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \
-e VLLM_FLOAT32_MATMUL_PRECISION=high \
-e VLLM_MARLIN_USE_ATOMIC_ADD=1 \
-e VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 \
-e VLLM_NO_USAGE_STATS=1 \
-e VLLM_SSM_CONV_STATE_LAYOUT=DS \ -e VLLM_SSM_CONV_STATE_LAYOUT=DS \
-e VLLM_USE_FLASHINFER_SAMPLER=1 \
-e VLLM_USE_FUSED_MOE_GROUPED_TOPK=1 \ -e VLLM_USE_FUSED_MOE_GROUPED_TOPK=1 \
-e VLLM_WORKER_MULTIPROC_METHOD=spawn \
-v /mnt/ssd/vLLM/Models:/root/.cache/huggingface \ -v /mnt/ssd/vLLM/Models:/root/.cache/huggingface \
-v /mnt/ssd/vLLM/Cache/torch_compile:/root/.cache/vllm/torch_compile_cache \ -v /mnt/ssd/vLLM/Cache/torch_compile:/root/.cache/vllm/torch_compile_cache \
-v /mnt/ssd/vLLM/Cache/triton:/root/.triton/cache \ -v /mnt/ssd/vLLM/Cache/triton:/root/.triton/cache \
-v /mnt/ssd/vLLM/Patches/genesis/vllm/_genesis:/usr/local/lib/python3.12/dist-packages/vllm/_genesis:ro \ -v /mnt/ssd/vLLM/Patches/genesis/vllm/_genesis:/usr/local/lib/python3.12/dist-packages/vllm/_genesis:ro \
-v /mnt/ssd/vLLM/Patches/patch_tolist_cudagraph.py:/patches/patch_tolist_cudagraph.py:ro \
-v /mnt/ssd/vLLM/Patches/patch_inputs_embeds_optional.py:/patches/patch_inputs_embeds_optional.py:ro \
-v /mnt/ssd/vLLM/Patches/patch_timings_07351e088.py:/patches/patch_timings_07351e088.py:ro \ -v /mnt/ssd/vLLM/Patches/patch_timings_07351e088.py:/patches/patch_timings_07351e088.py:ro \
-p ''${PORT}:8000 \ -p ''${PORT}:8000 \
--entrypoint /bin/bash \ --entrypoint /bin/bash \
vllm/vllm-openai:nightly-7a1eb8ac2ec4ea69338c51dc7afd4b15010abfa8 \ vllm/vllm-openai:nightly-01d4d1ad375dc5854779c593eee093bcebb0cada \
-c "${vllmCmdFlat}" -c "${vllmCmdFlat}"
''; '';
cmdStop = "${pkgs.docker}/bin/docker stop \${MODEL_ID}"; cmdStop = "${pkgs.docker}/bin/docker stop \${MODEL_ID}";
@@ -248,7 +246,6 @@ in
vllmCmd = '' vllmCmd = ''
set -e; pip install xxhash pandas scipy -q; set -e; pip install xxhash pandas scipy -q;
python3 -m vllm._genesis.patches.apply_all; python3 -m vllm._genesis.patches.apply_all;
python3 /patches/patch_tolist_cudagraph.py;
python3 /patches/patch_timings_07351e088.py; python3 /patches/patch_timings_07351e088.py;
exec vllm serve exec vllm serve
--served-model-name ''${MODEL_ID} --served-model-name ''${MODEL_ID}
@@ -278,27 +275,21 @@ in
${pkgs.docker}/bin/docker run --rm --device=nvidia.com/gpu=all \ ${pkgs.docker}/bin/docker run --rm --device=nvidia.com/gpu=all \
--name ''${MODEL_ID} \ --name ''${MODEL_ID} \
--ipc=host \ --ipc=host \
-e VLLM_WORKER_MULTIPROC_METHOD=spawn \
-e NCCL_CUMEM_ENABLE=0 \
-e NCCL_P2P_DISABLE=1 \
-e VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 \
-e VLLM_NO_USAGE_STATS=1 \
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:512 \
-e VLLM_FLOAT32_MATMUL_PRECISION=high \
-e VLLM_USE_FLASHINFER_SAMPLER=1 \
-e OMP_NUM_THREADS=1 \
-e CUDA_DEVICE_MAX_CONNECTIONS=8 \ -e CUDA_DEVICE_MAX_CONNECTIONS=8 \
-e CUDA_VISIBLE_DEVICES=0 \
-e CUDA_DEVICE_ORDER=PCI_BUS_ID \ -e CUDA_DEVICE_ORDER=PCI_BUS_ID \
-e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \ -e CUDA_VISIBLE_DEVICES=0 \
-e VLLM_MARLIN_USE_ATOMIC_ADD=1 \ -e GENESIS_BUFFER_MODE=shared \
-e TRITON_CACHE_DIR=/root/.triton/cache \ -e GENESIS_ENABLE_P100=1 \
-e GENESIS_ENABLE_P101=1 \
-e GENESIS_ENABLE_P103=1 \
-e GENESIS_ENABLE_P15B_FA_VARLEN_CLAMP=1 \
-e GENESIS_ENABLE_P38B_COMPILE_SAFE=1 \
-e GENESIS_ENABLE_P4=1 \ -e GENESIS_ENABLE_P4=1 \
-e GENESIS_ENABLE_P58_ASYNC_PLACEHOLDER_FIX=1 \ -e GENESIS_ENABLE_P58_ASYNC_PLACEHOLDER_FIX=1 \
-e GENESIS_ENABLE_P60_GDN_NGRAM_FIX=1 \
-e GENESIS_ENABLE_P60B_TRITON_KERNEL=1 \ -e GENESIS_ENABLE_P60B_TRITON_KERNEL=1 \
-e GENESIS_ENABLE_P61_QWEN3_MULTI_TOOL=1 \ -e GENESIS_ENABLE_P60_GDN_NGRAM_FIX=1 \
-e GENESIS_ENABLE_P61B_STREAMING_OVERLAP=1 \ -e GENESIS_ENABLE_P61B_STREAMING_OVERLAP=1 \
-e GENESIS_ENABLE_P61_QWEN3_MULTI_TOOL=1 \
-e GENESIS_ENABLE_P62_STRUCT_OUT_SPEC_TIMING=1 \ -e GENESIS_ENABLE_P62_STRUCT_OUT_SPEC_TIMING=1 \
-e GENESIS_ENABLE_P64_QWEN3CODER_MTP_STREAMING=1 \ -e GENESIS_ENABLE_P64_QWEN3CODER_MTP_STREAMING=1 \
-e GENESIS_ENABLE_P65_TURBOQUANT_SPEC_CG_DOWNGRADE=1 \ -e GENESIS_ENABLE_P65_TURBOQUANT_SPEC_CG_DOWNGRADE=1 \
@@ -306,26 +297,17 @@ in
-e GENESIS_ENABLE_P67_TQ_MULTI_QUERY_KERNEL=1 \ -e GENESIS_ENABLE_P67_TQ_MULTI_QUERY_KERNEL=1 \
-e GENESIS_ENABLE_P68_AUTO_FORCE_TOOL=1 \ -e GENESIS_ENABLE_P68_AUTO_FORCE_TOOL=1 \
-e GENESIS_ENABLE_P69_LONG_CTX_TOOL_REMINDER=1 \ -e GENESIS_ENABLE_P69_LONG_CTX_TOOL_REMINDER=1 \
-e GENESIS_P68_P69_LONG_CTX_THRESHOLD_CHARS=50000 \
-e GENESIS_ENABLE_P72_PROFILE_RUN_CAP=1 \ -e GENESIS_ENABLE_P72_PROFILE_RUN_CAP=1 \
-e GENESIS_PROFILE_RUN_CAP_M=4128 \
-e GENESIS_ENABLE_P74_CHUNK_CLAMP=1 \ -e GENESIS_ENABLE_P74_CHUNK_CLAMP=1 \
-e GENESIS_ENABLE_P78_TOLIST_CAPTURE_GUARD=0 \
-e GENESIS_ENABLE_P81_FP8_BLOCK_SCALED_M_LE_8=0 \
-e GENESIS_ENABLE_P82=0 \
-e GENESIS_ENABLE_P83=1 \ -e GENESIS_ENABLE_P83=1 \
-e GENESIS_ENABLE_P87=1 \ -e GENESIS_ENABLE_P87=1 \
-e GENESIS_ENABLE_P91=1 \ -e GENESIS_ENABLE_P91=1 \
-e GENESIS_ENABLE_P94=1 \ -e GENESIS_ENABLE_P94=1 \
-e GENESIS_ENABLE_P98=1 \ -e GENESIS_ENABLE_P98=1 \
-e GENESIS_ENABLE_P99=1 \ -e GENESIS_ENABLE_P99=1 \
-e GENESIS_ENABLE_P100=1 \
-e GENESIS_ENABLE_P101=1 \
-e GENESIS_ENABLE_P103=1 \
-e GENESIS_ENABLE_PN32_GDN_CHUNKED_PREFILL=1 \
-e GENESIS_PN32_GDN_CHUNK_SIZE=8192 \
-e GENESIS_PN32_GDN_CHUNK_THRESHOLD=16384 \
-e GENESIS_FLA_FWD_H_MAX_T=16384 \
-e GENESIS_ENABLE_PN34_WORKSPACE_LOCK_RELAX=1 \
-e GENESIS_ENABLE_PN8_MTP_DRAFT_ONLINE_QUANT=1 \
-e GENESIS_ENABLE_PN9_INDEPENDENT_DRAFTER_ATTN=1 \
-e GENESIS_ENABLE_PN11_GDN_AB_CONTIGUOUS=1 \ -e GENESIS_ENABLE_PN11_GDN_AB_CONTIGUOUS=1 \
-e GENESIS_ENABLE_PN12_FFN_INTERMEDIATE_POOL=1 \ -e GENESIS_ENABLE_PN12_FFN_INTERMEDIATE_POOL=1 \
-e GENESIS_ENABLE_PN13_CUDA_GRAPH_LAMBDA_ARITY=1 \ -e GENESIS_ENABLE_PN13_CUDA_GRAPH_LAMBDA_ARITY=1 \
@@ -333,31 +315,46 @@ in
-e GENESIS_ENABLE_PN17_FA2_LSE_CLAMP=1 \ -e GENESIS_ENABLE_PN17_FA2_LSE_CLAMP=1 \
-e GENESIS_ENABLE_PN19_SCOPED_MAX_SPLIT=1 \ -e GENESIS_ENABLE_PN19_SCOPED_MAX_SPLIT=1 \
-e GENESIS_ENABLE_PN22_LOCAL_ARGMAX_TP=1 \ -e GENESIS_ENABLE_PN22_LOCAL_ARGMAX_TP=1 \
-e GENESIS_ENABLE_PN25_SILU_INDUCTOR_SAFE=1 \
-e GENESIS_ENABLE_PN26_SPARSE_V=1 \ -e GENESIS_ENABLE_PN26_SPARSE_V=1 \
-e GENESIS_ENABLE_PN30_DS_LAYOUT_SPEC_DECODE=1 \
-e GENESIS_ENABLE_PN32_GDN_CHUNKED_PREFILL=1 \
-e GENESIS_ENABLE_PN34_WORKSPACE_LOCK_RELAX=1 \
-e GENESIS_ENABLE_PN59_STREAMING_GDN=1 \
-e GENESIS_ENABLE_PN8_MTP_DRAFT_ONLINE_QUANT=1 \
-e GENESIS_ENABLE_PN9_INDEPENDENT_DRAFTER_ATTN=1 \
-e GENESIS_FLA_FWD_H_MAX_T=16384 \
-e GENESIS_P68_P69_LONG_CTX_THRESHOLD_CHARS=50000 \
-e GENESIS_P82_THRESHOLD_SINGLE=0.3 \
-e GENESIS_PN26_SPARSE_V_BLOCK_KV=8 \ -e GENESIS_PN26_SPARSE_V_BLOCK_KV=8 \
-e GENESIS_PN26_SPARSE_V_NUM_WARPS=4 \ -e GENESIS_PN26_SPARSE_V_NUM_WARPS=4 \
-e GENESIS_PN26_SPARSE_V_THRESHOLD=0.01 \ -e GENESIS_PN26_SPARSE_V_THRESHOLD=0.01 \
-e GENESIS_ENABLE_P38B_COMPILE_SAFE=1 \ -e GENESIS_PN32_GDN_CHUNK_SIZE=8192 \
-e GENESIS_ENABLE_P15B_FA_VARLEN_CLAMP=1 \ -e GENESIS_PN32_GDN_CHUNK_THRESHOLD=16384 \
-e GENESIS_ENABLE_PN25_SILU_INDUCTOR_SAFE=1 \
-e GENESIS_ENABLE_PN30_DS_LAYOUT_SPEC_DECODE=1 \
-e GENESIS_PREALLOC_TOKEN_BUDGET=4128 \ -e GENESIS_PREALLOC_TOKEN_BUDGET=4128 \
-e GENESIS_BUFFER_MODE=shared \ -e GENESIS_PROFILE_RUN_CAP_M=4128 \
-e GENESIS_ENABLE_P78_TOLIST_CAPTURE_GUARD=0 \ -e NCCL_CUMEM_ENABLE=0 \
-e GENESIS_ENABLE_P81_FP8_BLOCK_SCALED_M_LE_8=0 \ -e NCCL_P2P_DISABLE=1 \
-e GENESIS_ENABLE_P82=0 \ -e OMP_NUM_THREADS=1 \
-e GENESIS_P82_THRESHOLD_SINGLE=0.3 \ -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:512 \
-e TRITON_CACHE_DIR=/root/.triton/cache \
-e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \
-e VLLM_FLOAT32_MATMUL_PRECISION=high \
-e VLLM_MARLIN_USE_ATOMIC_ADD=1 \
-e VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 \
-e VLLM_NO_USAGE_STATS=1 \
-e VLLM_SSM_CONV_STATE_LAYOUT=DS \ -e VLLM_SSM_CONV_STATE_LAYOUT=DS \
-e VLLM_USE_FLASHINFER_SAMPLER=1 \
-e VLLM_USE_FUSED_MOE_GROUPED_TOPK=1 \ -e VLLM_USE_FUSED_MOE_GROUPED_TOPK=1 \
-e VLLM_WORKER_MULTIPROC_METHOD=spawn \
-v /mnt/ssd/vLLM/Models:/root/.cache/huggingface \ -v /mnt/ssd/vLLM/Models:/root/.cache/huggingface \
-v /mnt/ssd/vLLM/Cache/torch_compile:/root/.cache/vllm/torch_compile_cache \ -v /mnt/ssd/vLLM/Cache/torch_compile:/root/.cache/vllm/torch_compile_cache \
-v /mnt/ssd/vLLM/Cache/triton:/root/.triton/cache \ -v /mnt/ssd/vLLM/Cache/triton:/root/.triton/cache \
-v /mnt/ssd/vLLM/Patches/genesis/vllm/_genesis:/usr/local/lib/python3.12/dist-packages/vllm/_genesis:ro \ -v /mnt/ssd/vLLM/Patches/genesis/vllm/_genesis:/usr/local/lib/python3.12/dist-packages/vllm/_genesis:ro \
-v /mnt/ssd/vLLM/Patches/patch_tolist_cudagraph.py:/patches/patch_tolist_cudagraph.py:ro \
-v /mnt/ssd/vLLM/Patches/patch_timings_07351e088.py:/patches/patch_timings_07351e088.py:ro \ -v /mnt/ssd/vLLM/Patches/patch_timings_07351e088.py:/patches/patch_timings_07351e088.py:ro \
-p ''${PORT}:8000 \ -p ''${PORT}:8000 \
--entrypoint /bin/bash \ --entrypoint /bin/bash \
vllm/vllm-openai:nightly-7a1eb8ac2ec4ea69338c51dc7afd4b15010abfa8 \ vllm/vllm-openai:nightly-01d4d1ad375dc5854779c593eee093bcebb0cada \
-c "${vllmCmdFlat}" -c "${vllmCmdFlat}"
''; '';
cmdStop = "${pkgs.docker}/bin/docker stop \${MODEL_ID}"; cmdStop = "${pkgs.docker}/bin/docker stop \${MODEL_ID}";
@@ -384,7 +381,6 @@ in
vllmCmd = '' vllmCmd = ''
set -e; pip install xxhash pandas scipy -q; set -e; pip install xxhash pandas scipy -q;
python3 -m vllm._genesis.patches.apply_all; python3 -m vllm._genesis.patches.apply_all;
python3 /patches/patch_tolist_cudagraph.py;
python3 /patches/patch_timings_07351e088.py; python3 /patches/patch_timings_07351e088.py;
exec vllm serve exec vllm serve
--served-model-name ''${MODEL_ID} --served-model-name ''${MODEL_ID}
@@ -414,45 +410,45 @@ in
${pkgs.docker}/bin/docker run --rm --device=nvidia.com/gpu=all \ ${pkgs.docker}/bin/docker run --rm --device=nvidia.com/gpu=all \
--name ''${MODEL_ID} \ --name ''${MODEL_ID} \
--ipc=host \ --ipc=host \
-e VLLM_WORKER_MULTIPROC_METHOD=spawn \
-e NCCL_CUMEM_ENABLE=0 \
-e NCCL_P2P_DISABLE=1 \
-e VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 \
-e VLLM_NO_USAGE_STATS=1 \
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:512 \
-e VLLM_FLOAT32_MATMUL_PRECISION=high \
-e VLLM_USE_FLASHINFER_SAMPLER=1 \
-e OMP_NUM_THREADS=1 \
-e CUDA_DEVICE_MAX_CONNECTIONS=8 \ -e CUDA_DEVICE_MAX_CONNECTIONS=8 \
-e CUDA_VISIBLE_DEVICES=0 \
-e CUDA_DEVICE_ORDER=PCI_BUS_ID \ -e CUDA_DEVICE_ORDER=PCI_BUS_ID \
-e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \ -e CUDA_VISIBLE_DEVICES=0 \
-e VLLM_MARLIN_USE_ATOMIC_ADD=1 \
-e TRITON_CACHE_DIR=/root/.triton/cache \
-e GENESIS_ENABLE_P58_ASYNC_PLACEHOLDER_FIX=1 \ -e GENESIS_ENABLE_P58_ASYNC_PLACEHOLDER_FIX=1 \
-e GENESIS_ENABLE_P64_QWEN3CODER_MTP_STREAMING=1 \ -e GENESIS_ENABLE_P64_QWEN3CODER_MTP_STREAMING=1 \
-e GENESIS_ENABLE_P66_CUDAGRAPH_SIZE_FILTER=1 \ -e GENESIS_ENABLE_P66_CUDAGRAPH_SIZE_FILTER=1 \
-e GENESIS_ENABLE_P68_AUTO_FORCE_TOOL=1 \ -e GENESIS_ENABLE_P68_AUTO_FORCE_TOOL=1 \
-e GENESIS_ENABLE_P69_LONG_CTX_TOOL_REMINDER=1 \ -e GENESIS_ENABLE_P69_LONG_CTX_TOOL_REMINDER=1 \
-e GENESIS_P68_P69_LONG_CTX_THRESHOLD_CHARS=50000 \
-e GENESIS_ENABLE_P72_PROFILE_RUN_CAP=1 \ -e GENESIS_ENABLE_P72_PROFILE_RUN_CAP=1 \
-e GENESIS_PROFILE_RUN_CAP_M=4128 \
-e GENESIS_ENABLE_P74_CHUNK_CLAMP=1 \ -e GENESIS_ENABLE_P74_CHUNK_CLAMP=1 \
-e GENESIS_ENABLE_P94=1 \ -e GENESIS_ENABLE_P94=1 \
-e GENESIS_ENABLE_PN8_MTP_DRAFT_ONLINE_QUANT=1 \
-e GENESIS_ENABLE_PN13_CUDA_GRAPH_LAMBDA_ARITY=1 \ -e GENESIS_ENABLE_PN13_CUDA_GRAPH_LAMBDA_ARITY=1 \
-e GENESIS_ENABLE_PN14_TQ_DECODE_OOB_CLAMP=1 \ -e GENESIS_ENABLE_PN14_TQ_DECODE_OOB_CLAMP=1 \
-e GENESIS_ENABLE_PN17_FA2_LSE_CLAMP=1 \ -e GENESIS_ENABLE_PN17_FA2_LSE_CLAMP=1 \
-e GENESIS_ENABLE_PN19_SCOPED_MAX_SPLIT=1 \ -e GENESIS_ENABLE_PN19_SCOPED_MAX_SPLIT=1 \
-e GENESIS_ENABLE_PN59_STREAMING_GDN=1 \
-e GENESIS_ENABLE_PN8_MTP_DRAFT_ONLINE_QUANT=1 \
-e GENESIS_P68_P69_LONG_CTX_THRESHOLD_CHARS=50000 \
-e GENESIS_PROFILE_RUN_CAP_M=4128 \
-e NCCL_CUMEM_ENABLE=0 \
-e NCCL_P2P_DISABLE=1 \
-e OMP_NUM_THREADS=1 \
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:512 \
-e TRITON_CACHE_DIR=/root/.triton/cache \
-e VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \
-e VLLM_FLOAT32_MATMUL_PRECISION=high \
-e VLLM_MARLIN_USE_ATOMIC_ADD=1 \
-e VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 \
-e VLLM_NO_USAGE_STATS=1 \
-e VLLM_USE_FLASHINFER_SAMPLER=1 \
-e VLLM_WORKER_MULTIPROC_METHOD=spawn \
-v /mnt/ssd/vLLM/Models:/root/.cache/huggingface \ -v /mnt/ssd/vLLM/Models:/root/.cache/huggingface \
-v /mnt/ssd/vLLM/Cache/torch_compile:/root/.cache/vllm/torch_compile_cache \ -v /mnt/ssd/vLLM/Cache/torch_compile:/root/.cache/vllm/torch_compile_cache \
-v /mnt/ssd/vLLM/Cache/triton:/root/.triton/cache \ -v /mnt/ssd/vLLM/Cache/triton:/root/.triton/cache \
-v /mnt/ssd/vLLM/Patches/genesis/vllm/_genesis:/usr/local/lib/python3.12/dist-packages/vllm/_genesis:ro \ -v /mnt/ssd/vLLM/Patches/genesis/vllm/_genesis:/usr/local/lib/python3.12/dist-packages/vllm/_genesis:ro \
-v /mnt/ssd/vLLM/Patches/patch_tolist_cudagraph.py:/patches/patch_tolist_cudagraph.py:ro \
-v /mnt/ssd/vLLM/Patches/patch_timings_07351e088.py:/patches/patch_timings_07351e088.py:ro \ -v /mnt/ssd/vLLM/Patches/patch_timings_07351e088.py:/patches/patch_timings_07351e088.py:ro \
-p ''${PORT}:8000 \ -p ''${PORT}:8000 \
--entrypoint /bin/bash \ --entrypoint /bin/bash \
vllm/vllm-openai:nightly-7a1eb8ac2ec4ea69338c51dc7afd4b15010abfa8 \ vllm/vllm-openai:nightly-01d4d1ad375dc5854779c593eee093bcebb0cada \
-c "${vllmCmdFlat}" -c "${vllmCmdFlat}"
''; '';
cmdStop = "${pkgs.docker}/bin/docker stop \${MODEL_ID}"; cmdStop = "${pkgs.docker}/bin/docker stop \${MODEL_ID}";

42
modules/nixos/services/llama-swap/setup-qwen36-vllm.sh
View File

@@ -5,33 +5,28 @@
# source patches to the Genesis tree, and fetches all boot-time sidecar # source patches to the Genesis tree, and fetches all boot-time sidecar
# patches into place under /mnt/ssd/vLLM/. # patches into place under /mnt/ssd/vLLM/.
# #
# Idempotent — safe to re-run; skips steps already completed. # Idempotent - safe to re-run; skips steps already completed.
# #
# Prerequisites: git (with git-lfs), docker # Prerequisites: git (with git-lfs), docker
set -euo pipefail set -euo pipefail
# Model Directories
MODEL_DIR="/mnt/ssd/vLLM/Models" MODEL_DIR="/mnt/ssd/vLLM/Models"
MODEL_SUBDIR="qwen3.6-27b-autoround-int4" MODEL_SUBDIR="qwen3.6-27b-autoround-int4"
PATCHES_DIR="/mnt/ssd/vLLM/Patches" PATCHES_DIR="/mnt/ssd/vLLM/Patches"
CACHE_DIR="/mnt/ssd/vLLM/Cache" CACHE_DIR="/mnt/ssd/vLLM/Cache"
GENESIS_DIR="${PATCHES_DIR}/genesis" GENESIS_DIR="${PATCHES_DIR}/genesis"
GENESIS_PIN="${GENESIS_PIN:-7b9fd319}"
# Pin Genesis to the validated commit (bump requires re-testing all composes) # 3090 Patches
GENESIS_PIN="${GENESIS_PIN:-2db18df}" BASE_3090_PATCH_URL="https://raw.githubusercontent.com/noonghunna/club-3090/v7.69-cliff2-test/models/qwen3.6-27b/vllm/patches"
TOLIST_PATCH="${PATCHES_DIR}/patch_tolist_cudagraph.py"
INPUTS_EMBEDS_PATCH="${PATCHES_DIR}/patch_inputs_embeds_optional.py" INPUTS_EMBEDS_PATCH="${PATCHES_DIR}/patch_inputs_embeds_optional.py"
WORKSPACE_LOCK_PATCH="${PATCHES_DIR}/patch_workspace_lock_disable.py"
PN25_REGISTER_PATCH="${PATCHES_DIR}/patch_pn25_genesis_register_fix.py" # Timings Patch
PN30_DST_PATCH="${PATCHES_DIR}/patch_pn30_dst_shaped_temp_fix.py"
PR40798_PATCH="${PATCHES_DIR}/patch_pr40798_workspace.py"
TIMINGS_PATCH="${PATCHES_DIR}/patch_timings_07351e088.py" TIMINGS_PATCH="${PATCHES_DIR}/patch_timings_07351e088.py"
TIMINGS_PATCH_URL="${TIMINGS_PATCH_URL:-https://gitea.va.reichard.io/evan/nix/raw/branch/master/modules/nixos/services/llama-swap/patches/patch_timings_07351e088.py}" TIMINGS_PATCH_URL="${TIMINGS_PATCH_URL:-https://gitea.va.reichard.io/evan/nix/raw/branch/master/modules/nixos/services/llama-swap/patches/patch_timings_07351e088.py}"
# Base URL for sidecar patches (club-3090 repo, v7.69-cliff2-test branch)
PATCH_BASE_URL="https://raw.githubusercontent.com/noonghunna/club-3090/v7.69-cliff2-test/models/qwen3.6-27b/vllm/patches"
# ---------- Preflight Checks ---------- # ---------- Preflight Checks ----------
for cmd in git git-lfs curl; do for cmd in git git-lfs curl; do
if ! command -v "$cmd" &>/dev/null; then if ! command -v "$cmd" &>/dev/null; then
@@ -56,7 +51,7 @@ fi
# ---------- Clone / Pin Genesis Patches ---------- # ---------- Clone / Pin Genesis Patches ----------
if [ -d "${GENESIS_DIR}/.git" ]; then if [ -d "${GENESIS_DIR}/.git" ]; then
echo "Genesis already cloned — fetching + checking out ${GENESIS_PIN} ..." echo "Genesis already cloned - fetching + checking out ${GENESIS_PIN} ..."
(cd "${GENESIS_DIR}" && git fetch origin && git checkout "${GENESIS_PIN}" 2>&1 | tail -3) (cd "${GENESIS_DIR}" && git fetch origin && git checkout "${GENESIS_PIN}" 2>&1 | tail -3)
else else
echo "Cloning Genesis patches at ${GENESIS_PIN} ..." echo "Cloning Genesis patches at ${GENESIS_PIN} ..."
@@ -64,7 +59,7 @@ else
(cd "${GENESIS_DIR}" && git checkout "${GENESIS_PIN}") (cd "${GENESIS_DIR}" && git checkout "${GENESIS_PIN}")
fi fi
# Sanity Check — v7.14+ layout # Sanity Check
if [[ ! -d "${GENESIS_DIR}/vllm/_genesis" ]]; then if [[ ! -d "${GENESIS_DIR}/vllm/_genesis" ]]; then
echo "ERROR: genesis tree at ${GENESIS_PIN} missing vllm/_genesis package." >&2 echo "ERROR: genesis tree at ${GENESIS_PIN} missing vllm/_genesis package." >&2
echo " Re-run with GENESIS_PIN=<other-ref> to try a different version." >&2 echo " Re-run with GENESIS_PIN=<other-ref> to try a different version." >&2
@@ -73,7 +68,6 @@ fi
echo "Genesis pinned to ${GENESIS_PIN} ($(cd "${GENESIS_DIR}" && git rev-parse --short HEAD))" echo "Genesis pinned to ${GENESIS_PIN} ($(cd "${GENESIS_DIR}" && git rev-parse --short HEAD))"
# ---------- Download Sidecar Patches ---------- # ---------- Download Sidecar Patches ----------
# Fetched from club-3090 repo so this script is self-contained.
download_patch() { download_patch() {
local dest="$1" local dest="$1"
local filename local filename
@@ -82,17 +76,12 @@ download_patch() {
echo "Patch ${filename} already present, skipping." echo "Patch ${filename} already present, skipping."
else else
echo "Downloading ${filename}..." echo "Downloading ${filename}..."
curl -fsSL "${PATCH_BASE_URL}/${filename}" -o "${dest}" curl -fsSL "${BASE_3090_PATCH_URL}/${filename}" -o "${dest}"
echo "Patch ${filename} written." echo "Patch ${filename} written."
fi fi
} }
download_patch "${TOLIST_PATCH}"
download_patch "${INPUTS_EMBEDS_PATCH}" download_patch "${INPUTS_EMBEDS_PATCH}"
download_patch "${WORKSPACE_LOCK_PATCH}"
download_patch "${PN25_REGISTER_PATCH}"
download_patch "${PN30_DST_PATCH}"
download_patch "${PR40798_PATCH}"
# ---------- Download Timing Patch ---------- # ---------- Download Timing Patch ----------
tmp_timings_patch="$(mktemp)" tmp_timings_patch="$(mktemp)"
@@ -113,8 +102,8 @@ fi
echo "" echo ""
echo "=== Setup Complete ===" echo "=== Setup Complete ==="
echo " Model: ${MODEL_DIR}/${MODEL_SUBDIR}" echo " Model: ${MODEL_DIR}/${MODEL_SUBDIR}"
echo " Genesis: ${GENESIS_DIR} (pinned: ${GENESIS_PIN})"
echo " Cache: ${CACHE_DIR}/{torch_compile,triton}" echo " Cache: ${CACHE_DIR}/{torch_compile,triton}"
echo " Genesis: ${GENESIS_DIR} (pinned: ${GENESIS_PIN})"
echo "" echo ""
echo "Expected layout:" echo "Expected layout:"
echo " /mnt/ssd/vLLM/" echo " /mnt/ssd/vLLM/"
@@ -124,12 +113,7 @@ echo " ├── Cache/"
echo " │ ├── torch_compile/ (torch.compile cache)" echo " │ ├── torch_compile/ (torch.compile cache)"
echo " │ └── triton/ (Triton kernel cache)" echo " │ └── triton/ (Triton kernel cache)"
echo " └── Patches/" echo " └── Patches/"
echo " ├── genesis/ (Genesis v7.69 @ ${GENESIS_PIN})" echo " ├── genesis/ (Genesis @ ${GENESIS_PIN})"
echo " │ └── vllm/_genesis/ (mounted into container; PN25+PN30+PN34 native)" echo " │ └── vllm/_genesis/ (mounted into container)"
echo " ├── patch_tolist_cudagraph.py (boot-time: cudagraph capture fix)"
echo " ├── patch_inputs_embeds_optional.py (boot-time: vllm#35975 backport, text-only models)" echo " ├── patch_inputs_embeds_optional.py (boot-time: vllm#35975 backport, text-only models)"
echo " ├── patch_workspace_lock_disable.py (rollback: superseded by PN34 in v7.69)"
echo " ├── patch_pn25_genesis_register_fix.py (rollback: folded into v7.69 natively)"
echo " ├── patch_pn30_dst_shaped_temp_fix.py (rollback: folded into v7.69 natively)"
echo " ├── patch_pr40798_workspace.py (PR40798 workspace fix)"
echo " └── patch_timings_07351e088.py (boot-time: llama.cpp-compatible timings)" echo " └── patch_timings_07351e088.py (boot-time: llama.cpp-compatible timings)"