feat(llama-swap): sync vLLM configs from club-3090, add evan API key
Sync all three vLLM model configs from club-3090 master (ae4846f). Update to Genesis v7.65 full PROD env set with new patches. Update docker image to nightly-7a1eb8ac. Add torch_compile and triton cache dirs. Add agent setup guide (AGENTS.md). Add 'evan' API key to llama-swap sops secrets.
This commit is contained in:
@@ -1,33 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
# Setup script for vLLM Qwen3.6-27B on a single 3090.
|
||||
#
|
||||
# Downloads the model, clones Genesis patches, and fetches all sidecar
|
||||
# Downloads the model, clones Genesis patches (pinned), applies setup-time
|
||||
# source patches to the Genesis tree, and fetches all boot-time sidecar
|
||||
# patches into place under /mnt/ssd/vLLM/.
|
||||
#
|
||||
# Idempotent — safe to re-run; skips steps already completed.
|
||||
#
|
||||
# Prerequisites: git (with git-lfs), docker
|
||||
# Prerequisites: git (with git-lfs), docker, python3
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
MODEL_DIR="/mnt/ssd/vLLM/Models"
|
||||
MODEL_SUBDIR="qwen3.6-27b-autoround-int4"
|
||||
PATCHES_DIR="/mnt/ssd/vLLM/Patches"
|
||||
CACHE_DIR="/mnt/ssd/vLLM/Cache"
|
||||
GENESIS_DIR="${PATCHES_DIR}/genesis"
|
||||
|
||||
# Pin Genesis to the validated commit (bump requires re-testing all composes)
|
||||
GENESIS_PIN="${GENESIS_PIN:-fc89395}"
|
||||
|
||||
TOLIST_PATCH="${PATCHES_DIR}/patch_tolist_cudagraph.py"
|
||||
PN12_FFN_PATCH="${PATCHES_DIR}/patch_pn12_ffn_pool_anchor.py"
|
||||
PN12_COMPILE_PATCH="${PATCHES_DIR}/patch_pn12_compile_safe_custom_op.py"
|
||||
FA_CLAMP_PATCH="${PATCHES_DIR}/patch_fa_max_seqlen_clamp.py"
|
||||
WORKSPACE_LOCK_PATCH="${PATCHES_DIR}/patch_workspace_lock_disable.py"
|
||||
PN25_REGISTER_PATCH="${PATCHES_DIR}/patch_pn25_genesis_register_fix.py"
|
||||
PN30_DST_PATCH="${PATCHES_DIR}/patch_pn30_dst_shaped_temp_fix.py"
|
||||
PR40798_PATCH="${PATCHES_DIR}/patch_pr40798_workspace.py"
|
||||
TIMINGS_PATCH="${PATCHES_DIR}/patch_timings_07351e088.py"
|
||||
TIMINGS_PATCH_URL="${TIMINGS_PATCH_URL:-https://gitea.va.reichard.io/evan/nix/raw/branch/master/modules/nixos/services/llama-swap/patches/patch_timings_07351e088.py}"
|
||||
|
||||
# Base URLs for sidecar patches (club-3090 repo)
|
||||
# Base URL for sidecar patches (club-3090 repo)
|
||||
PATCH_BASE_URL="https://raw.githubusercontent.com/noonghunna/club-3090/master/models/qwen3.6-27b/vllm/patches"
|
||||
PATCH_EXPERIMENTAL_BASE_URL="https://raw.githubusercontent.com/noonghunna/club-3090/v0.20-experimental/models/qwen3.6-27b/vllm/patches"
|
||||
|
||||
# ---------- Preflight Checks ----------
|
||||
for cmd in git git-lfs curl; do
|
||||
for cmd in git git-lfs curl python3; do
|
||||
if ! command -v "$cmd" &>/dev/null; then
|
||||
echo "ERROR: '$cmd' not found in PATH." >&2
|
||||
exit 1
|
||||
@@ -36,7 +41,7 @@ done
|
||||
|
||||
# ---------- Create Directories ----------
|
||||
echo "Creating directories..."
|
||||
mkdir -p "${MODEL_DIR}" "${PATCHES_DIR}"
|
||||
mkdir -p "${MODEL_DIR}" "${PATCHES_DIR}" "${CACHE_DIR}/torch_compile" "${CACHE_DIR}/triton"
|
||||
|
||||
# ---------- Download Model ----------
|
||||
if [ -d "${MODEL_DIR}/${MODEL_SUBDIR}/.git" ]; then
|
||||
@@ -48,16 +53,24 @@ else
|
||||
echo "Model cloned."
|
||||
fi
|
||||
|
||||
# ---------- Clone Genesis Patches ----------
|
||||
# ---------- Clone / Pin Genesis Patches ----------
|
||||
if [ -d "${GENESIS_DIR}/.git" ]; then
|
||||
echo "Genesis patches already cloned at ${GENESIS_DIR}, pulling latest..."
|
||||
git -C "${GENESIS_DIR}" pull --ff-only || echo "Pull failed (non-fatal), using existing."
|
||||
echo "Genesis already cloned — fetching + checking out ${GENESIS_PIN} ..."
|
||||
(cd "${GENESIS_DIR}" && git fetch origin && git checkout "${GENESIS_PIN}" 2>&1 | tail -3)
|
||||
else
|
||||
echo "Cloning Genesis patches..."
|
||||
echo "Cloning Genesis patches at ${GENESIS_PIN} ..."
|
||||
git clone https://github.com/Sandermage/genesis-vllm-patches "${GENESIS_DIR}"
|
||||
echo "Genesis patches cloned."
|
||||
(cd "${GENESIS_DIR}" && git checkout "${GENESIS_PIN}")
|
||||
fi
|
||||
|
||||
# Sanity Check — v7.14+ layout
|
||||
if [[ ! -d "${GENESIS_DIR}/vllm/_genesis" ]]; then
|
||||
echo "ERROR: genesis tree at ${GENESIS_PIN} missing vllm/_genesis package." >&2
|
||||
echo " Re-run with GENESIS_PIN=<other-ref> to try a different version." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Genesis pinned to ${GENESIS_PIN} ($(cd "${GENESIS_DIR}" && git rev-parse --short HEAD))"
|
||||
|
||||
# ---------- Download Sidecar Patches ----------
|
||||
# Fetched from club-3090 repo so this script is self-contained.
|
||||
download_patch() {
|
||||
@@ -74,21 +87,47 @@ download_patch() {
|
||||
}
|
||||
|
||||
download_patch "${TOLIST_PATCH}"
|
||||
download_patch "${PN12_FFN_PATCH}"
|
||||
download_patch "${PN12_COMPILE_PATCH}"
|
||||
download_patch "${FA_CLAMP_PATCH}"
|
||||
download_patch "${WORKSPACE_LOCK_PATCH}"
|
||||
download_patch "${PN25_REGISTER_PATCH}"
|
||||
download_patch "${PN30_DST_PATCH}"
|
||||
download_patch "${PR40798_PATCH}"
|
||||
|
||||
# ---------- Download v0.20 Workspace Patch ----------
|
||||
if [ -f "${WORKSPACE_LOCK_PATCH}" ]; then
|
||||
echo "Patch $(basename "${WORKSPACE_LOCK_PATCH}") already present, skipping."
|
||||
else
|
||||
echo "Downloading $(basename "${WORKSPACE_LOCK_PATCH}") from v0.20-experimental..."
|
||||
curl -fsSL \
|
||||
"${PATCH_EXPERIMENTAL_BASE_URL}/$(basename "${WORKSPACE_LOCK_PATCH}")" \
|
||||
-o "${WORKSPACE_LOCK_PATCH}"
|
||||
echo "Patch $(basename "${WORKSPACE_LOCK_PATCH}") written."
|
||||
# ---------- Apply Setup-Time Genesis Source Patches ----------
|
||||
# These modify the Genesis checkout in-place. The Genesis tree is mounted
|
||||
# read-only into the container, so these MUST run at setup time, not boot.
|
||||
|
||||
# The setup-time patches use hardcoded relative paths like
|
||||
# "models/qwen3.6-27b/vllm/patches/genesis/vllm/_genesis/..."
|
||||
# rooted at the club-3090 repo root. Our Genesis clone lives at
|
||||
# ${PATCHES_DIR}/genesis, so we create a temporary symlink tree
|
||||
# so the relative paths resolve correctly.
|
||||
PATCH_WORKDIR="$(mktemp -d)"
|
||||
mkdir -p "${PATCH_WORKDIR}/models/qwen3.6-27b/vllm/patches"
|
||||
ln -sfn "${GENESIS_DIR}" "${PATCH_WORKDIR}/models/qwen3.6-27b/vllm/patches/genesis"
|
||||
|
||||
# PN25 Worker-Spawn Registration Fix
|
||||
# Registers the PN25 opaque op at module import time instead of inside
|
||||
# the compiled forward path. Required for TP=1 spawned workers.
|
||||
if [ -f "${PN25_REGISTER_PATCH}" ]; then
|
||||
echo "Applying PN25 genesis register fix to Genesis tree..."
|
||||
(cd "${PATCH_WORKDIR}" && python3 "${PN25_REGISTER_PATCH}") || {
|
||||
echo "WARN: PN25 register fix did not apply cleanly. PN25 may not work in workers." >&2
|
||||
}
|
||||
fi
|
||||
|
||||
# PN30 DS Conv-State Dst-Shaped Temp Fix
|
||||
# Corrects DS layout corruption in PN30's speculative decode path by
|
||||
# building a destination-shaped temp instead of compacting source tail.
|
||||
if [ -f "${PN30_DST_PATCH}" ]; then
|
||||
echo "Applying PN30 dst-shaped temp fix to Genesis tree..."
|
||||
(cd "${PATCH_WORKDIR}" && python3 "${PN30_DST_PATCH}") || {
|
||||
echo "WARN: PN30 dst-shaped temp fix did not apply cleanly. DS layout may not work." >&2
|
||||
}
|
||||
fi
|
||||
|
||||
# Clean Up Symlink Workdir
|
||||
rm -rf "${PATCH_WORKDIR}"
|
||||
|
||||
# ---------- Download Timing Patch ----------
|
||||
tmp_timings_patch="$(mktemp)"
|
||||
trap 'rm -f "${tmp_timings_patch}"' EXIT
|
||||
@@ -108,21 +147,22 @@ fi
|
||||
echo ""
|
||||
echo "=== Setup Complete ==="
|
||||
echo " Model: ${MODEL_DIR}/${MODEL_SUBDIR}"
|
||||
echo " Genesis: ${GENESIS_DIR}"
|
||||
echo " Patch: ${TOLIST_PATCH}"
|
||||
echo " Workspace: ${WORKSPACE_LOCK_PATCH}"
|
||||
echo " Timings: ${TIMINGS_PATCH}"
|
||||
echo " Genesis: ${GENESIS_DIR} (pinned: ${GENESIS_PIN})"
|
||||
echo " Cache: ${CACHE_DIR}/{torch_compile,triton}"
|
||||
echo ""
|
||||
echo "Expected layout:"
|
||||
echo " /mnt/ssd/vLLM/"
|
||||
echo " ├── Models/"
|
||||
echo " │ └── qwen3.6-27b-autoround-int4/ (model weights)"
|
||||
echo " ├── Cache/"
|
||||
echo " │ ├── torch_compile/ (torch.compile cache)"
|
||||
echo " │ └── triton/ (Triton kernel cache)"
|
||||
echo " └── Patches/"
|
||||
echo " ├── genesis/ (Genesis v7.14+ repo)"
|
||||
echo " │ └── vllm/_genesis/ (mounted into container)"
|
||||
echo " ├── patch_tolist_cudagraph.py (cudagraph capture fix)"
|
||||
echo " ├── patch_pn12_ffn_pool_anchor.py (PN12 FFN pool anchor fix)"
|
||||
echo " ├── patch_pn12_compile_safe_custom_op.py (PN12 compile-safe custom op)"
|
||||
echo " ├── patch_fa_max_seqlen_clamp.py (FA softmax_lse clamp — P104)"
|
||||
echo " ├── patch_workspace_lock_disable.py (v0.20 WorkspaceManager lock workaround)"
|
||||
echo " └── patch_timings_07351e088.py (llama.cpp-compatible timings)"
|
||||
echo " ├── genesis/ (Genesis v7.65 @ ${GENESIS_PIN})"
|
||||
echo " │ └── vllm/_genesis/ (mounted into container, PN25+PN30 applied)"
|
||||
echo " ├── patch_tolist_cudagraph.py (boot-time: cudagraph capture fix)"
|
||||
echo " ├── patch_workspace_lock_disable.py (boot-time: v0.20 WorkspaceManager lock workaround)"
|
||||
echo " ├── patch_pn25_genesis_register_fix.py (setup-time: applied to Genesis tree)"
|
||||
echo " ├── patch_pn30_dst_shaped_temp_fix.py (setup-time: applied to Genesis tree)"
|
||||
echo " ├── patch_pr40798_workspace.py (PR40798 workspace fix)"
|
||||
echo " └── patch_timings_07351e088.py (boot-time: llama.cpp-compatible timings)"
|
||||
|
||||
Reference in New Issue
Block a user