diff --git a/.agents/skills/update-package-hashes/SKILL.md b/.agents/skills/update-package-hashes/SKILL.md index da190e8..011ffe7 100644 --- a/.agents/skills/update-package-hashes/SKILL.md +++ b/.agents/skills/update-package-hashes/SKILL.md @@ -12,7 +12,7 @@ If the user provides only a **package name** (no version), look up the latest ve ## Hard Rules — Read First 1. **Never run `nix build .#`** or `.#packages..`. That compiles the package. Only realise **FOD sub-attributes** (`.src`, `.goModules`, `.npmDeps`, `.cargoDeps`) — those are pure downloads, not builds. -2. **Never** use `nix-prefetch-git`, `nix-prefetch-url`, `nix hash path`, `git clone` + manual hashing, `builtins.fetchGit`, or any other ad-hoc method to compute hashes. They produce hashes in formats that don't match what `fetchgit`/`fetchFromGitHub`/etc. expect, and you will waste time chasing mismatches. +2. **Never** use `nix-prefetch-git`, `nix-prefetch-github`, `nix-prefetch-url`, `nix hash path`, `nix hash file` (on a raw patch/tarball), `git clone` + manual hashing, `builtins.fetchGit`, or any other ad-hoc method to compute hashes. They produce hashes in formats that don't match what `fetchgit`/`fetchFromGitHub`/`fetchpatch` expect (notably: `fetchFromGitHub { leaveDotGit = true; }` is non-deterministic across machines, and `fetchpatch` normalizes patches — strips `index abc..def`, `From `, signatures — so its hash ≠ `nix hash file` of the raw `.patch`). 3. There are exactly **two** correct ways to get a hash, both listed below. If neither fits, stop and ask the user — don't improvise. ## The Only Two Methods @@ -29,16 +29,26 @@ Copy the `hash = "sha256-..."` line from the output into the package's `src` blo ### Method B — FOD mismatch trick (for everything else) -For `vendorHash`, `npmDepsHash`, `cargoHash`, `cargoLock.outputHashes.`, or any `src` using a custom fetcher (`leaveDotGit`, `postFetch`, `fetchSubmodules`, etc. — applies to `llama-cpp` and `llama-swap`), realise the **specific FOD sub-attribute** and read the `got:` line from the error. +For `vendorHash`, `npmDepsHash`, `cargoHash`, `cargoLock.outputHashes.`, `fetchpatch` hashes, or any `src` using a custom fetcher (`leaveDotGit`, `postFetch`, `fetchSubmodules`, etc. — applies to `llama-cpp` and `llama-swap`), realise the **specific FOD sub-attribute** and read the `got:` line from the error. ```bash nix build .#.src --no-link 2>&1 | tee /tmp/hash.log # for src nix build .#.goModules --no-link 2>&1 | tee /tmp/hash.log # for vendorHash nix build .#.npmDeps --no-link 2>&1 | tee /tmp/hash.log # for npmDepsHash nix build .#.cargoDeps --no-link 2>&1 | tee /tmp/hash.log # for cargoHash +nix build .# --no-link 2>&1 | tee /tmp/hash.log # for fetchpatch / other input FODs (see note) grep -E '^[[:space:]]*got:' /tmp/hash.log | tail -1 | awk '{print $2}' ``` +**`fetchpatch` note:** patches don't have a dedicated sub-attribute, so you must target the package itself. This is safe *only* when the patch hash is wrong (e.g. `lib.fakeHash`) — Nix realizes the patch FOD before compilation starts, so a hash mismatch aborts with `0 built (1 failed)` and zero compile work. If you accidentally fix all FODs correctly, `nix build .#` will start compiling. To guard against this: always start patch hashes as `lib.fakeHash`, run the build, copy `got:`, paste, and only then re-verify with `.src` / sub-attribute builds (never re-run `.#` to confirm). + +**GitHub PR patches — `.patch` vs `.diff`:** When fetching a patch from a GitHub pull request, prefer the `.diff` endpoint over `.patch`. + +- `https://github.com///pull/.patch` — a `git format-patch` **mbox** containing each commit in the PR separately. `git apply` (which `fetchpatch` and the Nix `patchPhase` use) does **not** replay commit history; it applies hunks against the working tree. PRs that create a file in one commit and delete/rename it in a later commit will fail with errors like `The next patch would delete the file X, which does not exist`. +- `https://github.com///pull/.diff` — a **squashed** unified diff of the PR's net change. Applies cleanly against any base the PR is mergeable against. + +Default to `.diff`. Only fall back to `.patch` if you specifically need authorship metadata (rare for Nix patching). If a previously-working `.patch` URL suddenly fails to apply, switching to `.diff` is the first thing to try. + Setting the hash to `lib.fakeHash` (preferred when `lib` is in scope), `sha256-AAAA...` (44 A's), or leaving the old one in place all work — the build will fail at the FOD with `got: sha256-...` which is the correct value. **Note:** `.src`, `.goModules`, etc. are sub-attributes of the derivation. They download but do not compile. `nix build .#` (without the `.src` suffix) compiles — never do that. diff --git a/modules/nixos/services/llama-swap/config.nix b/modules/nixos/services/llama-swap/config.nix index 437fa71..8319755 100644 --- a/modules/nixos/services/llama-swap/config.nix +++ b/modules/nixos/services/llama-swap/config.nix @@ -75,7 +75,7 @@ in --presence-penalty 0.0 \ -ctk q8_0 \ -ctv q8_0 \ - --spec-type mtp \ + --spec-type draft-mtp \ --spec-draft-n-max 3 \ -dev CUDA0 \ -fit off \ @@ -150,6 +150,51 @@ in }; }; + # https://huggingface.co/Lorbus/Qwen3.6-27B-int4-AutoRound + # Vanilla single-GPU vLLM config pinned to CUDA0 with 50K context. + "qwen3.6-27b-vllm-50k" = { + name = "Qwen 3.6 27B INT4 AutoRound (vLLM - Single GPU - 50K ctx)"; + checkEndpoint = "/v1/models"; + macros.ctx = "50000"; + proxy = "http://127.0.0.1:\${PORT}"; + cmd = '' + ${pkgs.docker}/bin/docker run --rm --device=nvidia.com/gpu=all \ + --name ''${MODEL_ID} \ + -e CUDA_DEVICE_ORDER=PCI_BUS_ID \ + -e CUDA_VISIBLE_DEVICES=0 \ + -e VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 \ + -v /mnt/ssd/vLLM/Models:/root/.cache/huggingface \ + -p ''${PORT}:8000 \ + vllm/vllm-openai:latest \ + /root/.cache/huggingface/qwen3.6-27b-autoround-int4 \ + --served-model-name ''${MODEL_ID} \ + --quantization auto_round \ + --dtype float16 \ + --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.97 \ + --max-model-len ''${ctx} \ + --max-num-seqs 1 \ + --max-num-batched-tokens 4128 \ + --kv-cache-dtype fp8_e5m2 \ + --enable-chunked-prefill \ + --enable-prefix-caching \ + --speculative-config '{"method":"mtp","num_speculative_tokens":3}' \ + --enable-auto-tool-choice \ + --tool-call-parser qwen3_coder \ + --trust-remote-code \ + --default-chat-template-kwargs '{"enable_thinking": false}' \ + --host 0.0.0.0 \ + --port 8000 + ''; + cmdStop = "${pkgs.docker}/bin/docker stop \${MODEL_ID}"; + metadata = { + type = [ + "text-generation" + "coding" + ]; + }; + }; + # https://github.com/noonghunna/club-3090/tree/master/models/qwen3.6-27b/vllm # Upstream: club-3090 83bf73d (2026-05-10) - single/long-text.yml # Long-text variant - 180K context, text-only (no vision) @@ -656,7 +701,7 @@ in --presence-penalty 0.0 \ -ctk q8_0 \ -ctv q8_0 \ - --spec-type mtp \ + --spec-type draft-mtp \ --spec-draft-n-max 3 \ -dev CUDA0,CUDA1 \ -ts 75,25 \ @@ -772,6 +817,7 @@ in vlt = "vllm-qwen3.6-27b-long-text"; vtt = "vllm-qwen3.6-27b-tools-text"; vlv = "vllm-qwen3.6-27b-long-vision"; + v50 = "qwen3.6-27b-vllm-50k"; go = "gpt-oss-20b-thinking"; g4 = "gemma-4-26b-vision"; q36a = "qwen3.6-35b-thinking"; @@ -789,7 +835,7 @@ in }; sets = { - concurrent = "(go | g4 | q36a | q36b | iq36 | vlt | vtt | vlv | zi | qie | qi | cr) & (qv | q4 | q9)"; + concurrent = "(go | g4 | q36a | q36b | iq36 | vlt | vtt | vlv | v50 | zi | qie | qi | cr) & (qv | q4 | q9)"; }; }; } diff --git a/packages/llama-cpp/AGENTS.md b/packages/llama-cpp/AGENTS.md new file mode 100644 index 0000000..5c92d52 --- /dev/null +++ b/packages/llama-cpp/AGENTS.md @@ -0,0 +1,35 @@ +# llama-cpp — Agent Notes + +Override of `pkgs.llama-cpp` with CUDA + Vulkan + BLAS, custom CMake flags, and an optional fork pin. + +## Pitfalls + +### `version` must be numeric + +Upstream `pkgs/by-name/ll/llama-cpp/package.nix` passes `version` straight through as a C integer via: + +```nix +(cmakeFeature "LLAMA_BUILD_NUMBER" finalAttrs.version) +``` + +`build-info.cpp` then emits `int LLAMA_BUILD_NUMBER = ;`. A non-numeric `version` (e.g. `"mtp-clean-08b1474"`) breaks the build with: + +``` +error: '' was not declared in this scope + int LLAMA_BUILD_NUMBER = ; +``` + +**Convention:** +- Upstream tag pins: use the bare build number, e.g. `version = "9048";` with `tag = "b${version}";`. +- Fork / arbitrary commit pins: use a `YYYYMMDD` date derived from the commit's author/commit date (`gh api repos///commits/` → `.commit.committer.date`). + +### `leaveDotGit` + `postFetch` + +We keep `.git` only long enough to record the short SHA into `$out/COMMIT`, then strip it. Preserve this pattern when changing `src` so downstream tooling that reads `COMMIT` keeps working. + +## Refreshing the pinned commit (fork) + +1. `git ls-remote https://github.com//llama.cpp refs/heads/` → get the full SHA. +2. `nix run nixpkgs#nix-prefetch-github -- llama.cpp --rev --leave-dot-git` → get the hash. +3. Look up the commit date: `curl -s https://api.github.com/repos//llama.cpp/commits/ | jq -r '.commit.committer.date'`. +4. Update `src.{owner,rev,hash}` and set `version = "YYYYMMDD"`. diff --git a/packages/llama-cpp/default.nix b/packages/llama-cpp/default.nix index 406db6d..857aafc 100644 --- a/packages/llama-cpp/default.nix +++ b/packages/llama-cpp/default.nix @@ -1,4 +1,75 @@ { pkgs }: +let + version = "9159"; + + src = pkgs.fetchFromGitHub { + owner = "ggml-org"; + repo = "llama.cpp"; + tag = "b${version}"; + hash = "sha256-y69ZmVFxo7bQvLTT6/GWwkb5j4Ll8eXSVXFpfXVkvyg="; + leaveDotGit = true; + postFetch = '' + git -C "$out" rev-parse --short HEAD > $out/COMMIT + find "$out" -name .git -print0 | xargs -0 rm -rf + ''; + }; + + # MTP Patch (PR #22673) + # Use the .diff (squashed) endpoint, not .patch (mbox of commits). + mtpPatch = pkgs.fetchpatch { + name = "mtp.patch"; + url = "https://github.com/ggml-org/llama.cpp/pull/22673.diff"; + hash = "sha256-8W02V7oqq2/SzSrDUHEa5Zm+dGBFkasOiVinww3V85U="; + }; + + # Pre-Built WebUI Assets + # As of b9151 llama.cpp removed the prebuilt WebUI from the repo and tries to + # curl them from a HuggingFace bucket at build time. That fails in the Nix + # sandbox. We build the WebUI from source in a separate derivation and drop + # the 4 output files into tools/server/public/ so cmake's "Priority 1: local + # assets present" branch short-circuits the network fetch. + # Vendored npm deps. The npmDeps FOD uses default unpack and looks for + # package-lock.json at sourceRoot, so point it directly at the webui dir. + webuiNpmDeps = pkgs.fetchNpmDeps { + name = "llama-webui-${version}-npm-deps"; + inherit src; + sourceRoot = "${src.name}/tools/server/webui"; + hash = "sha256-WaEePrEZ7O/7deP2KJhe0AwiSKYA8HOqETmMHUkmBe0="; + }; + + webui = pkgs.buildNpmPackage { + pname = "llama-webui"; + inherit version src; + + # Custom unpack: the vite plugin writes to ../public, so both webui/ and + # public/ must be writable siblings under tools/server/. Plain sourceRoot + # leaves the parent dirs in the read-only Nix store. + unpackPhase = '' + runHook preUnpack + cp -r ${src}/tools/server tools-server + chmod -R u+w tools-server + cd tools-server/webui + runHook postUnpack + ''; + + npmDeps = webuiNpmDeps; + + # The vite plugin writes to ../public; ensure it exists. + preBuild = '' + mkdir -p ../public + ''; + + installPhase = '' + runHook preInstall + mkdir -p $out + install -Dm644 ../public/index.html $out/index.html + install -Dm644 ../public/bundle.js $out/bundle.js + install -Dm644 ../public/bundle.css $out/bundle.css + install -Dm644 ../public/loading.html $out/loading.html + runHook postInstall + ''; + }; +in (pkgs.llama-cpp.override { cudaSupport = true; blasSupport = true; @@ -6,19 +77,8 @@ metalSupport = false; vulkanSupport = true; }).overrideAttrs - (oldAttrs: rec { - version = "9048"; - src = pkgs.fetchFromGitHub { - owner = "ggml-org"; - repo = "llama.cpp"; - tag = "b${version}"; - hash = "sha256-lYtX0hLReCnFw1+xOKefly+WunuoN89ZFEFl5mK5pQ4="; - leaveDotGit = true; - postFetch = '' - git -C "$out" rev-parse --short HEAD > $out/COMMIT - find "$out" -name .git -print0 | xargs -0 rm -rf - ''; - }; + (oldAttrs: { + inherit version src; # Add SPIR-V Headers for Vulkan Backend # Newer llama.cpp requires spirv/unified1/spirv.hpp which isn't @@ -39,11 +99,17 @@ # Apply Patches patchFlags = [ "-p1" ]; - patches = (oldAttrs.patches or [ ]) ++ [ - (pkgs.fetchpatch { - name = "mtp.patch"; - url = "https://github.com/ggml-org/llama.cpp/pull/22673.patch"; - hash = "sha256-HqpchhOpxuw5mY4a/OCWGDr2Y32rC4FeOHuhaVt+mvY="; - }) - ]; + patches = (oldAttrs.patches or [ ]) ++ [ mtpPatch ]; + + # Drop pre-built WebUI assets into tools/server/public/ so cmake's + # Priority 1 path picks them up and skips the HF Bucket fetch. + postPatch = '' + ${oldAttrs.postPatch or ""} + mkdir -p tools/server/public + cp ${webui}/* tools/server/public/ + ''; + + # Expose the WebUI sub-derivation so it can be built/tested in isolation: + # nix build .#llama-cpp.webui --builders '' + passthru = (oldAttrs.passthru or { }) // { inherit webui; }; })