chore: added llama-cpp and migrate office

This commit is contained in:
2025-09-05 22:48:19 -04:00
parent 9433abcaf4
commit 26f32d3225
7 changed files with 218 additions and 166 deletions

View File

@@ -1,175 +1,70 @@
{ config, pkgs, ... }:
{ namespace, pkgs, config, lib, ... }:
let
cuda-llama = (pkgs.llama-cpp.override {
cudaSupport = true;
}).overrideAttrs (oldAttrs: {
cmakeFlags = oldAttrs.cmakeFlags ++ [
"-DGGML_CUDA_ENABLE_UNIFIED_MEMORY=1"
# Disable CPU Instructions - Intel(R) Core(TM) i5-3570K CPU @ 3.40GHz
"-DLLAMA_FMA=OFF"
"-DLLAMA_AVX2=OFF"
"-DLLAMA_AVX512=OFF"
"-DGGML_FMA=OFF"
"-DGGML_AVX2=OFF"
"-DGGML_AVX512=OFF"
];
});
# Define Model Vars
modelDir = "/models";
# 7B
# modelName = "qwen2.5-coder-7b-q8_0.gguf";
# modelUrl = "https://huggingface.co/ggml-org/Qwen2.5-Coder-7B-Q8_0-GGUF/resolve/main/${modelName}?download=true";
# 3B
modelName = "qwen2.5-coder-3b-q8_0.gguf";
modelUrl = "https://huggingface.co/ggml-org/Qwen2.5-Coder-3B-Q8_0-GGUF/resolve/main/${modelName}?download=true";
modelPath = "${modelDir}/${modelName}";
inherit (lib.${namespace}) enabled;
cfg = config.${namespace}.user;
in
{
# Allow Nvidia & CUDA
system.stateVersion = "25.05";
time.timeZone = "America/New_York";
nixpkgs.config.allowUnfree = true;
# Enable Graphics
hardware.graphics = {
enable = true;
enable32Bit = true;
extraPackages = [ pkgs.cudatoolkit ];
};
# System Config
reichard = {
nix = enabled;
# Load Nvidia Driver Module
services.xserver.videoDrivers = [ "nvidia" ];
# Nvidia Package Configuration
hardware.nvidia = {
package = config.boot.kernelPackages.nvidiaPackages.stable;
modesetting.enable = true;
powerManagement.enable = true;
open = false;
nvidiaSettings = true;
};
# Networking Configuration
networking.firewall = {
enable = true;
allowedTCPPorts = [
1234 # RTL-TCP
8080 # LLama API
];
};
# RTL-SDR
hardware.rtl-sdr.enable = true;
systemd.services = {
# LLama Download Model
download-model = {
description = "Download Model";
wantedBy = [ "multi-user.target" ];
before = [ "llama-cpp.service" ];
path = [ pkgs.curl pkgs.coreutils ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
User = "root";
Group = "root";
system = {
boot = {
enable = true;
silentBoot = true;
};
disk = {
enable = true;
diskPath = "/dev/sda";
};
networking = {
enable = true;
useStatic = {
interface = "enp5s0";
address = "10.0.50.120";
defaultGateway = "10.0.50.254";
nameservers = [ "10.0.20.20" ];
};
};
script = ''
set -euo pipefail
if [ ! -f "${modelPath}" ]; then
mkdir -p "${modelDir}"
# Add -f flag to follow redirects and -L for location
# Add --fail flag to exit with error on HTTP errors
# Add -C - to resume interrupted downloads
curl -f -L -C - \
-H "Accept: application/octet-stream" \
--retry 3 \
--retry-delay 5 \
--max-time 1800 \
"${modelUrl}" \
-o "${modelPath}.tmp" && \
mv "${modelPath}.tmp" "${modelPath}"
fi
'';
};
# RTL-SDR TCP Server Service
rtl-tcp = {
description = "RTL-SDR TCP Server";
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
ExecStart = "${pkgs.rtl-sdr}/bin/rtl_tcp -a 0.0.0.0 -f 1090000000 -s 2400000";
Restart = "on-failure";
RestartSec = "10s";
User = "root";
Group = "root";
hardware = {
opengl = {
enable = true;
enableNvidia = true;
};
};
services = {
openssh = enabled;
llama-cpp = enabled;
rtl-tcp = enabled;
};
};
# Setup LLama API Service
systemd.services.llama-cpp = {
after = [ "download-model.service" ];
requires = [ "download-model.service" ];
};
# Enable LLama API
services.llama-cpp = {
enable = true;
host = "0.0.0.0";
package = cuda-llama;
model = modelPath;
port = 8080;
openFirewall = true;
# 7B
# extraFlags = [
# "-ngl"
# "99"
# "-fa"
# "-ub"
# "512"
# "-b"
# "512"
# "-dt"
# "0.1"
# "--ctx-size"
# "4096"
# "--cache-reuse"
# "256"
# ];
# 3B
extraFlags = [
"-ngl"
"99"
"-fa"
"-ub"
"1024"
"-b"
"1024"
"--ctx-size"
"0"
"--cache-reuse"
"256"
];
users.users.${cfg.name} = {
openssh = {
authorizedKeys.keys = [
# evanreichard@lin-va-mbp-personal
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILJJoyXQOv9cAjGUHrUcvsW7vY9W0PmuPMQSI9AMZvNY"
# evanreichard@mac-va-mbp-personal
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMWj6rd6uDtHj/gGozgIEgxho/vBKebgN5Kce/N6vQWV"
# evanreichard@lin-va-thinkpad
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAq5JQr/6WJMIHhR434nK95FrDmf2ApW2Ahd2+cBKwDz"
];
};
};
# System Packages
environment.systemPackages = with pkgs; [
htop
nvtopPackages.full
rtl-sdr
btop
git
tmux
vim
wget
];
}