evan/nix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

configure cluster

Browse Source
This commit is contained in:
Evan Reichard
2025-01-27 20:22:18 -05:00
parent 286ae5375c
commit 6ebd1bf202
17 changed files with 929 additions and 457 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
hosts/k3s.nix
View File

@@ -1,123 +0,0 @@
{ config, pkgs, ... }:
{
imports = [
../k8s
];
k8s.manifestsDir = "/var/lib/rancher/k3s/server/manifests";
# Enable Flakes
nix.settings.experimental-features = [ "nix-command" "flakes" ];
# System Configuration
boot.kernelModules = [ "nvme_tcp" ]; # OpenEBS Mayastor Requirement
boot.kernel.sysctl = {
"vm.nr_hugepages" = 1024;
};
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
boot.loader.efi.efiSysMountPoint = "/boot";
# Disk Configuration
disko.devices = {
disk = {
main = {
type = "disk";
content = {
type = "gpt";
partitions = {
boot = {
size = "512M";
type = "EF00"; # EFI
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot";
mountOptions = [ "umask=0077" ];
};
};
root = {
size = "100%";
content = {
type = "filesystem";
format = "ext4";
mountpoint = "/";
};
};
};
};
};
};
};
# Network Configuration
networking = {
networkmanager.enable = true;
firewall = {
enable = true;
# Single Node Required Ports
allowedTCPPorts = [ 6443 ];
# Multi Node Required Ports
# allowedTCPPorts = [ 6443 2379 2380 10250 ];
# allowedUDPPorts = [ 8472 ];
};
};
# Enable K3s
services.k3s = {
enable = true;
role = "server";
extraFlags = toString [
"--disable=traefik" # Should we enable?
"--disable=servicelb"
];
};
# Enable SSH Server
services.openssh = {
enable = true;
settings = {
PasswordAuthentication = false; # Disable Password Login
PermitRootLogin = "prohibit-password"; # Disable Password Login
};
};
# User Configuration
users.users.root = {
openssh.authorizedKeys.keys = [
"ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEA8P84lWL/p13ZBFNwITm/dLWWL8s9pVmdOImM5gaJAiTLY+DheUvG6YsveB2/5STseiJ34g7Na9TW1mtTLL8zDqPvj3NbprQiYlLJKMbCk6dtfdD4nLMHl8B48e1h699XiZDp2/c+jJb0MkLOFrps+FbPqt7pFt1Pj29tFy8BCg0LGndu6KO+HqYS+aM5tp5hZESo1RReiJ8aHsu5X7wW46brN4gfyyu+8X4etSZAB9raWqlln9NKK7G6as6X+uPypvSjYGSTC8TSePV1iTPwOxPk2+1xBsK7EBLg3jNrrYaiXLnZvBOOhm11JmHzqEJ6386FfQO+0r4iDVxmvi+ojw== rsa-key-20141114"
];
hashedPassword = null; # Disable Password Login
};
# System Packages
environment.systemPackages = with pkgs; [
k9s
kubectl
kubernetes-helm
nfs-utils
vim
];
# Enable Container Features
virtualisation = {
docker.enable = false;
containerd = {
enable = true;
settings = {
version = 2;
plugins."io.containerd.grpc.v1.cri" = {
containerd.runtimes.runc = {
runtime_type = "io.containerd.runc.v2";
};
};
};
};
};
# System State Version
system.stateVersion = "24.11";
}

101
hosts/llama-server.nix
View File

@@ -5,7 +5,7 @@ let
cudaSupport = true;
}).overrideAttrs (oldAttrs: {
cmakeFlags = oldAttrs.cmakeFlags ++ [
"-DGGML_CUDA_ENABLE_UNIFIED_MEMORY=ON"
"-DGGML_CUDA_ENABLE_UNIFIED_MEMORY=1"
# Disable CPU Instructions - Intel(R) Core(TM) i5-3570K CPU @ 3.40GHz
"-DLLAMA_FMA=OFF"
@@ -19,20 +19,19 @@ let
# Define Model Vars
modelDir = "/models";
modelName = "qwen2.5-coder-7b-q8_0.gguf";
# 7B
# modelName = "qwen2.5-coder-7b-q8_0.gguf";
# modelUrl = "https://huggingface.co/ggml-org/Qwen2.5-Coder-7B-Q8_0-GGUF/resolve/main/${modelName}?download=true";
# 3B
modelName = "qwen2.5-coder-3b-q8_0.gguf";
modelUrl = "https://huggingface.co/ggml-org/Qwen2.5-Coder-3B-Q8_0-GGUF/resolve/main/${modelName}?download=true";
modelPath = "${modelDir}/${modelName}";
modelUrl = "https://huggingface.co/ggml-org/Qwen2.5-Coder-7B-Q8_0-GGUF/resolve/main/${modelName}?download=true";
in
{
# Enable Flakes
nix.settings.experimental-features = [ "nix-command" "flakes" ];
# System Configuration
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
boot.loader.efi.efiSysMountPoint = "/boot";
# Allow Nvidia & CUDA
nixpkgs.config.allowUnfree = true;
@@ -55,39 +54,6 @@ in
nvidiaSettings = true;
};
# Disk Configuration
disko.devices = {
disk = {
main = {
type = "disk";
content = {
type = "gpt";
partitions = {
boot = {
size = "512M";
type = "EF00"; # EFI
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot";
mountOptions = [ "umask=0077" ];
};
};
root = {
size = "100%";
content = {
type = "filesystem";
format = "ext4";
mountpoint = "/";
};
};
};
};
};
};
};
# Network Configuration
networking.networkmanager.enable = true;
@@ -138,40 +104,40 @@ in
model = modelPath;
port = 8080;
openFirewall = true;
# 7B
# extraFlags = [
# "-ngl"
# "99"
# "-fa"
# "-ub"
# "512"
# "-b"
# "512"
# "-dt"
# "0.1"
# "--ctx-size"
# "4096"
# "--cache-reuse"
# "256"
# ];
# 3B
extraFlags = [
"-ngl"
"99"
"-fa"
"-ub"
"512"
"1024"
"-b"
"512"
"-dt"
"0.1"
"1024"
"--ctx-size"
"4096"
"0"
"--cache-reuse"
"256"
];
};
# Enable SSH Server
services.openssh = {
enable = true;
settings = {
PasswordAuthentication = false; # Disable Password Login
PermitRootLogin = "prohibit-password"; # Disable Password Login
};
};
# User Configuration
users.users.root = {
openssh.authorizedKeys.keys = [
"ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEA8P84lWL/p13ZBFNwITm/dLWWL8s9pVmdOImM5gaJAiTLY+DheUvG6YsveB2/5STseiJ34g7Na9TW1mtTLL8zDqPvj3NbprQiYlLJKMbCk6dtfdD4nLMHl8B48e1h699XiZDp2/c+jJb0MkLOFrps+FbPqt7pFt1Pj29tFy8BCg0LGndu6KO+HqYS+aM5tp5hZESo1RReiJ8aHsu5X7wW46brN4gfyyu+8X4etSZAB9raWqlln9NKK7G6as6X+uPypvSjYGSTC8TSePV1iTPwOxPk2+1xBsK7EBLg3jNrrYaiXLnZvBOOhm11JmHzqEJ6386FfQO+0r4iDVxmvi+ojw== rsa-key-20141114"
];
hashedPassword = null; # Disable Password Login
};
# System Packages
environment.systemPackages = with pkgs; [
htop
@@ -180,7 +146,4 @@ in
vim
wget
];
# System State Version
system.stateVersion = "24.11";
}

147
hosts/rke2-ceph.nix Normal file
View File

@@ -0,0 +1,147 @@
{ config, pkgs, lib, ... }:
{
# Node Nix Config
options = {
dataDiskID = lib.mkOption {
type = lib.types.str;
description = "The device ID for the data disk";
};
serverAddr = lib.mkOption {
type = lib.types.str;
description = "The server to join";
default = "";
};
networkConfig = lib.mkOption {
type = lib.types.submodule {
options = {
interface = lib.mkOption {
type = lib.types.str;
description = "Network interface name";
example = "enp0s3";
};
address = lib.mkOption {
type = lib.types.str;
description = "Static IP address";
example = "10.0.20.200";
};
defaultGateway = lib.mkOption {
type = lib.types.str;
description = "Default gateway IP";
example = "10.0.20.254";
};
nameservers = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "List of DNS servers";
example = [ "10.0.20.254" "8.8.8.8" ];
default = [ "8.8.8.8" "8.8.4.4" ];
};
};
};
description = "Network configuration";
};
};
config = {
# ----------------------------------------
# ---------- Base Configuration ----------
# ----------------------------------------
# Ceph Requirements
boot.kernelModules = [ "rbd" ];
# Network Configuration
networking = {
hostName = config.hostName;
networkmanager.enable = false;
# Interface Configuration
inherit (config.networkConfig) defaultGateway nameservers;
interfaces.${config.networkConfig.interface}.ipv4.addresses = [{
inherit (config.networkConfig) address;
prefixLength = 24;
}];
firewall = {
enable = true;
allowedTCPPorts = [
# RKE2 Ports - https://docs.rke2.io/install/requirements#networking
6443 # Kubernetes API
9345 # RKE2 supervisor API
2379 # etcd Client Port
2380 # etcd Peer Port
2381 # etcd Metrics Port
10250 # kubelet metrics
9099 # Canal CNI health checks
# Ceph Ports
3300 # Ceph MON daemon
6789 # Ceph MON service
] ++ lib.range 6800 7300; # Ceph OSD range
allowedUDPPorts = [
# RKE2 Ports - https://docs.rke2.io/install/requirements#networking
8472 # Canal CNI with VXLAN
# 51820 # Canal CNI with WireGuard IPv4 (if using encryption)
# 51821 # Canal CNI with WireGuard IPv6 (if using encryption)
];
};
};
# System Packages
environment.systemPackages = with pkgs; [
htop
k9s
kubectl
kubernetes-helm
nfs-utils
tmux
vim
];
# ----------------------------------------
# ---------- RKE2 Configuration ----------
# ----------------------------------------
# RKE2 Join Token
environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") {
source = ../rke2-token;
mode = "0600";
user = "root";
group = "root";
};
# Enable RKE2
services.rke2 = {
enable = true;
role = "server";
disable = [
# Disable - Utilizing Traefik
"rke2-ingress-nginx"
# Distable - Utilizing OpenEBS's Snapshot Controller
"rke2-snapshot-controller"
"rke2-snapshot-controller-crd"
"rke2-snapshot-validation-webhook"
];
} // lib.optionalAttrs (config.serverAddr != "") {
serverAddr = config.serverAddr;
tokenFile = "/etc/rancher/rke2/node-token";
};
# Bootstrap Kubernetes Manifests
system.activationScripts.k8s-manifests = {
deps = [ ];
text = ''
mkdir -p /var/lib/rancher/rke2/server/manifests
# Base Configs
cp ${../k8s/ceph.yaml} /var/lib/rancher/rke2/server/manifests/ceph-base.yaml
cp ${../k8s/kasten.yaml} /var/lib/rancher/rke2/server/manifests/kasten-base.yaml
'';
};
};
}

162
hosts/rke2-openebs.nix Normal file
View File

@@ -0,0 +1,162 @@
{ config, pkgs, lib, ... }:
{
# Node Nix Config
options = {
dataDiskID = lib.mkOption {
type = lib.types.str;
description = "The device ID for the data disk";
};
serverAddr = lib.mkOption {
type = lib.types.str;
description = "The server to join";
default = "";
};
networkConfig = lib.mkOption {
type = lib.types.submodule {
options = {
interface = lib.mkOption {
type = lib.types.str;
description = "Network interface name";
example = "enp0s3";
};
address = lib.mkOption {
type = lib.types.str;
description = "Static IP address";
example = "10.0.20.200";
};
defaultGateway = lib.mkOption {
type = lib.types.str;
description = "Default gateway IP";
example = "10.0.20.254";
};
nameservers = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "List of DNS servers";
example = [ "10.0.20.254" "8.8.8.8" ];
default = [ "8.8.8.8" "8.8.4.4" ];
};
};
};
description = "Network configuration";
};
};
config = {
# ----------------------------------------
# ---------- Base Configuration ----------
# ----------------------------------------
# OpenEBS Mayastor Requirements
boot.kernelModules = [ "nvme_tcp" ];
boot.kernel.sysctl = {
"vm.nr_hugepages" = 1024;
};
# Network Configuration
networking = {
hostName = config.hostName;
networkmanager.enable = false;
# Interface Configuration
inherit (config.networkConfig) defaultGateway nameservers;
interfaces.${config.networkConfig.interface}.ipv4.addresses = [{
inherit (config.networkConfig) address;
prefixLength = 24;
}];
firewall = {
enable = true;
allowedTCPPorts = [
# RKE2 Ports - https://docs.rke2.io/install/requirements#networking
6443 # Kubernetes API
9345 # RKE2 supervisor API
2379 # etcd Client Port
2380 # etcd Peer Port
2381 # etcd Metrics Port
10250 # kubelet metrics
9099 # Canal CNI health checks
# OpenEBS Mayastor - https://openebs.io/docs/user-guides/replicated-storage-user-guide/replicated-pv-mayastor/rs-installation#network-requirements
10124 # REST API
8420 # NVMf
4421 # NVMf
];
allowedUDPPorts = [
# RKE2 Ports - https://docs.rke2.io/install/requirements#networking
8472 # Canal CNI with VXLAN
# 51820 # Canal CNI with WireGuard IPv4 (if using encryption)
# 51821 # Canal CNI with WireGuard IPv6 (if using encryption)
];
};
};
# System Packages
environment.systemPackages = with pkgs; [
htop
k9s
kubectl
kubernetes-helm
nfs-utils
vim
];
# ----------------------------------------
# ---------- RKE2 Configuration ----------
# ----------------------------------------
# RKE2 Join Token
environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") {
source = ../rke2-token;
mode = "0600";
user = "root";
group = "root";
};
# Enable RKE2
services.rke2 = {
enable = true;
role = "server";
disable = [
# Disable - Utilizing Traefik
"rke2-ingress-nginx"
# Distable - Utilizing OpenEBS's Snapshot Controller
"rke2-snapshot-controller"
"rke2-snapshot-controller-crd"
"rke2-snapshot-validation-webhook"
];
# OpenEBS Scheduleable
nodeLabel = [
"openebs.io/engine=mayastor"
];
} // lib.optionalAttrs (config.serverAddr != "") {
serverAddr = config.serverAddr;
tokenFile = "/etc/rancher/rke2/node-token";
};
# Bootstrap Kubernetes Manifests
system.activationScripts.k8s-manifests = {
deps = [ ];
text = ''
mkdir -p /var/lib/rancher/rke2/server/manifests
# Base Configs
cp ${../k8s/openebs.yaml} /var/lib/rancher/rke2/server/manifests/openebs-base.yaml
cp ${../k8s/kasten.yaml} /var/lib/rancher/rke2/server/manifests/kasten-base.yaml
# OpenEBS Disk Pool
cp ${pkgs.substituteAll {
src = ../k8s/openebs-disk-pool.yaml;
hostName = config.hostName;
dataDiskID = config.dataDiskID;
}} /var/lib/rancher/rke2/server/manifests/openebs-disk-pool-${config.hostName}.yaml
'';
};
};
}

278
hosts/rke2.nix
View File

@@ -1,147 +1,185 @@
{ config, pkgs, ... }:
{ config, pkgs, lib, ... }:
{
imports = [
../k8s
];
k8s.manifestsDir = "/var/lib/rancher/rke2/server/manifests";
# Enable Flakes
nix.settings.experimental-features = [ "nix-command" "flakes" ];
# System Configuration
boot.kernelModules = [ "nvme_tcp" ]; # OpenEBS Mayastor Requirement
boot.kernel.sysctl = {
"vm.nr_hugepages" = 1024;
# Node Nix Config
options = {
dataDiskID = lib.mkOption {
type = lib.types.str;
description = "The device ID for the data disk";
};
serverAddr = lib.mkOption {
type = lib.types.str;
description = "The server to join";
default = "";
};
networkConfig = lib.mkOption {
type = lib.types.submodule {
options = {
interface = lib.mkOption {
type = lib.types.str;
description = "Network interface name";
example = "enp0s3";
};
address = lib.mkOption {
type = lib.types.str;
description = "Static IP address";
example = "10.0.20.200";
};
defaultGateway = lib.mkOption {
type = lib.types.str;
description = "Default gateway IP";
example = "10.0.20.254";
};
nameservers = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "List of DNS servers";
example = [ "10.0.20.254" "8.8.8.8" ];
default = [ "8.8.8.8" "8.8.4.4" ];
};
};
};
description = "Network configuration";
};
};
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
boot.loader.efi.efiSysMountPoint = "/boot";
# Disk Configuration
disko.devices = {
disk = {
main = {
config = {
# ----------------------------------------
# ---------- Base Configuration ----------
# ----------------------------------------
# Longhorn Requirements
boot.kernelModules = [
"iscsi_tcp"
"dm_crypt"
];
# Longhorn Data Disk
disko.devices = {
disk.longhorn = {
type = "disk";
device = config.dataDiskID;
content = {
type = "gpt";
partitions = {
boot = {
size = "512M";
type = "EF00"; # EFI
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot";
mountOptions = [ "umask=0077" ];
};
};
root = {
longhorn = {
size = "100%";
content = {
type = "filesystem";
format = "ext4";
mountpoint = "/";
format = "xfs";
mountpoint = "/storage/longhorn";
mountOptions = [ "defaults" "nofail" ];
extraArgs = [ "-d" "su=128k,sw=8" ];
};
};
};
};
};
};
};
# Network Configuration
networking = {
networkmanager.enable = true;
firewall = {
# Network Configuration
networking = {
hostName = config.hostName;
networkmanager.enable = false;
# Interface Configuration
inherit (config.networkConfig) defaultGateway nameservers;
interfaces.${config.networkConfig.interface}.ipv4.addresses = [{
inherit (config.networkConfig) address;
prefixLength = 24;
}];
firewall = {
enable = true;
allowedTCPPorts = [
# RKE2 Ports - https://docs.rke2.io/install/requirements#networking
6443 # Kubernetes API
9345 # RKE2 supervisor API
2379 # etcd Client Port
2380 # etcd Peer Port
2381 # etcd Metrics Port
10250 # kubelet metrics
9099 # Canal CNI health checks
# iSCSI Port
3260
];
allowedUDPPorts = [
# RKE2 Ports - https://docs.rke2.io/install/requirements#networking
8472 # Canal CNI with VXLAN
# 51820 # Canal CNI with WireGuard IPv4 (if using encryption)
# 51821 # Canal CNI with WireGuard IPv6 (if using encryption)
];
};
};
# System Packages
environment.systemPackages = with pkgs; [
htop
k9s
kubectl
kubernetes-helm
nfs-utils
openiscsi
tmux
vim
];
# ----------------------------------------
# ---------- RKE2 Configuration ----------
# ----------------------------------------
# RKE2 Join Token
environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") {
source = ../rke2-token;
mode = "0600";
user = "root";
group = "root";
};
# Enable RKE2
services.rke2 = {
enable = true;
role = "server";
# https://docs.rke2.io/install/requirements#networking
allowedTCPPorts = [
# K8s Control Plane
6443 # Kubernetes API
9345 # RKE2 supervisor API
2379 # etcd Client Port
2380 # etcd Peer Port
2381 # etcd Metrics Port
disable = [
# Disable - Utilizing Traefik
"rke2-ingress-nginx"
# K8s Node Communication
10250 # kubelet metrics
9099 # Canal CNI health checks
# OpenEBS Mayastor
10124 # Mayastor REST API
8420 # NVMf
4421 # NVMf
];
allowedUDPPorts = [
8472 # Canal CNI with VXLAN
# 51820 # Canal CNI with WireGuard IPv4 (if using encryption)
# 51821 # Canal CNI with WireGuard IPv6 (if using encryption)
# Disable - Utilizing Longhorn's Snapshot Controller
"rke2-snapshot-controller"
"rke2-snapshot-controller-crd"
"rke2-snapshot-validation-webhook"
];
} // lib.optionalAttrs (config.serverAddr != "") {
serverAddr = config.serverAddr;
tokenFile = "/etc/rancher/rke2/node-token";
};
};
# Enable RKE2
services.rke2 = {
enable = true;
disable = [
# Utilize Traefik
"rke2-ingress-nginx"
# Utilize OpenEBS's Snapshot Controller
"rke2-snapshot-controller"
"rke2-snapshot-controller-crd"
"rke2-snapshot-validation-webhook"
];
nodeLabel = [
"openebs.io/engine=mayastor"
];
role = "server";
# -------------------
# --- Server Node ---
# -------------------
# -------------------
# --- Worker Node ---
# -------------------
# role = "agent";
# serverAddr = "https://10.0.0.10:6443"
# tokenFile = "";
# agentTokenFile = "";
};
# Enable SSH Server
services.openssh = {
enable = true;
settings = {
PasswordAuthentication = false; # Disable Password Login
PermitRootLogin = "prohibit-password"; # Disable Password Login
# Enable OpeniSCSI
services.openiscsi = {
enable = true;
name = "iqn.2025-01.${config.hostName}:initiator";
};
# Bootstrap Kubernetes Manifests
system.activationScripts.k8s-manifests = {
deps = [ ];
text = ''
mkdir -p /var/lib/rancher/rke2/server/manifests
# Base Configs
cp ${../k8s/longhorn.yaml} /var/lib/rancher/rke2/server/manifests/longhorn-base.yaml
# cp ${../k8s/kasten.yaml} /var/lib/rancher/rke2/server/manifests/kasten-base.yaml
'';
};
# Add Symlinks Expected by Longhorn
system.activationScripts.add-symlinks = ''
mkdir -p /usr/bin
ln -sf ${pkgs.openiscsi}/bin/iscsiadm /usr/bin/iscsiadm
ln -sf ${pkgs.openiscsi}/bin/iscsid /usr/bin/iscsid
'';
};
# User Configuration
users.users.root = {
openssh.authorizedKeys.keys = [
"ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEA8P84lWL/p13ZBFNwITm/dLWWL8s9pVmdOImM5gaJAiTLY+DheUvG6YsveB2/5STseiJ34g7Na9TW1mtTLL8zDqPvj3NbprQiYlLJKMbCk6dtfdD4nLMHl8B48e1h699XiZDp2/c+jJb0MkLOFrps+FbPqt7pFt1Pj29tFy8BCg0LGndu6KO+HqYS+aM5tp5hZESo1RReiJ8aHsu5X7wW46brN4gfyyu+8X4etSZAB9raWqlln9NKK7G6as6X+uPypvSjYGSTC8TSePV1iTPwOxPk2+1xBsK7EBLg3jNrrYaiXLnZvBOOhm11JmHzqEJ6386FfQO+0r4iDVxmvi+ojw== rsa-key-20141114"
];
hashedPassword = null; # Disable Password Login
};
# System Packages
environment.systemPackages = with pkgs; [
htop
k9s
kubectl
kubernetes-helm
nfs-utils
vim
];
# System State Version
system.stateVersion = "24.11";
}