xen + democratic csi
This commit is contained in:
@@ -106,7 +106,7 @@
|
||||
|
||||
# RKE2 Join Token
|
||||
environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") {
|
||||
source = ../rke2-token;
|
||||
source = ../_scratch/rke2-token;
|
||||
mode = "0600";
|
||||
user = "root";
|
||||
group = "root";
|
||||
|
||||
185
hosts/rke2-longhorn.nix
Normal file
185
hosts/rke2-longhorn.nix
Normal file
@@ -0,0 +1,185 @@
|
||||
{ config, pkgs, lib, ... }:
|
||||
|
||||
{
|
||||
# Node Nix Config
|
||||
options = {
|
||||
dataDiskID = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "The device ID for the data disk";
|
||||
};
|
||||
serverAddr = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "The server to join";
|
||||
default = "";
|
||||
};
|
||||
networkConfig = lib.mkOption {
|
||||
type = lib.types.submodule {
|
||||
options = {
|
||||
interface = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Network interface name";
|
||||
example = "enp0s3";
|
||||
};
|
||||
address = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Static IP address";
|
||||
example = "10.0.20.200";
|
||||
};
|
||||
defaultGateway = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Default gateway IP";
|
||||
example = "10.0.20.254";
|
||||
};
|
||||
nameservers = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
description = "List of DNS servers";
|
||||
example = [ "10.0.20.254" "8.8.8.8" ];
|
||||
default = [ "8.8.8.8" "8.8.4.4" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
description = "Network configuration";
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
# ----------------------------------------
|
||||
# ---------- Base Configuration ----------
|
||||
# ----------------------------------------
|
||||
|
||||
# Longhorn Requirements
|
||||
boot.kernelModules = [
|
||||
"iscsi_tcp"
|
||||
"dm_crypt"
|
||||
];
|
||||
|
||||
# Longhorn Data Disk
|
||||
disko.devices = {
|
||||
disk.longhorn = {
|
||||
type = "disk";
|
||||
device = config.dataDiskID;
|
||||
content = {
|
||||
type = "gpt";
|
||||
partitions = {
|
||||
longhorn = {
|
||||
size = "100%";
|
||||
content = {
|
||||
type = "filesystem";
|
||||
format = "xfs";
|
||||
mountpoint = "/storage/longhorn";
|
||||
mountOptions = [ "defaults" "nofail" ];
|
||||
extraArgs = [ "-d" "su=128k,sw=8" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
# Network Configuration
|
||||
networking = {
|
||||
hostName = config.hostName;
|
||||
networkmanager.enable = false;
|
||||
|
||||
# Interface Configuration
|
||||
inherit (config.networkConfig) defaultGateway nameservers;
|
||||
interfaces.${config.networkConfig.interface}.ipv4.addresses = [{
|
||||
inherit (config.networkConfig) address;
|
||||
prefixLength = 24;
|
||||
}];
|
||||
|
||||
firewall = {
|
||||
enable = true;
|
||||
|
||||
allowedTCPPorts = [
|
||||
# RKE2 Ports - https://docs.rke2.io/install/requirements#networking
|
||||
6443 # Kubernetes API
|
||||
9345 # RKE2 supervisor API
|
||||
2379 # etcd Client Port
|
||||
2380 # etcd Peer Port
|
||||
2381 # etcd Metrics Port
|
||||
10250 # kubelet metrics
|
||||
9099 # Canal CNI health checks
|
||||
|
||||
# iSCSI Port
|
||||
3260
|
||||
];
|
||||
|
||||
allowedUDPPorts = [
|
||||
# RKE2 Ports - https://docs.rke2.io/install/requirements#networking
|
||||
8472 # Canal CNI with VXLAN
|
||||
# 51820 # Canal CNI with WireGuard IPv4 (if using encryption)
|
||||
# 51821 # Canal CNI with WireGuard IPv6 (if using encryption)
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
# System Packages
|
||||
environment.systemPackages = with pkgs; [
|
||||
htop
|
||||
k9s
|
||||
kubectl
|
||||
kubernetes-helm
|
||||
nfs-utils
|
||||
openiscsi
|
||||
tmux
|
||||
vim
|
||||
];
|
||||
|
||||
# ----------------------------------------
|
||||
# ---------- RKE2 Configuration ----------
|
||||
# ----------------------------------------
|
||||
|
||||
# RKE2 Join Token
|
||||
environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") {
|
||||
source = ../_scratch/rke2-token;
|
||||
mode = "0600";
|
||||
user = "root";
|
||||
group = "root";
|
||||
};
|
||||
|
||||
# Enable RKE2
|
||||
services.rke2 = {
|
||||
enable = true;
|
||||
role = "server";
|
||||
|
||||
disable = [
|
||||
# Disable - Utilizing Traefik
|
||||
"rke2-ingress-nginx"
|
||||
|
||||
# Disable - Utilizing Longhorn's Snapshot Controller
|
||||
"rke2-snapshot-controller"
|
||||
"rke2-snapshot-controller-crd"
|
||||
"rke2-snapshot-validation-webhook"
|
||||
];
|
||||
} // lib.optionalAttrs (config.serverAddr != "") {
|
||||
serverAddr = config.serverAddr;
|
||||
tokenFile = "/etc/rancher/rke2/node-token";
|
||||
};
|
||||
|
||||
# Enable OpeniSCSI
|
||||
services.openiscsi = {
|
||||
enable = true;
|
||||
name = "iqn.2025-01.${config.hostName}:initiator";
|
||||
};
|
||||
|
||||
# Bootstrap Kubernetes Manifests
|
||||
system.activationScripts.k8s-manifests = {
|
||||
deps = [ ];
|
||||
text = ''
|
||||
mkdir -p /var/lib/rancher/rke2/server/manifests
|
||||
|
||||
# Base Configs
|
||||
cp ${../k8s/longhorn.yaml} /var/lib/rancher/rke2/server/manifests/longhorn-base.yaml
|
||||
# cp ${../k8s/kasten.yaml} /var/lib/rancher/rke2/server/manifests/kasten-base.yaml
|
||||
'';
|
||||
};
|
||||
|
||||
# Add Symlinks Expected by Longhorn
|
||||
system.activationScripts.add-symlinks = ''
|
||||
mkdir -p /usr/bin
|
||||
ln -sf ${pkgs.openiscsi}/bin/iscsiadm /usr/bin/iscsiadm
|
||||
ln -sf ${pkgs.openiscsi}/bin/iscsid /usr/bin/iscsid
|
||||
'';
|
||||
};
|
||||
}
|
||||
@@ -109,7 +109,7 @@
|
||||
|
||||
# RKE2 Join Token
|
||||
environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") {
|
||||
source = ../rke2-token;
|
||||
source = ../_scratch/rke2-token;
|
||||
mode = "0600";
|
||||
user = "root";
|
||||
group = "root";
|
||||
|
||||
140
hosts/rke2.nix
140
hosts/rke2.nix
@@ -1,17 +1,37 @@
|
||||
{ config, pkgs, lib, ... }:
|
||||
|
||||
let
|
||||
formatPrivateKey = indentLevel: value:
|
||||
let
|
||||
indent = lib.strings.fixedWidthString indentLevel " " "";
|
||||
indentedLines = lib.strings.concatMapStrings
|
||||
(line: "${indent}${line}\n")
|
||||
(lib.strings.splitString "\n" value);
|
||||
in
|
||||
"|\n${indentedLines}";
|
||||
in
|
||||
{
|
||||
# Node Nix Config
|
||||
options = {
|
||||
dataDiskID = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "The device ID for the data disk";
|
||||
};
|
||||
serverAddr = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "The server to join";
|
||||
default = "";
|
||||
};
|
||||
democraticConfig = lib.mkOption {
|
||||
type = lib.types.submodule {
|
||||
options = {
|
||||
apiKeyFile = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
description = "Path to file containing the TrueNAS API key";
|
||||
};
|
||||
sshKeyFile = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
description = "Path to file containing the TrueNAS User SSH private key";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
networkConfig = lib.mkOption {
|
||||
type = lib.types.submodule {
|
||||
options = {
|
||||
@@ -46,36 +66,25 @@
|
||||
# ----------------------------------------
|
||||
# ---------- Base Configuration ----------
|
||||
# ----------------------------------------
|
||||
|
||||
# Longhorn Requirements
|
||||
boot.kernelModules = [
|
||||
"iscsi_tcp"
|
||||
"dm_crypt"
|
||||
];
|
||||
|
||||
# Longhorn Data Disk
|
||||
disko.devices = {
|
||||
disk.longhorn = {
|
||||
type = "disk";
|
||||
device = config.dataDiskID;
|
||||
content = {
|
||||
type = "gpt";
|
||||
partitions = {
|
||||
longhorn = {
|
||||
size = "100%";
|
||||
content = {
|
||||
type = "filesystem";
|
||||
format = "xfs";
|
||||
mountpoint = "/storage/longhorn";
|
||||
mountOptions = [ "defaults" "nofail" ];
|
||||
extraArgs = [ "-d" "su=128k,sw=8" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
# Democratic Requirements
|
||||
boot.initrd = {
|
||||
availableKernelModules = [ "xen_blkfront" "xen_netfront" ];
|
||||
kernelModules = [ "xen_netfront" "xen_blkfront" ];
|
||||
supportedFilesystems = [ "ext4" "xenfs" ];
|
||||
};
|
||||
|
||||
boot.kernelModules = [
|
||||
# Xen VM Requirements
|
||||
"xen_netfront"
|
||||
"xen_blkfront"
|
||||
"xenfs"
|
||||
|
||||
# iSCSI & Multipath
|
||||
"iscsi_tcp"
|
||||
"dm_multipath"
|
||||
"dm_round_robin"
|
||||
];
|
||||
|
||||
# Network Configuration
|
||||
networking = {
|
||||
hostName = config.hostName;
|
||||
@@ -83,10 +92,13 @@
|
||||
|
||||
# Interface Configuration
|
||||
inherit (config.networkConfig) defaultGateway nameservers;
|
||||
interfaces.${config.networkConfig.interface}.ipv4.addresses = [{
|
||||
inherit (config.networkConfig) address;
|
||||
prefixLength = 24;
|
||||
}];
|
||||
interfaces."${config.networkConfig.interface}" = {
|
||||
mtu = 9000;
|
||||
ipv4.addresses = [{
|
||||
address = config.networkConfig.address;
|
||||
prefixLength = 24;
|
||||
}];
|
||||
};
|
||||
|
||||
firewall = {
|
||||
enable = true;
|
||||
@@ -100,9 +112,6 @@
|
||||
2381 # etcd Metrics Port
|
||||
10250 # kubelet metrics
|
||||
9099 # Canal CNI health checks
|
||||
|
||||
# iSCSI Port
|
||||
3260
|
||||
];
|
||||
|
||||
allowedUDPPorts = [
|
||||
@@ -120,7 +129,6 @@
|
||||
k9s
|
||||
kubectl
|
||||
kubernetes-helm
|
||||
nfs-utils
|
||||
openiscsi
|
||||
tmux
|
||||
vim
|
||||
@@ -132,7 +140,7 @@
|
||||
|
||||
# RKE2 Join Token
|
||||
environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") {
|
||||
source = ../rke2-token;
|
||||
source = ../_scratch/rke2-token;
|
||||
mode = "0600";
|
||||
user = "root";
|
||||
group = "root";
|
||||
@@ -147,22 +155,46 @@
|
||||
# Disable - Utilizing Traefik
|
||||
"rke2-ingress-nginx"
|
||||
|
||||
# Disable - Utilizing Longhorn's Snapshot Controller
|
||||
"rke2-snapshot-controller"
|
||||
"rke2-snapshot-controller-crd"
|
||||
"rke2-snapshot-validation-webhook"
|
||||
# Disable
|
||||
# "rke2-snapshot-controller"
|
||||
# "rke2-snapshot-controller-crd"
|
||||
# "rke2-snapshot-validation-webhook"
|
||||
];
|
||||
} // lib.optionalAttrs (config.serverAddr != "") {
|
||||
serverAddr = config.serverAddr;
|
||||
tokenFile = "/etc/rancher/rke2/node-token";
|
||||
};
|
||||
|
||||
# Enable Xe Guest Utilities
|
||||
services.xe-guest-utilities.enable = true;
|
||||
|
||||
# Enable OpeniSCSI
|
||||
services.openiscsi = {
|
||||
enable = true;
|
||||
name = "iqn.2025-01.${config.hostName}:initiator";
|
||||
name = "iqn.2025-02.${config.hostName}:initiator";
|
||||
};
|
||||
|
||||
# Enable Multipath
|
||||
services.multipath = {
|
||||
enable = true;
|
||||
defaults = ''
|
||||
defaults {
|
||||
user_friendly_names yes
|
||||
find_multipaths yes
|
||||
}
|
||||
'';
|
||||
pathGroups = [ ];
|
||||
};
|
||||
|
||||
time.timeZone = "UTC";
|
||||
|
||||
# Add Symlinks Expected by Democratic
|
||||
system.activationScripts.add-symlinks = ''
|
||||
mkdir -p /usr/bin
|
||||
ln -sf ${pkgs.openiscsi}/bin/iscsiadm /usr/bin/iscsiadm
|
||||
ln -sf ${pkgs.openiscsi}/bin/iscsid /usr/bin/iscsid
|
||||
'';
|
||||
|
||||
# Bootstrap Kubernetes Manifests
|
||||
system.activationScripts.k8s-manifests = {
|
||||
deps = [ ];
|
||||
@@ -170,16 +202,14 @@
|
||||
mkdir -p /var/lib/rancher/rke2/server/manifests
|
||||
|
||||
# Base Configs
|
||||
cp ${../k8s/longhorn.yaml} /var/lib/rancher/rke2/server/manifests/longhorn-base.yaml
|
||||
# cp ${../k8s/kasten.yaml} /var/lib/rancher/rke2/server/manifests/kasten-base.yaml
|
||||
cp ${pkgs.substituteAll {
|
||||
src = ../k8s/democratic.yaml;
|
||||
apiKey = lib.strings.removeSuffix "\n" (builtins.readFile config.democraticConfig.apiKeyFile);
|
||||
privateKey = formatPrivateKey 12 (lib.strings.removeSuffix "\n" (builtins.readFile config.democraticConfig.sshKeyFile));
|
||||
}} /var/lib/rancher/rke2/server/manifests/democratic-base.yaml
|
||||
|
||||
cp ${../k8s/kasten.yaml} /var/lib/rancher/rke2/server/manifests/kasten-base.yaml
|
||||
'';
|
||||
};
|
||||
|
||||
# Add Symlinks Expected by Longhorn
|
||||
system.activationScripts.add-symlinks = ''
|
||||
mkdir -p /usr/bin
|
||||
ln -sf ${pkgs.openiscsi}/bin/iscsiadm /usr/bin/iscsiadm
|
||||
ln -sf ${pkgs.openiscsi}/bin/iscsid /usr/bin/iscsid
|
||||
'';
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user