xen + democratic csi

.gitignore

@@ -1,2 +1,2 @@
 .DS_Store
-rke2-token
+_scratch

README.md

@@ -1,12 +1,20 @@
-# Deploy NixOS
+# Description
 
-## Copy Config
+This repository contains the configuration for multiple machines, as well as my home / IDE config (home-manager).
+
+## Home Manager
+
+Utilizing [Home Manager](https://nix-community.github.io/home-manager/)
+
+## NixOS
+
+### Copy Config
 
 ```bash
 scp -r * root@10.10.10.10:/etc/nixos
 ```
 
-## Partition Drives
+### Partition Drives
 
 ```bash
 # Validate Disk
@@ -21,7 +29,7 @@ sudo nix \
     --flake /etc/nixos#lin-va-rke1
 ```
 
-## Install NixOS
+### Install NixOS
 
 ```bash
 # Install
@@ -31,13 +39,13 @@ sudo nixos-install --flake /etc/nixos#lin-va-rke1
 sudo reboot
 ```
 
-## Copy Config Back to Host
+### Copy Config Back to Host
 
 ```bash
 scp -r * nixos@10.0.20.201:/etc/nixos
 ```
 
-## Rebuild NixOS
+### Rebuild NixOS
 
 ```bash
 sudo nixos-rebuild switch
@@ -53,13 +61,13 @@ sudo nixos-install --flake /etc/nixos#lin-va-rke1
 cat /var/lib/rancher/rke2/server/node-token
 
 # Deploy Following Nodes
-echo "<TOKEN>" > rke2-token
+echo "<TOKEN>" > ./_scratch/rke2-token
 sudo nixos-install --flake /etc/nixos#lin-va-rke2
 ```
 
-## Notes
+### Notes
 
-## Kasten Port Forward
+### Kasten Port Forward
 
 ```bash
 kubectl port-forward -n kasten svc/gateway 8000:80

flake.nix

@@ -35,11 +35,15 @@
           systemConfig = ./hosts/rke2.nix;
           moduleConfig = {
             hostName = "lin-va-rke1";
-            mainDiskID = "/dev/disk/by-id/ata-VBOX_HARDDISK_VB0af7d668-04b70404";
+            mainDiskID = "/dev/xvda";
-            dataDiskID = "/dev/disk/by-id/ata-VBOX_HARDDISK_VBcd9425b8-d666f9b8";
+
+            democraticConfig = {
+              apiKeyFile = ./_scratch/truenas-api;
+              sshKeyFile = ./_scratch/truenas-ssh;
+            };
 
             networkConfig = {
-              interface = "enp0s3";
+              interface = "enX0";
               address = "10.0.20.201";
               defaultGateway = "10.0.20.254";
               nameservers = [ "10.0.20.254" ];

hosts/rke2-ceph.nix

@@ -106,7 +106,7 @@
 
     # RKE2 Join Token
     environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") {
-      source = ../rke2-token;
+      source = ../_scratch/rke2-token;
       mode = "0600";
       user = "root";
       group = "root";

hosts/rke2-longhorn.nix

@@ -0,0 +1,185 @@
+{ config, pkgs, lib, ... }:
+
+{
+  # Node Nix Config
+  options = {
+    dataDiskID = lib.mkOption {
+      type = lib.types.str;
+      description = "The device ID for the data disk";
+    };
+    serverAddr = lib.mkOption {
+      type = lib.types.str;
+      description = "The server to join";
+      default = "";
+    };
+    networkConfig = lib.mkOption {
+      type = lib.types.submodule {
+        options = {
+          interface = lib.mkOption {
+            type = lib.types.str;
+            description = "Network interface name";
+            example = "enp0s3";
+          };
+          address = lib.mkOption {
+            type = lib.types.str;
+            description = "Static IP address";
+            example = "10.0.20.200";
+          };
+          defaultGateway = lib.mkOption {
+            type = lib.types.str;
+            description = "Default gateway IP";
+            example = "10.0.20.254";
+          };
+          nameservers = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            description = "List of DNS servers";
+            example = [ "10.0.20.254" "8.8.8.8" ];
+            default = [ "8.8.8.8" "8.8.4.4" ];
+          };
+        };
+      };
+      description = "Network configuration";
+    };
+  };
+
+  config = {
+    # ----------------------------------------
+    # ---------- Base Configuration ----------
+    # ----------------------------------------
+
+    # Longhorn Requirements
+    boot.kernelModules = [
+      "iscsi_tcp"
+      "dm_crypt"
+    ];
+
+    # Longhorn Data Disk
+    disko.devices = {
+      disk.longhorn = {
+        type = "disk";
+        device = config.dataDiskID;
+        content = {
+          type = "gpt";
+          partitions = {
+            longhorn = {
+              size = "100%";
+              content = {
+                type = "filesystem";
+                format = "xfs";
+                mountpoint = "/storage/longhorn";
+                mountOptions = [ "defaults" "nofail" ];
+                extraArgs = [ "-d" "su=128k,sw=8" ];
+              };
+            };
+          };
+        };
+      };
+    };
+
+    # Network Configuration
+    networking = {
+      hostName = config.hostName;
+      networkmanager.enable = false;
+
+      # Interface Configuration
+      inherit (config.networkConfig) defaultGateway nameservers;
+      interfaces.${config.networkConfig.interface}.ipv4.addresses = [{
+        inherit (config.networkConfig) address;
+        prefixLength = 24;
+      }];
+
+      firewall = {
+        enable = true;
+
+        allowedTCPPorts = [
+          # RKE2 Ports - https://docs.rke2.io/install/requirements#networking
+          6443 # Kubernetes API
+          9345 # RKE2 supervisor API
+          2379 # etcd Client Port
+          2380 # etcd Peer Port
+          2381 # etcd Metrics Port
+          10250 # kubelet metrics
+          9099 # Canal CNI health checks
+
+          # iSCSI Port
+          3260
+        ];
+
+        allowedUDPPorts = [
+          # RKE2 Ports - https://docs.rke2.io/install/requirements#networking
+          8472 # Canal CNI with VXLAN
+          # 51820 # Canal CNI with WireGuard IPv4 (if using encryption)
+          # 51821 # Canal CNI with WireGuard IPv6 (if using encryption)
+        ];
+      };
+    };
+
+    # System Packages
+    environment.systemPackages = with pkgs; [
+      htop
+      k9s
+      kubectl
+      kubernetes-helm
+      nfs-utils
+      openiscsi
+      tmux
+      vim
+    ];
+
+    # ----------------------------------------
+    # ---------- RKE2 Configuration ----------
+    # ----------------------------------------
+
+    # RKE2 Join Token
+    environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") {
+      source = ../_scratch/rke2-token;
+      mode = "0600";
+      user = "root";
+      group = "root";
+    };
+
+    # Enable RKE2
+    services.rke2 = {
+      enable = true;
+      role = "server";
+
+      disable = [
+        # Disable - Utilizing Traefik
+        "rke2-ingress-nginx"
+
+        # Disable - Utilizing Longhorn's Snapshot Controller
+        "rke2-snapshot-controller"
+        "rke2-snapshot-controller-crd"
+        "rke2-snapshot-validation-webhook"
+      ];
+    } // lib.optionalAttrs (config.serverAddr != "") {
+      serverAddr = config.serverAddr;
+      tokenFile = "/etc/rancher/rke2/node-token";
+    };
+
+    # Enable OpeniSCSI
+    services.openiscsi = {
+      enable = true;
+      name = "iqn.2025-01.${config.hostName}:initiator";
+    };
+
+    # Bootstrap Kubernetes Manifests
+    system.activationScripts.k8s-manifests = {
+      deps = [ ];
+      text = ''
+        mkdir -p /var/lib/rancher/rke2/server/manifests
+
+        # Base Configs
+        cp ${../k8s/longhorn.yaml} /var/lib/rancher/rke2/server/manifests/longhorn-base.yaml
+        # cp ${../k8s/kasten.yaml} /var/lib/rancher/rke2/server/manifests/kasten-base.yaml
+      '';
+    };
+
+    # Add Symlinks Expected by Longhorn
+    system.activationScripts.add-symlinks = ''
+      mkdir -p /usr/bin
+      ln -sf ${pkgs.openiscsi}/bin/iscsiadm /usr/bin/iscsiadm
+      ln -sf ${pkgs.openiscsi}/bin/iscsid /usr/bin/iscsid
+    '';
+  };
+}

hosts/rke2-openebs.nix

@@ -109,7 +109,7 @@
 
     # RKE2 Join Token
     environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") {
-      source = ../rke2-token;
+      source = ../_scratch/rke2-token;
       mode = "0600";
       user = "root";
       group = "root";

hosts/rke2.nix

@@ -1,17 +1,37 @@
 { config, pkgs, lib, ... }:
 
+let
+  formatPrivateKey = indentLevel: value:
+    let
+      indent = lib.strings.fixedWidthString indentLevel " " "";
+      indentedLines = lib.strings.concatMapStrings
+        (line: "${indent}${line}\n")
+        (lib.strings.splitString "\n" value);
+    in
+    "|\n${indentedLines}";
+in
 {
   # Node Nix Config
   options = {
-    dataDiskID = lib.mkOption {
-      type = lib.types.str;
-      description = "The device ID for the data disk";
-    };
     serverAddr = lib.mkOption {
       type = lib.types.str;
       description = "The server to join";
       default = "";
     };
+    democraticConfig = lib.mkOption {
+      type = lib.types.submodule {
+        options = {
+          apiKeyFile = lib.mkOption {
+            type = lib.types.path;
+            description = "Path to file containing the TrueNAS API key";
+          };
+          sshKeyFile = lib.mkOption {
+            type = lib.types.path;
+            description = "Path to file containing the TrueNAS User SSH private key";
+          };
+        };
+      };
+    };
     networkConfig = lib.mkOption {
       type = lib.types.submodule {
         options = {
@@ -46,36 +66,25 @@
     # ----------------------------------------
     # ---------- Base Configuration ----------
     # ----------------------------------------
-
+    # Democratic Requirements
-    # Longhorn Requirements
+    boot.initrd = {
-    boot.kernelModules = [
+      availableKernelModules = [ "xen_blkfront" "xen_netfront" ];
-      "iscsi_tcp"
+      kernelModules = [ "xen_netfront" "xen_blkfront" ];
-      "dm_crypt"
+      supportedFilesystems = [ "ext4" "xenfs" ];
-    ];
-
-    # Longhorn Data Disk
-    disko.devices = {
-      disk.longhorn = {
-        type = "disk";
-        device = config.dataDiskID;
-        content = {
-          type = "gpt";
-          partitions = {
-            longhorn = {
-              size = "100%";
-              content = {
-                type = "filesystem";
-                format = "xfs";
-                mountpoint = "/storage/longhorn";
-                mountOptions = [ "defaults" "nofail" ];
-                extraArgs = [ "-d" "su=128k,sw=8" ];
-              };
-            };
-          };
-        };
-      };
     };
 
+    boot.kernelModules = [
+      # Xen VM Requirements
+      "xen_netfront"
+      "xen_blkfront"
+      "xenfs"
+
+      # iSCSI & Multipath
+      "iscsi_tcp"
+      "dm_multipath"
+      "dm_round_robin"
+    ];
+
     # Network Configuration
     networking = {
       hostName = config.hostName;
@@ -83,10 +92,13 @@
 
       # Interface Configuration
       inherit (config.networkConfig) defaultGateway nameservers;
-      interfaces.${config.networkConfig.interface}.ipv4.addresses = [{
+      interfaces."${config.networkConfig.interface}" = {
-        inherit (config.networkConfig) address;
+        mtu = 9000;
-        prefixLength = 24;
+        ipv4.addresses = [{
-      }];
+          address = config.networkConfig.address;
+          prefixLength = 24;
+        }];
+      };
 
       firewall = {
         enable = true;
@@ -100,9 +112,6 @@
           2381 # etcd Metrics Port
           10250 # kubelet metrics
           9099 # Canal CNI health checks
-
-          # iSCSI Port
-          3260
         ];
 
         allowedUDPPorts = [
@@ -120,7 +129,6 @@
       k9s
       kubectl
       kubernetes-helm
-      nfs-utils
       openiscsi
       tmux
       vim
@@ -132,7 +140,7 @@
 
     # RKE2 Join Token
     environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") {
-      source = ../rke2-token;
+      source = ../_scratch/rke2-token;
       mode = "0600";
       user = "root";
       group = "root";
@@ -147,22 +155,46 @@
         # Disable - Utilizing Traefik
         "rke2-ingress-nginx"
 
-        # Disable - Utilizing Longhorn's Snapshot Controller
+        # Disable
-        "rke2-snapshot-controller"
+        # "rke2-snapshot-controller"
-        "rke2-snapshot-controller-crd"
+        # "rke2-snapshot-controller-crd"
-        "rke2-snapshot-validation-webhook"
+        # "rke2-snapshot-validation-webhook"
       ];
     } // lib.optionalAttrs (config.serverAddr != "") {
       serverAddr = config.serverAddr;
       tokenFile = "/etc/rancher/rke2/node-token";
     };
 
+    # Enable Xe Guest Utilities
+    services.xe-guest-utilities.enable = true;
+
     # Enable OpeniSCSI
     services.openiscsi = {
       enable = true;
-      name = "iqn.2025-01.${config.hostName}:initiator";
+      name = "iqn.2025-02.${config.hostName}:initiator";
     };
 
+    # Enable Multipath
+    services.multipath = {
+      enable = true;
+      defaults = ''
+        defaults {
+            user_friendly_names yes
+            find_multipaths yes
+        }
+      '';
+      pathGroups = [ ];
+    };
+
+    time.timeZone = "UTC";
+
+    # Add Symlinks Expected by Democratic
+    system.activationScripts.add-symlinks = ''
+      mkdir -p /usr/bin
+      ln -sf ${pkgs.openiscsi}/bin/iscsiadm /usr/bin/iscsiadm
+      ln -sf ${pkgs.openiscsi}/bin/iscsid /usr/bin/iscsid
+    '';
+
     # Bootstrap Kubernetes Manifests
     system.activationScripts.k8s-manifests = {
       deps = [ ];
@@ -170,16 +202,14 @@
         mkdir -p /var/lib/rancher/rke2/server/manifests
 
         # Base Configs
-        cp ${../k8s/longhorn.yaml} /var/lib/rancher/rke2/server/manifests/longhorn-base.yaml
+        cp ${pkgs.substituteAll {
-        # cp ${../k8s/kasten.yaml} /var/lib/rancher/rke2/server/manifests/kasten-base.yaml
+          src = ../k8s/democratic.yaml;
+          apiKey = lib.strings.removeSuffix "\n" (builtins.readFile config.democraticConfig.apiKeyFile);
+          privateKey = formatPrivateKey 12 (lib.strings.removeSuffix "\n" (builtins.readFile config.democraticConfig.sshKeyFile));
+        }} /var/lib/rancher/rke2/server/manifests/democratic-base.yaml
+
+        cp ${../k8s/kasten.yaml} /var/lib/rancher/rke2/server/manifests/kasten-base.yaml
       '';
     };
-
-    # Add Symlinks Expected by Longhorn
-    system.activationScripts.add-symlinks = ''
-      mkdir -p /usr/bin
-      ln -sf ${pkgs.openiscsi}/bin/iscsiadm /usr/bin/iscsiadm
-      ln -sf ${pkgs.openiscsi}/bin/iscsid /usr/bin/iscsid
-    '';
   };
 }

k8s/democratic.yaml

@@ -0,0 +1,73 @@
+---
+# Namespace
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    name: democratic-csi
+  name: democratic-csi
+
+---
+# HelmChart
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: democratic-csi
+  namespace: kube-system
+spec:
+  repo: https://democratic-csi.github.io/charts/
+  chart: democratic-csi
+  targetNamespace: democratic-csi
+  valuesContent: |-
+    csiDriver:
+      name: "org.democratic-csi.iscsi"
+
+    storageClasses:
+    - name: truenas-iscsi
+      defaultClass: true
+      reclaimPolicy: Delete
+      volumeBindingMode: Immediate
+      allowVolumeExpansion: true
+      parameters:
+        fsType: xfs
+
+    driver:
+      config:
+        driver: freenas-iscsi
+        instance_id: kube
+        httpConnection:
+          protocol: http
+          host: 10.0.20.138
+          port: 80
+          apiKey: @apiKey@
+          apiVersion: 2
+        sshConnection:
+          host: 10.0.20.138
+          port: 22
+          username: k8s-csi
+          privateKey: @privateKey@
+        zfs:
+          cli:
+            sudoEnabled: true
+            paths:
+              zfs: /sbin/zfs
+              zpool: /sbin/zpool
+              sudo: /usr/bin/sudo
+              chroot: /usr/sbin/chroot
+          datasetParentName: KubeStorage/pv/iscsi/v
+          detachedSnapshotsDatasetParentName: KubeStorage/pv/iscsi/s
+          zvolEnableReservation: false
+        iscsi:
+          targetPortal: "10.0.20.138:3260"
+          targetPortals: []
+          namePrefix: csi-
+          nameSuffix: "-cluster"
+          targetGroups:
+            - targetGroupPortalGroup: 1
+              targetGroupInitiatorGroup: 1
+              targetGroupAuthType: None
+          extentInsecureTpc: true
+          extentXenCompat: false
+          extentDisablePhysicalBlocksize: true
+          extentBlocksize: 4096
+          extentAvailThreshold: 0

k8s/kasten.yaml

@@ -32,9 +32,9 @@ spec:
 apiVersion: v1
 kind: Namespace
 metadata:
+  name: kasten
   labels:
     name: kasten
-  name: kasten
 ---
 apiVersion: helm.cattle.io/v1
 kind: HelmChart
@@ -71,12 +71,12 @@ metadata:
   name: storage-class-rename
   namespace: kasten
 spec:
-  comment: Renames cstor-r1 to ceph-block-triple
+  comment: Renames cstor-r1 to truenas-iscsi
   transforms:
     - json:
         - op: replace
           path: /spec/storageClassName
-          value: ceph-block-triple
+          value: truenas-iscsi
       name: StorageClassRename
       subject:
         name: ""