nix/hosts/rke2.nix

{ config, pkgs, lib, ... }:

let
  formatPrivateKey = indentLevel: value:
    let
      indent = lib.strings.fixedWidthString indentLevel " " "";
      indentedLines = lib.strings.concatMapStrings
        (line: "${indent}${line}\n")
        (lib.strings.splitString "\n" value);
    in
    "|\n${indentedLines}";
in
{
  # Node Nix Config
  options = {
    serverAddr = lib.mkOption {
      type = lib.types.str;
      description = "The server to join";
      default = "";
    };
    democraticConfig = lib.mkOption {
      type = lib.types.submodule {
        options = {
          apiKeyFile = lib.mkOption {
            type = lib.types.path;
            description = "Path to file containing the TrueNAS API key";
          };
          sshKeyFile = lib.mkOption {
            type = lib.types.path;
            description = "Path to file containing the TrueNAS User SSH private key";
          };
        };
      };
    };
    networkConfig = lib.mkOption {
      type = lib.types.submodule {
        options = {
          interface = lib.mkOption {
            type = lib.types.str;
            description = "Network interface name";
            example = "enp0s3";
          };
          address = lib.mkOption {
            type = lib.types.str;
            description = "Static IP address";
            example = "10.0.20.200";
          };
          defaultGateway = lib.mkOption {
            type = lib.types.str;
            description = "Default gateway IP";
            example = "10.0.20.254";
          };
          nameservers = lib.mkOption {
            type = lib.types.listOf lib.types.str;
            description = "List of DNS servers";
            example = [ "10.0.20.254" "8.8.8.8" ];
            default = [ "8.8.8.8" "8.8.4.4" ];
          };
        };
      };
      description = "Network configuration";
    };
  };

  config = {
    # ----------------------------------------
    # ---------- Base Configuration ----------
    # ----------------------------------------
    # Democratic Requirements
    boot.initrd = {
      availableKernelModules = [ "xen_blkfront" "xen_netfront" ];
      kernelModules = [ "xen_netfront" "xen_blkfront" ];
      supportedFilesystems = [ "ext4" "xenfs" ];
    };

    boot.kernelModules = [
      # Xen VM Requirements
      "xen_netfront"
      "xen_blkfront"
      "xenfs"

      # iSCSI & Multipath
      "iscsi_tcp"
      "dm_multipath"
      "dm_round_robin"
    ];

    # Network Configuration
    networking = {
      hostName = config.hostName;
      networkmanager.enable = false;

      # Interface Configuration
      inherit (config.networkConfig) defaultGateway nameservers;
      interfaces."${config.networkConfig.interface}" = {
        mtu = 9000;
        ipv4.addresses = [{
          address = config.networkConfig.address;
          prefixLength = 24;
        }];
      };

      firewall = {
        enable = true;

        allowedTCPPorts = [
          # RKE2 Ports - https://docs.rke2.io/install/requirements#networking
          6443 # Kubernetes API
          9345 # RKE2 supervisor API
          2379 # etcd Client Port
          2380 # etcd Peer Port
          2381 # etcd Metrics Port
          10250 # kubelet metrics
          9099 # Canal CNI health checks
        ];

        allowedUDPPorts = [
          # RKE2 Ports - https://docs.rke2.io/install/requirements#networking
          8472 # Canal CNI with VXLAN
          # 51820 # Canal CNI with WireGuard IPv4 (if using encryption)
          # 51821 # Canal CNI with WireGuard IPv6 (if using encryption)
        ];
      };
    };

    # System Packages
    environment.systemPackages = with pkgs; [
      htop
      k9s
      kubectl
      kubernetes-helm
      openiscsi
      tmux
      vim
    ];

    # ----------------------------------------
    # ---------- RKE2 Configuration ----------
    # ----------------------------------------

    # RKE2 Join Token
    environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") {
      source = ../_scratch/rke2-token;
      mode = "0600";
      user = "root";
      group = "root";
    };

    # Enable RKE2
    services.rke2 = {
      enable = true;
      role = "server";

      disable = [
        # Disable - Utilizing Traefik
        "rke2-ingress-nginx"

        # Disable
        # "rke2-snapshot-controller"
        # "rke2-snapshot-controller-crd"
        # "rke2-snapshot-validation-webhook"
      ];
    } // lib.optionalAttrs (config.serverAddr != "") {
      serverAddr = config.serverAddr;
      tokenFile = "/etc/rancher/rke2/node-token";
    };

    # Enable Xe Guest Utilities
    services.xe-guest-utilities.enable = true;

    # Enable OpeniSCSI
    services.openiscsi = {
      enable = true;
      name = "iqn.2025-02.${config.hostName}:initiator";
    };

    # Enable Multipath
    services.multipath = {
      enable = true;
      defaults = ''
        defaults {
            user_friendly_names yes
            find_multipaths yes
        }
      '';
      pathGroups = [ ];
    };

    time.timeZone = "UTC";

    # Add Symlinks Expected by Democratic
    system.activationScripts.add-symlinks = ''
      mkdir -p /usr/bin
      ln -sf ${pkgs.openiscsi}/bin/iscsiadm /usr/bin/iscsiadm
      ln -sf ${pkgs.openiscsi}/bin/iscsid /usr/bin/iscsid
    '';

    # Bootstrap Kubernetes Manifests
    system.activationScripts.k8s-manifests = {
      deps = [ ];
      text = ''
        mkdir -p /var/lib/rancher/rke2/server/manifests

        # Base Configs
        cp ${pkgs.substituteAll {
          src = ../k8s/democratic.yaml;
          apiKey = lib.strings.removeSuffix "\n" (builtins.readFile config.democraticConfig.apiKeyFile);
          privateKey = formatPrivateKey 12 (lib.strings.removeSuffix "\n" (builtins.readFile config.democraticConfig.sshKeyFile));
        }} /var/lib/rancher/rke2/server/manifests/democratic-base.yaml

        cp ${../k8s/kasten.yaml} /var/lib/rancher/rke2/server/manifests/kasten-base.yaml
      '';
    };
  };
}