diff --git a/.gitignore b/.gitignore index 0731c5a..2837d05 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ .DS_Store -rke2-token +_scratch diff --git a/README.md b/README.md index 9b2a7e4..ba7cf92 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,20 @@ -# Deploy NixOS +# Description -## Copy Config +This repository contains the configuration for multiple machines, as well as my home / IDE config (home-manager). + +## Home Manager + +Utilizing [Home Manager](https://nix-community.github.io/home-manager/) + +## NixOS + +### Copy Config ```bash scp -r * root@10.10.10.10:/etc/nixos ``` -## Partition Drives +### Partition Drives ```bash # Validate Disk @@ -21,7 +29,7 @@ sudo nix \ --flake /etc/nixos#lin-va-rke1 ``` -## Install NixOS +### Install NixOS ```bash # Install @@ -31,13 +39,13 @@ sudo nixos-install --flake /etc/nixos#lin-va-rke1 sudo reboot ``` -## Copy Config Back to Host +### Copy Config Back to Host ```bash scp -r * nixos@10.0.20.201:/etc/nixos ``` -## Rebuild NixOS +### Rebuild NixOS ```bash sudo nixos-rebuild switch @@ -53,13 +61,13 @@ sudo nixos-install --flake /etc/nixos#lin-va-rke1 cat /var/lib/rancher/rke2/server/node-token # Deploy Following Nodes -echo "" > rke2-token +echo "" > ./_scratch/rke2-token sudo nixos-install --flake /etc/nixos#lin-va-rke2 ``` -## Notes +### Notes -## Kasten Port Forward +### Kasten Port Forward ```bash kubectl port-forward -n kasten svc/gateway 8000:80 diff --git a/flake.nix b/flake.nix index 4bf4ccd..a081998 100644 --- a/flake.nix +++ b/flake.nix @@ -35,11 +35,15 @@ systemConfig = ./hosts/rke2.nix; moduleConfig = { hostName = "lin-va-rke1"; - mainDiskID = "/dev/disk/by-id/ata-VBOX_HARDDISK_VB0af7d668-04b70404"; - dataDiskID = "/dev/disk/by-id/ata-VBOX_HARDDISK_VBcd9425b8-d666f9b8"; + mainDiskID = "/dev/xvda"; + + democraticConfig = { + apiKeyFile = ./_scratch/truenas-api; + sshKeyFile = ./_scratch/truenas-ssh; + }; networkConfig = { - interface = "enp0s3"; + interface = "enX0"; address = "10.0.20.201"; defaultGateway = "10.0.20.254"; nameservers = [ "10.0.20.254" ]; diff --git a/hosts/rke2-ceph.nix b/hosts/rke2-ceph.nix index de2013a..e83c0ea 100644 --- a/hosts/rke2-ceph.nix +++ b/hosts/rke2-ceph.nix @@ -106,7 +106,7 @@ # RKE2 Join Token environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") { - source = ../rke2-token; + source = ../_scratch/rke2-token; mode = "0600"; user = "root"; group = "root"; diff --git a/hosts/rke2-longhorn.nix b/hosts/rke2-longhorn.nix new file mode 100644 index 0000000..e63b601 --- /dev/null +++ b/hosts/rke2-longhorn.nix @@ -0,0 +1,185 @@ +{ config, pkgs, lib, ... }: + +{ + # Node Nix Config + options = { + dataDiskID = lib.mkOption { + type = lib.types.str; + description = "The device ID for the data disk"; + }; + serverAddr = lib.mkOption { + type = lib.types.str; + description = "The server to join"; + default = ""; + }; + networkConfig = lib.mkOption { + type = lib.types.submodule { + options = { + interface = lib.mkOption { + type = lib.types.str; + description = "Network interface name"; + example = "enp0s3"; + }; + address = lib.mkOption { + type = lib.types.str; + description = "Static IP address"; + example = "10.0.20.200"; + }; + defaultGateway = lib.mkOption { + type = lib.types.str; + description = "Default gateway IP"; + example = "10.0.20.254"; + }; + nameservers = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = "List of DNS servers"; + example = [ "10.0.20.254" "8.8.8.8" ]; + default = [ "8.8.8.8" "8.8.4.4" ]; + }; + }; + }; + description = "Network configuration"; + }; + }; + + config = { + # ---------------------------------------- + # ---------- Base Configuration ---------- + # ---------------------------------------- + + # Longhorn Requirements + boot.kernelModules = [ + "iscsi_tcp" + "dm_crypt" + ]; + + # Longhorn Data Disk + disko.devices = { + disk.longhorn = { + type = "disk"; + device = config.dataDiskID; + content = { + type = "gpt"; + partitions = { + longhorn = { + size = "100%"; + content = { + type = "filesystem"; + format = "xfs"; + mountpoint = "/storage/longhorn"; + mountOptions = [ "defaults" "nofail" ]; + extraArgs = [ "-d" "su=128k,sw=8" ]; + }; + }; + }; + }; + }; + }; + + # Network Configuration + networking = { + hostName = config.hostName; + networkmanager.enable = false; + + # Interface Configuration + inherit (config.networkConfig) defaultGateway nameservers; + interfaces.${config.networkConfig.interface}.ipv4.addresses = [{ + inherit (config.networkConfig) address; + prefixLength = 24; + }]; + + firewall = { + enable = true; + + allowedTCPPorts = [ + # RKE2 Ports - https://docs.rke2.io/install/requirements#networking + 6443 # Kubernetes API + 9345 # RKE2 supervisor API + 2379 # etcd Client Port + 2380 # etcd Peer Port + 2381 # etcd Metrics Port + 10250 # kubelet metrics + 9099 # Canal CNI health checks + + # iSCSI Port + 3260 + ]; + + allowedUDPPorts = [ + # RKE2 Ports - https://docs.rke2.io/install/requirements#networking + 8472 # Canal CNI with VXLAN + # 51820 # Canal CNI with WireGuard IPv4 (if using encryption) + # 51821 # Canal CNI with WireGuard IPv6 (if using encryption) + ]; + }; + }; + + # System Packages + environment.systemPackages = with pkgs; [ + htop + k9s + kubectl + kubernetes-helm + nfs-utils + openiscsi + tmux + vim + ]; + + # ---------------------------------------- + # ---------- RKE2 Configuration ---------- + # ---------------------------------------- + + # RKE2 Join Token + environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") { + source = ../_scratch/rke2-token; + mode = "0600"; + user = "root"; + group = "root"; + }; + + # Enable RKE2 + services.rke2 = { + enable = true; + role = "server"; + + disable = [ + # Disable - Utilizing Traefik + "rke2-ingress-nginx" + + # Disable - Utilizing Longhorn's Snapshot Controller + "rke2-snapshot-controller" + "rke2-snapshot-controller-crd" + "rke2-snapshot-validation-webhook" + ]; + } // lib.optionalAttrs (config.serverAddr != "") { + serverAddr = config.serverAddr; + tokenFile = "/etc/rancher/rke2/node-token"; + }; + + # Enable OpeniSCSI + services.openiscsi = { + enable = true; + name = "iqn.2025-01.${config.hostName}:initiator"; + }; + + # Bootstrap Kubernetes Manifests + system.activationScripts.k8s-manifests = { + deps = [ ]; + text = '' + mkdir -p /var/lib/rancher/rke2/server/manifests + + # Base Configs + cp ${../k8s/longhorn.yaml} /var/lib/rancher/rke2/server/manifests/longhorn-base.yaml + # cp ${../k8s/kasten.yaml} /var/lib/rancher/rke2/server/manifests/kasten-base.yaml + ''; + }; + + # Add Symlinks Expected by Longhorn + system.activationScripts.add-symlinks = '' + mkdir -p /usr/bin + ln -sf ${pkgs.openiscsi}/bin/iscsiadm /usr/bin/iscsiadm + ln -sf ${pkgs.openiscsi}/bin/iscsid /usr/bin/iscsid + ''; + }; +} diff --git a/hosts/rke2-openebs.nix b/hosts/rke2-openebs.nix index fc437f3..e6eaf92 100644 --- a/hosts/rke2-openebs.nix +++ b/hosts/rke2-openebs.nix @@ -109,7 +109,7 @@ # RKE2 Join Token environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") { - source = ../rke2-token; + source = ../_scratch/rke2-token; mode = "0600"; user = "root"; group = "root"; diff --git a/hosts/rke2.nix b/hosts/rke2.nix index ed7f42d..0a73f83 100644 --- a/hosts/rke2.nix +++ b/hosts/rke2.nix @@ -1,17 +1,37 @@ { config, pkgs, lib, ... }: +let + formatPrivateKey = indentLevel: value: + let + indent = lib.strings.fixedWidthString indentLevel " " ""; + indentedLines = lib.strings.concatMapStrings + (line: "${indent}${line}\n") + (lib.strings.splitString "\n" value); + in + "|\n${indentedLines}"; +in { # Node Nix Config options = { - dataDiskID = lib.mkOption { - type = lib.types.str; - description = "The device ID for the data disk"; - }; serverAddr = lib.mkOption { type = lib.types.str; description = "The server to join"; default = ""; }; + democraticConfig = lib.mkOption { + type = lib.types.submodule { + options = { + apiKeyFile = lib.mkOption { + type = lib.types.path; + description = "Path to file containing the TrueNAS API key"; + }; + sshKeyFile = lib.mkOption { + type = lib.types.path; + description = "Path to file containing the TrueNAS User SSH private key"; + }; + }; + }; + }; networkConfig = lib.mkOption { type = lib.types.submodule { options = { @@ -46,36 +66,25 @@ # ---------------------------------------- # ---------- Base Configuration ---------- # ---------------------------------------- - - # Longhorn Requirements - boot.kernelModules = [ - "iscsi_tcp" - "dm_crypt" - ]; - - # Longhorn Data Disk - disko.devices = { - disk.longhorn = { - type = "disk"; - device = config.dataDiskID; - content = { - type = "gpt"; - partitions = { - longhorn = { - size = "100%"; - content = { - type = "filesystem"; - format = "xfs"; - mountpoint = "/storage/longhorn"; - mountOptions = [ "defaults" "nofail" ]; - extraArgs = [ "-d" "su=128k,sw=8" ]; - }; - }; - }; - }; - }; + # Democratic Requirements + boot.initrd = { + availableKernelModules = [ "xen_blkfront" "xen_netfront" ]; + kernelModules = [ "xen_netfront" "xen_blkfront" ]; + supportedFilesystems = [ "ext4" "xenfs" ]; }; + boot.kernelModules = [ + # Xen VM Requirements + "xen_netfront" + "xen_blkfront" + "xenfs" + + # iSCSI & Multipath + "iscsi_tcp" + "dm_multipath" + "dm_round_robin" + ]; + # Network Configuration networking = { hostName = config.hostName; @@ -83,10 +92,13 @@ # Interface Configuration inherit (config.networkConfig) defaultGateway nameservers; - interfaces.${config.networkConfig.interface}.ipv4.addresses = [{ - inherit (config.networkConfig) address; - prefixLength = 24; - }]; + interfaces."${config.networkConfig.interface}" = { + mtu = 9000; + ipv4.addresses = [{ + address = config.networkConfig.address; + prefixLength = 24; + }]; + }; firewall = { enable = true; @@ -100,9 +112,6 @@ 2381 # etcd Metrics Port 10250 # kubelet metrics 9099 # Canal CNI health checks - - # iSCSI Port - 3260 ]; allowedUDPPorts = [ @@ -120,7 +129,6 @@ k9s kubectl kubernetes-helm - nfs-utils openiscsi tmux vim @@ -132,7 +140,7 @@ # RKE2 Join Token environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") { - source = ../rke2-token; + source = ../_scratch/rke2-token; mode = "0600"; user = "root"; group = "root"; @@ -147,22 +155,46 @@ # Disable - Utilizing Traefik "rke2-ingress-nginx" - # Disable - Utilizing Longhorn's Snapshot Controller - "rke2-snapshot-controller" - "rke2-snapshot-controller-crd" - "rke2-snapshot-validation-webhook" + # Disable + # "rke2-snapshot-controller" + # "rke2-snapshot-controller-crd" + # "rke2-snapshot-validation-webhook" ]; } // lib.optionalAttrs (config.serverAddr != "") { serverAddr = config.serverAddr; tokenFile = "/etc/rancher/rke2/node-token"; }; + # Enable Xe Guest Utilities + services.xe-guest-utilities.enable = true; + # Enable OpeniSCSI services.openiscsi = { enable = true; - name = "iqn.2025-01.${config.hostName}:initiator"; + name = "iqn.2025-02.${config.hostName}:initiator"; }; + # Enable Multipath + services.multipath = { + enable = true; + defaults = '' + defaults { + user_friendly_names yes + find_multipaths yes + } + ''; + pathGroups = [ ]; + }; + + time.timeZone = "UTC"; + + # Add Symlinks Expected by Democratic + system.activationScripts.add-symlinks = '' + mkdir -p /usr/bin + ln -sf ${pkgs.openiscsi}/bin/iscsiadm /usr/bin/iscsiadm + ln -sf ${pkgs.openiscsi}/bin/iscsid /usr/bin/iscsid + ''; + # Bootstrap Kubernetes Manifests system.activationScripts.k8s-manifests = { deps = [ ]; @@ -170,16 +202,14 @@ mkdir -p /var/lib/rancher/rke2/server/manifests # Base Configs - cp ${../k8s/longhorn.yaml} /var/lib/rancher/rke2/server/manifests/longhorn-base.yaml - # cp ${../k8s/kasten.yaml} /var/lib/rancher/rke2/server/manifests/kasten-base.yaml + cp ${pkgs.substituteAll { + src = ../k8s/democratic.yaml; + apiKey = lib.strings.removeSuffix "\n" (builtins.readFile config.democraticConfig.apiKeyFile); + privateKey = formatPrivateKey 12 (lib.strings.removeSuffix "\n" (builtins.readFile config.democraticConfig.sshKeyFile)); + }} /var/lib/rancher/rke2/server/manifests/democratic-base.yaml + + cp ${../k8s/kasten.yaml} /var/lib/rancher/rke2/server/manifests/kasten-base.yaml ''; }; - - # Add Symlinks Expected by Longhorn - system.activationScripts.add-symlinks = '' - mkdir -p /usr/bin - ln -sf ${pkgs.openiscsi}/bin/iscsiadm /usr/bin/iscsiadm - ln -sf ${pkgs.openiscsi}/bin/iscsid /usr/bin/iscsid - ''; }; } diff --git a/k8s/democratic.yaml b/k8s/democratic.yaml new file mode 100644 index 0000000..a6b5cf8 --- /dev/null +++ b/k8s/democratic.yaml @@ -0,0 +1,73 @@ +--- +# Namespace +apiVersion: v1 +kind: Namespace +metadata: + labels: + name: democratic-csi + name: democratic-csi + +--- +# HelmChart +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: democratic-csi + namespace: kube-system +spec: + repo: https://democratic-csi.github.io/charts/ + chart: democratic-csi + targetNamespace: democratic-csi + valuesContent: |- + csiDriver: + name: "org.democratic-csi.iscsi" + + storageClasses: + - name: truenas-iscsi + defaultClass: true + reclaimPolicy: Delete + volumeBindingMode: Immediate + allowVolumeExpansion: true + parameters: + fsType: xfs + + driver: + config: + driver: freenas-iscsi + instance_id: kube + httpConnection: + protocol: http + host: 10.0.20.138 + port: 80 + apiKey: @apiKey@ + apiVersion: 2 + sshConnection: + host: 10.0.20.138 + port: 22 + username: k8s-csi + privateKey: @privateKey@ + zfs: + cli: + sudoEnabled: true + paths: + zfs: /sbin/zfs + zpool: /sbin/zpool + sudo: /usr/bin/sudo + chroot: /usr/sbin/chroot + datasetParentName: KubeStorage/pv/iscsi/v + detachedSnapshotsDatasetParentName: KubeStorage/pv/iscsi/s + zvolEnableReservation: false + iscsi: + targetPortal: "10.0.20.138:3260" + targetPortals: [] + namePrefix: csi- + nameSuffix: "-cluster" + targetGroups: + - targetGroupPortalGroup: 1 + targetGroupInitiatorGroup: 1 + targetGroupAuthType: None + extentInsecureTpc: true + extentXenCompat: false + extentDisablePhysicalBlocksize: true + extentBlocksize: 4096 + extentAvailThreshold: 0 diff --git a/k8s/kasten.yaml b/k8s/kasten.yaml index ae38d63..0e4c3ec 100644 --- a/k8s/kasten.yaml +++ b/k8s/kasten.yaml @@ -32,9 +32,9 @@ spec: apiVersion: v1 kind: Namespace metadata: + name: kasten labels: name: kasten - name: kasten --- apiVersion: helm.cattle.io/v1 kind: HelmChart @@ -71,12 +71,12 @@ metadata: name: storage-class-rename namespace: kasten spec: - comment: Renames cstor-r1 to ceph-block-triple + comment: Renames cstor-r1 to truenas-iscsi transforms: - json: - op: replace path: /spec/storageClassName - value: ceph-block-triple + value: truenas-iscsi name: StorageClassRename subject: name: ""