From 2cfa70f96aceb7683b157e893489c328e73894ae Mon Sep 17 00:00:00 2001 From: Evan Reichard Date: Wed, 26 Feb 2025 16:49:34 -0500 Subject: [PATCH] rke image build --- README.md | 20 +++++ flake.lock | 99 +++++++++++++++++++++++ flake.nix | 37 ++++++++- hosts/rke2-image.nix | 184 +++++++++++++++++++++++++++++++++++++++++++ k8s/democratic.yaml | 6 +- 5 files changed, 342 insertions(+), 4 deletions(-) create mode 100644 flake.lock create mode 100644 hosts/rke2-image.nix diff --git a/README.md b/README.md index ba7cf92..137dc96 100644 --- a/README.md +++ b/README.md @@ -70,5 +70,25 @@ sudo nixos-install --flake /etc/nixos#lin-va-rke2 ### Kasten Port Forward ```bash +# http://localhost:8000/k10/#/dashboard kubectl port-forward -n kasten svc/gateway 8000:80 ``` + +### Cloud Init + +``` +#cloud-config +hostname: rke-test +``` + +``` +network: + version: 1 + config: + - type: physical + name: enX0 + subnets: + - type: static + address: 10.0.50.5/24 + gateway: 10.0.50.254 +``` diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..879b4b4 --- /dev/null +++ b/flake.lock @@ -0,0 +1,99 @@ +{ + "nodes": { + "disko": { + "inputs": { + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1739841949, + "narHash": "sha256-lSOXdgW/1zi/SSu7xp71v+55D5Egz8ACv0STkj7fhbs=", + "owner": "nix-community", + "repo": "disko", + "rev": "15dbf8cebd8e2655a883b74547108e089f051bf0", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "disko", + "type": "github" + } + }, + "nixlib": { + "locked": { + "lastModified": 1736643958, + "narHash": "sha256-tmpqTSWVRJVhpvfSN9KXBvKEXplrwKnSZNAoNPf/S/s=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "1418bc28a52126761c02dd3d89b2d8ca0f521181", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "nixos-generators": { + "inputs": { + "nixlib": "nixlib", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1737057290, + "narHash": "sha256-3Pe0yKlCc7EOeq1X/aJVDH0CtNL+tIBm49vpepwL1MQ=", + "owner": "nix-community", + "repo": "nixos-generators", + "rev": "d002ce9b6e7eb467cd1c6bb9aef9c35d191b5453", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixos-generators", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1738136902, + "narHash": "sha256-pUvLijVGARw4u793APze3j6mU1Zwdtz7hGkGGkD87qw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9a5db3142ce450045840cc8d832b13b8a2018e0c", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1739758141, + "narHash": "sha256-uq6A2L7o1/tR6VfmYhZWoVAwb3gTy7j4Jx30MIrH0rE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c618e28f70257593de75a7044438efc1c1fc0791", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "disko": "disko", + "nixos-generators": "nixos-generators", + "nixpkgs": "nixpkgs_2" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix index a081998..1ca6003 100644 --- a/flake.nix +++ b/flake.nix @@ -4,9 +4,13 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11"; disko.url = "github:nix-community/disko"; + nixos-generators = { + url = "github:nix-community/nixos-generators"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; - outputs = { self, nixpkgs, disko }: + outputs = { self, nixpkgs, disko, nixos-generators }: let mkSystem = { systemConfig, moduleConfig }: nixpkgs.lib.nixosSystem { system = "x86_64-linux"; @@ -20,6 +24,16 @@ }; in { + packages.x86_64-linux = { + rke2-image = nixos-generators.nixosGenerate { + system = "x86_64-linux"; + format = "vmware"; + modules = [ + ./hosts/rke2-image.nix + ]; + }; + }; + nixosConfigurations = { # LLaMA C++ Server lin-va-llama1 = mkSystem { @@ -30,6 +44,27 @@ }; }; + # RKE2 Primary Server + lin-va-kube1 = mkSystem { + systemConfig = ./hosts/rke2.nix; + moduleConfig = { + hostName = "lin-va-kube1"; + mainDiskID = "/dev/xvda"; + + democraticConfig = { + apiKeyFile = ./_scratch/truenas-api; + sshKeyFile = ./_scratch/truenas-ssh; + }; + + networkConfig = { + interface = "enX0"; + address = "10.0.50.50"; + defaultGateway = "10.0.50.254"; + nameservers = [ "10.0.50.254" ]; + }; + }; + }; + # RKE2 Primary Server lin-va-rke1 = mkSystem { systemConfig = ./hosts/rke2.nix; diff --git a/hosts/rke2-image.nix b/hosts/rke2-image.nix new file mode 100644 index 0000000..f29a28d --- /dev/null +++ b/hosts/rke2-image.nix @@ -0,0 +1,184 @@ +{ pkgs, lib, modulesPath, ... }: +{ + imports = [ + (modulesPath + "/profiles/qemu-guest.nix") + ]; + config = { + # Basic System + system.stateVersion = "24.11"; + nix.settings.experimental-features = [ "nix-command" "flakes" ]; + time.timeZone = "UTC"; + + fileSystems."/" = { + device = "/dev/disk/by-label/nixos"; + fsType = "ext4"; + autoResize = true; + }; + + boot = { + initrd = { + availableKernelModules = [ + # Xen + "xen_blkfront" + "xen_netfront" + ]; + kernelModules = [ "xen_netfront" "xen_blkfront" ]; + supportedFilesystems = [ "ext4" "xenfs" ]; + }; + kernelModules = [ + # Xen VM Requirements + "xen_netfront" + "xen_blkfront" + "xenfs" + + # iSCSI & Multipath + "iscsi_tcp" + "dm_multipath" + "dm_round_robin" + ]; + }; + + # Network Configuration + networking = { + hostName = lib.mkForce ""; + useNetworkd = true; + useDHCP = false; + + firewall = { + enable = true; + + allowedTCPPorts = [ + # RKE2 Ports - https://docs.rke2.io/install/requirements#networking + 6443 # Kubernetes API + 9345 # RKE2 supervisor API + 2379 # etcd Client Port + 2380 # etcd Peer Port + 2381 # etcd Metrics Port + 10250 # kubelet metrics + 9099 # Canal CNI health checks + ]; + + allowedUDPPorts = [ + # RKE2 Ports - https://docs.rke2.io/install/requirements#networking + 8472 # Canal CNI with VXLAN + # 51820 # Canal CNI with WireGuard IPv4 (if using encryption) + # 51821 # Canal CNI with WireGuard IPv6 (if using encryption) + ]; + }; + }; + + services = { + # Enable Xen Guest Utilities + xe-guest-utilities.enable = true; + + # Enable iSCSI + openiscsi = { + enable = true; + name = "iqn.2025.placeholder:initiator"; # Overridden @ Runtime + }; + + # Enable Multipath + multipath = { + enable = true; + defaults = '' + defaults { + user_friendly_names yes + find_multipaths yes + } + ''; + pathGroups = [ ]; + }; + + # Cloud Init + cloud-init = { + enable = true; + network.enable = true; + settings = { + datasource_list = [ "NoCloud" ]; + preserve_hostname = false; + system_info.distro = "nixos"; + system_info.network.renderers = [ "networkd" ]; + }; + }; + + # Enable SSH + openssh = { + enable = true; + settings = { + PasswordAuthentication = false; + PermitRootLogin = "prohibit-password"; + }; + }; + + # Enable RKE2 + rke2 = { + enable = true; + disable = [ "rke2-ingress-nginx" ]; + }; + }; + + systemd.services = { + # RKE2 - Wait Cloud Init + rke2-server = { + after = [ "cloud-final.service" ]; + requires = [ "cloud-final.service" ]; + }; + + # Runtime iSCSI Initiator Setup + iscsi-initiator-setup = { + description = "Setup iSCSI Initiator Name"; + requires = [ "cloud-final.service" ]; + before = [ "iscsid.service" ]; + after = [ "cloud-final.service" ]; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + + path = [ pkgs.hostname pkgs.util-linux ]; + script = '' + mkdir -p /run/iscsi + echo "InitiatorName=iqn.2025.org.nixos:$(hostname)" > /run/iscsi/initiatorname.iscsi + mount --bind /run/iscsi/initiatorname.iscsi /etc/iscsi/initiatorname.iscsi + ''; + }; + }; + + # User Authorized Keys + users.users.root = { + openssh.authorizedKeys.keys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEA8P84lWL/p13ZBFNwITm/dLWWL8s9pVmdOImM5gaJAiTLY+DheUvG6YsveB2/5STseiJ34g7Na9TW1mtTLL8zDqPvj3NbprQiYlLJKMbCk6dtfdD4nLMHl8B48e1h699XiZDp2/c+jJb0MkLOFrps+FbPqt7pFt1Pj29tFy8BCg0LGndu6KO+HqYS+aM5tp5hZESo1RReiJ8aHsu5X7wW46brN4gfyyu+8X4etSZAB9raWqlln9NKK7G6as6X+uPypvSjYGSTC8TSePV1iTPwOxPk2+1xBsK7EBLg3jNrrYaiXLnZvBOOhm11JmHzqEJ6386FfQO+0r4iDVxmvi+ojw== rsa-key-20141114" + ]; + hashedPassword = null; + }; + + # Add Symlinks Expected by Democratic + system.activationScripts = { + iscsi-initiator = '' + # Democratic CSI Requirements + mkdir -p /usr/bin + ln -sf ${pkgs.openiscsi}/bin/iscsiadm /usr/bin/iscsiadm + ln -sf ${pkgs.openiscsi}/bin/iscsid /usr/bin/iscsid + ''; + }; + + # System Packages + environment = { + systemPackages = with pkgs; [ + htop + k9s + kubectl + kubernetes-helm + nfs-utils + openiscsi + tmux + vim + ]; + + # Don't Manage - Runtime Generation + etc."iscsi/initiatorname.iscsi".enable = false; + }; + }; +} diff --git a/k8s/democratic.yaml b/k8s/democratic.yaml index a6b5cf8..c15747b 100644 --- a/k8s/democratic.yaml +++ b/k8s/democratic.yaml @@ -37,12 +37,12 @@ spec: instance_id: kube httpConnection: protocol: http - host: 10.0.20.138 + host: 10.0.50.60 port: 80 apiKey: @apiKey@ apiVersion: 2 sshConnection: - host: 10.0.20.138 + host: 10.0.50.60 port: 22 username: k8s-csi privateKey: @privateKey@ @@ -58,7 +58,7 @@ spec: detachedSnapshotsDatasetParentName: KubeStorage/pv/iscsi/s zvolEnableReservation: false iscsi: - targetPortal: "10.0.20.138:3260" + targetPortal: "10.0.50.60:3260" targetPortals: [] namePrefix: csi- nameSuffix: "-cluster"