From 0c6060def2e28dc44190174913ea8b2f6694e546 Mon Sep 17 00:00:00 2001 From: Evan Reichard Date: Sun, 26 Jan 2025 17:07:18 -0500 Subject: [PATCH] rke2 master & add --- README.md | 23 ++++++++++++---- flake.nix | 60 +++++++++++++++++++++++++++++++++++++++--- hosts/rke2.nix | 2 +- k8s/.gitignore | 1 + k8s/config/kasten.yaml | 51 +++++++++++++++++++++++++++++++++++ k8s/default.nix | 5 ++++ 6 files changed, 133 insertions(+), 9 deletions(-) create mode 100644 k8s/.gitignore create mode 100644 k8s/config/kasten.yaml diff --git a/README.md b/README.md index d9e9414..3c7b940 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ## Copy Config ```bash -scp -r * nixos@10.10.10.10:/tmp/ +scp -r * root@10.10.10.10:/etc/nixos ``` ## Partition Drives @@ -17,21 +17,20 @@ sudo nix \ --experimental-features "nix-command flakes" \ run github:nix-community/disko -- \ --mode disko \ - --flake /tmp#lin-va-rke1 + --flake /etc/nixos#lin-va-rke1 ``` ## Install NixOS ```bash # Install -sudo nixos-install --flake /tmp#lin-va-llama1 -sudo nixos-install --flake /tmp#lin-va-rke1 +sudo nixos-install --flake /etc/nixos#lin-va-rke1 # Reboot sudo reboot ``` -## Copy Config to Host +## Copy Config Back to Host ```bash scp -r * nixos@10.10.10.10:/etc/nixos @@ -42,3 +41,17 @@ scp -r * nixos@10.10.10.10:/etc/nixos ```bash sudo nixos-rebuild switch ``` + +# Install Kubernetes (RKE2) + +``` +# Deploy First Node +sudo nixos-install --flake /etc/nixos#lin-va-rke1 + +# Reboot & Get Token +cat /var/lib/rancher/rke2/server/node-token + +# Deploy Following Nodes +echo "" > ./k8s/rke2-token +sudo nixos-install --flake /etc/nixos#lin-va-rke2 +``` diff --git a/flake.nix b/flake.nix index 911cfbd..9febd0d 100644 --- a/flake.nix +++ b/flake.nix @@ -34,7 +34,7 @@ ]; }; - # RKE2 Server + # RKE2 Primary Server nixosConfigurations.lin-va-rke1 = nixpkgs.lib.nixosSystem { system = "x86_64-linux"; modules = [ @@ -42,8 +42,62 @@ ./hosts/rke2.nix { networking.hostName = "lin-va-rke1"; - disko.devices.disk.main.device = "/dev/nvme0n1"; - k8s.diskPoolID = "/dev/disk/by-id/ata-INTEL_SSDSC2BW240A4_CVDA418201Z42403GN"; + + # Partitions + disko.devices.disk.main.device = "/dev/disk/by-id/ata-VBOX_HARDDISK_VB0af7d668-04b70404"; + k8s.diskPoolID = "/dev/disk/by-id/ata-VBOX_HARDDISK_VBcd9425b8-d666f9b8"; + } + ]; + }; + + # RKE2 Second Server + nixosConfigurations.lin-va-rke2 = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ + disko.nixosModules.disko + ./hosts/rke2.nix + { + networking.hostName = "lin-va-rke2"; + + # Partitions + disko.devices.disk.main.device = "/dev/disk/by-id/ata-VBOX_HARDDISK_VBf55aaccc-688cfd0d"; + k8s.diskPoolID = "/dev/disk/by-id/ata-VBOX_HARDDISK_VBfd391256-6e368424"; + + # Set RKE2 Join + services.rke2.serverAddr = "https://10.0.20.147:9345"; + services.rke2.tokenFile = "/etc/rancher/rke2/node-token"; + environment.etc."rancher/rke2/node-token" = { + source = ./k8s/rke2-token; + mode = "0600"; + user = "root"; + group = "root"; + }; + } + ]; + }; + + # RKE2 Third Server + nixosConfigurations.lin-va-rke3 = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ + disko.nixosModules.disko + ./hosts/rke2.nix + { + networking.hostName = "lin-va-rke3"; + + # Partitions + disko.devices.disk.main.device = "/dev/disk/by-id/ata-VBOX_HARDDISK_VBe9edacd5-ac4ed4fa"; + k8s.diskPoolID = "/dev/disk/by-id/ata-VBOX_HARDDISK_VBa1fc46d0-19380495"; + + # Set RKE2 Join + services.rke2.serverAddr = "https://10.0.20.147:9345"; + services.rke2.tokenFile = "/etc/rancher/rke2/node-token"; + environment.etc."rancher/rke2/node-token" = { + source = ./k8s/rke2-token; + mode = "0600"; + user = "root"; + group = "root"; + }; } ]; }; diff --git a/hosts/rke2.nix b/hosts/rke2.nix index ee26d54..bf24c68 100644 --- a/hosts/rke2.nix +++ b/hosts/rke2.nix @@ -96,10 +96,10 @@ "openebs.io/engine=mayastor" ]; + role = "server"; # ------------------- # --- Server Node --- # ------------------- - role = "server"; # ------------------- # --- Worker Node --- diff --git a/k8s/.gitignore b/k8s/.gitignore new file mode 100644 index 0000000..5dbb189 --- /dev/null +++ b/k8s/.gitignore @@ -0,0 +1 @@ +rke2-token diff --git a/k8s/config/kasten.yaml b/k8s/config/kasten.yaml new file mode 100644 index 0000000..644dcb8 --- /dev/null +++ b/k8s/config/kasten.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: va-unraid-backup-rw +spec: + capacity: + storage: 100Ti + accessModes: + - ReadWriteMany + storageClassName: "va-unraid-backup-rw" + persistentVolumeReclaimPolicy: "Retain" + mountOptions: + - "vers=4.2,proto=tcp,port=2049" + nfs: + server: 10.0.20.180 + path: "/mnt/user/KubernetesBackup" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: va-unraid-backup-rw + namespace: kasten +spec: + accessModes: + - ReadWriteMany + storageClassName: "va-unraid-backup-rw" + resources: + requests: + storage: 100Ti +--- +apiVersion: v1 +kind: Namespace +metadata: + labels: + name: kasten + name: kasten +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: k10 + namespace: kube-system +spec: + repo: https://charts.kasten.io/ + chart: k10 + targetNamespace: kasten + valuesContent: |- + global: + persistence: + storageClass: mayastor-r1 diff --git a/k8s/default.nix b/k8s/default.nix index ba63255..7beee50 100644 --- a/k8s/default.nix +++ b/k8s/default.nix @@ -18,11 +18,16 @@ deps = [ ]; text = '' mkdir -p ${config.k8s.manifestsDir} + + # Storage - OpenEBS cp ${pkgs.substituteAll { src = ./config/openebs.yaml; nodeName = config.networking.hostName; diskPoolID = config.k8s.diskPoolID; }} ${config.k8s.manifestsDir}/openebs.yaml + + # Backup - Kasten + cp ${./config/kasten.yaml} ${config.k8s.manifestsDir}/kasten.yaml ''; }; };