clean up
This commit is contained in:
parent
6f049807e6
commit
29070dd277
69
README.md
69
README.md
@ -4,24 +4,25 @@ This repository contains the configuration for multiple machines, as well as my
|
||||
|
||||
## Home Manager
|
||||
|
||||
Utilizing [Home Manager](https://nix-community.github.io/home-manager/)
|
||||
Utilizing [Home Manager](https://nix-community.github.io/home-manager/). Check out the [README.md](./home-manager/README.md).
|
||||
|
||||
## NixOS
|
||||
|
||||
### Image Build
|
||||
### NixOS Generators
|
||||
|
||||
```bash
|
||||
# Remote Build
|
||||
nix build .#packages.x86_64-linux.rke2-image -j0
|
||||
nix build .#packages.x86_64-linux.rke2-image
|
||||
```
|
||||
|
||||
### Copy Config
|
||||
### NixOS Hosts
|
||||
|
||||
#### Copy Config
|
||||
|
||||
```bash
|
||||
scp -r * root@10.10.10.10:/etc/nixos
|
||||
rsync -av --exclude='.git' . root@HOST:/etc/nixos
|
||||
```
|
||||
|
||||
### Partition Drives
|
||||
#### Partition Drives
|
||||
|
||||
```bash
|
||||
# Validate Disk
|
||||
@ -33,69 +34,27 @@ sudo nix \
|
||||
--experimental-features "nix-command flakes" \
|
||||
run github:nix-community/disko -- \
|
||||
--mode disko \
|
||||
--flake /etc/nixos#lin-va-rke1
|
||||
--flake /etc/nixos#HOST_CONFIG
|
||||
```
|
||||
|
||||
### Install NixOS
|
||||
#### Install NixOS
|
||||
|
||||
```bash
|
||||
# Install
|
||||
sudo nixos-install --flake /etc/nixos#lin-va-rke1
|
||||
sudo nixos-install --flake /etc/nixos#HOST_CONFIG
|
||||
|
||||
# Reboot
|
||||
sudo reboot
|
||||
```
|
||||
|
||||
### Copy Config Back to Host
|
||||
#### Copy Config Back to Host
|
||||
|
||||
```bash
|
||||
scp -r * nixos@10.0.20.201:/etc/nixos
|
||||
rsync -av --exclude='.git' . root@HOST:/etc/nixos
|
||||
```
|
||||
|
||||
### Rebuild NixOS
|
||||
#### Rebuild NixOS
|
||||
|
||||
```bash
|
||||
sudo nixos-rebuild switch
|
||||
```
|
||||
|
||||
# Install Kubernetes (RKE2)
|
||||
|
||||
```
|
||||
# Deploy First Node
|
||||
sudo nixos-install --flake /etc/nixos#lin-va-rke1
|
||||
|
||||
# Reboot & Get Token
|
||||
cat /var/lib/rancher/rke2/server/node-token
|
||||
|
||||
# Deploy Following Nodes
|
||||
echo "<TOKEN>" > ./_scratch/rke2-token
|
||||
sudo nixos-install --flake /etc/nixos#lin-va-rke2
|
||||
```
|
||||
|
||||
### Notes
|
||||
|
||||
### Kasten Port Forward
|
||||
|
||||
```bash
|
||||
# http://localhost:8000/k10/#/dashboard
|
||||
kubectl port-forward -n kasten svc/gateway 8000:80
|
||||
```
|
||||
|
||||
### Cloud Init
|
||||
|
||||
```
|
||||
#cloud-config
|
||||
hostname: rke-test
|
||||
```
|
||||
|
||||
```
|
||||
network:
|
||||
version: 1
|
||||
config:
|
||||
- type: physical
|
||||
name: enX0
|
||||
subnets:
|
||||
- type: static
|
||||
address: 10.0.50.5/24
|
||||
gateway: 10.0.50.254
|
||||
```
|
||||
|
||||
@ -1,147 +0,0 @@
|
||||
{ config, pkgs, lib, ... }:
|
||||
|
||||
{
|
||||
# Node Nix Config
|
||||
options = {
|
||||
dataDiskID = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "The device ID for the data disk";
|
||||
};
|
||||
serverAddr = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "The server to join";
|
||||
default = "";
|
||||
};
|
||||
networkConfig = lib.mkOption {
|
||||
type = lib.types.submodule {
|
||||
options = {
|
||||
interface = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Network interface name";
|
||||
example = "enp0s3";
|
||||
};
|
||||
address = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Static IP address";
|
||||
example = "10.0.20.200";
|
||||
};
|
||||
defaultGateway = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Default gateway IP";
|
||||
example = "10.0.20.254";
|
||||
};
|
||||
nameservers = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
description = "List of DNS servers";
|
||||
example = [ "10.0.20.254" "8.8.8.8" ];
|
||||
default = [ "8.8.8.8" "8.8.4.4" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
description = "Network configuration";
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
# ----------------------------------------
|
||||
# ---------- Base Configuration ----------
|
||||
# ----------------------------------------
|
||||
|
||||
# Ceph Requirements
|
||||
boot.kernelModules = [ "rbd" ];
|
||||
|
||||
# Network Configuration
|
||||
networking = {
|
||||
hostName = config.hostName;
|
||||
networkmanager.enable = false;
|
||||
|
||||
# Interface Configuration
|
||||
inherit (config.networkConfig) defaultGateway nameservers;
|
||||
interfaces.${config.networkConfig.interface}.ipv4.addresses = [{
|
||||
inherit (config.networkConfig) address;
|
||||
prefixLength = 24;
|
||||
}];
|
||||
|
||||
firewall = {
|
||||
enable = true;
|
||||
|
||||
allowedTCPPorts = [
|
||||
# RKE2 Ports - https://docs.rke2.io/install/requirements#networking
|
||||
6443 # Kubernetes API
|
||||
9345 # RKE2 supervisor API
|
||||
2379 # etcd Client Port
|
||||
2380 # etcd Peer Port
|
||||
2381 # etcd Metrics Port
|
||||
10250 # kubelet metrics
|
||||
9099 # Canal CNI health checks
|
||||
|
||||
# Ceph Ports
|
||||
3300 # Ceph MON daemon
|
||||
6789 # Ceph MON service
|
||||
] ++ lib.range 6800 7300; # Ceph OSD range
|
||||
|
||||
allowedUDPPorts = [
|
||||
# RKE2 Ports - https://docs.rke2.io/install/requirements#networking
|
||||
8472 # Canal CNI with VXLAN
|
||||
# 51820 # Canal CNI with WireGuard IPv4 (if using encryption)
|
||||
# 51821 # Canal CNI with WireGuard IPv6 (if using encryption)
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
# System Packages
|
||||
environment.systemPackages = with pkgs; [
|
||||
htop
|
||||
k9s
|
||||
kubectl
|
||||
kubernetes-helm
|
||||
nfs-utils
|
||||
tmux
|
||||
vim
|
||||
];
|
||||
|
||||
# ----------------------------------------
|
||||
# ---------- RKE2 Configuration ----------
|
||||
# ----------------------------------------
|
||||
|
||||
# RKE2 Join Token
|
||||
environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") {
|
||||
source = ../_scratch/rke2-token;
|
||||
mode = "0600";
|
||||
user = "root";
|
||||
group = "root";
|
||||
};
|
||||
|
||||
# Enable RKE2
|
||||
services.rke2 = {
|
||||
enable = true;
|
||||
role = "server";
|
||||
|
||||
disable = [
|
||||
# Disable - Utilizing Traefik
|
||||
"rke2-ingress-nginx"
|
||||
|
||||
# Distable - Utilizing OpenEBS's Snapshot Controller
|
||||
"rke2-snapshot-controller"
|
||||
"rke2-snapshot-controller-crd"
|
||||
"rke2-snapshot-validation-webhook"
|
||||
];
|
||||
|
||||
} // lib.optionalAttrs (config.serverAddr != "") {
|
||||
serverAddr = config.serverAddr;
|
||||
tokenFile = "/etc/rancher/rke2/node-token";
|
||||
};
|
||||
|
||||
# Bootstrap Kubernetes Manifests
|
||||
system.activationScripts.k8s-manifests = {
|
||||
deps = [ ];
|
||||
text = ''
|
||||
mkdir -p /var/lib/rancher/rke2/server/manifests
|
||||
|
||||
# Base Configs
|
||||
cp ${../k8s/ceph.yaml} /var/lib/rancher/rke2/server/manifests/ceph-base.yaml
|
||||
cp ${../k8s/kasten.yaml} /var/lib/rancher/rke2/server/manifests/kasten-base.yaml
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
@ -1,185 +0,0 @@
|
||||
{ config, pkgs, lib, ... }:
|
||||
|
||||
{
|
||||
# Node Nix Config
|
||||
options = {
|
||||
dataDiskID = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "The device ID for the data disk";
|
||||
};
|
||||
serverAddr = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "The server to join";
|
||||
default = "";
|
||||
};
|
||||
networkConfig = lib.mkOption {
|
||||
type = lib.types.submodule {
|
||||
options = {
|
||||
interface = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Network interface name";
|
||||
example = "enp0s3";
|
||||
};
|
||||
address = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Static IP address";
|
||||
example = "10.0.20.200";
|
||||
};
|
||||
defaultGateway = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Default gateway IP";
|
||||
example = "10.0.20.254";
|
||||
};
|
||||
nameservers = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
description = "List of DNS servers";
|
||||
example = [ "10.0.20.254" "8.8.8.8" ];
|
||||
default = [ "8.8.8.8" "8.8.4.4" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
description = "Network configuration";
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
# ----------------------------------------
|
||||
# ---------- Base Configuration ----------
|
||||
# ----------------------------------------
|
||||
|
||||
# Longhorn Requirements
|
||||
boot.kernelModules = [
|
||||
"iscsi_tcp"
|
||||
"dm_crypt"
|
||||
];
|
||||
|
||||
# Longhorn Data Disk
|
||||
disko.devices = {
|
||||
disk.longhorn = {
|
||||
type = "disk";
|
||||
device = config.dataDiskID;
|
||||
content = {
|
||||
type = "gpt";
|
||||
partitions = {
|
||||
longhorn = {
|
||||
size = "100%";
|
||||
content = {
|
||||
type = "filesystem";
|
||||
format = "xfs";
|
||||
mountpoint = "/storage/longhorn";
|
||||
mountOptions = [ "defaults" "nofail" ];
|
||||
extraArgs = [ "-d" "su=128k,sw=8" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
# Network Configuration
|
||||
networking = {
|
||||
hostName = config.hostName;
|
||||
networkmanager.enable = false;
|
||||
|
||||
# Interface Configuration
|
||||
inherit (config.networkConfig) defaultGateway nameservers;
|
||||
interfaces.${config.networkConfig.interface}.ipv4.addresses = [{
|
||||
inherit (config.networkConfig) address;
|
||||
prefixLength = 24;
|
||||
}];
|
||||
|
||||
firewall = {
|
||||
enable = true;
|
||||
|
||||
allowedTCPPorts = [
|
||||
# RKE2 Ports - https://docs.rke2.io/install/requirements#networking
|
||||
6443 # Kubernetes API
|
||||
9345 # RKE2 supervisor API
|
||||
2379 # etcd Client Port
|
||||
2380 # etcd Peer Port
|
||||
2381 # etcd Metrics Port
|
||||
10250 # kubelet metrics
|
||||
9099 # Canal CNI health checks
|
||||
|
||||
# iSCSI Port
|
||||
3260
|
||||
];
|
||||
|
||||
allowedUDPPorts = [
|
||||
# RKE2 Ports - https://docs.rke2.io/install/requirements#networking
|
||||
8472 # Canal CNI with VXLAN
|
||||
# 51820 # Canal CNI with WireGuard IPv4 (if using encryption)
|
||||
# 51821 # Canal CNI with WireGuard IPv6 (if using encryption)
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
# System Packages
|
||||
environment.systemPackages = with pkgs; [
|
||||
htop
|
||||
k9s
|
||||
kubectl
|
||||
kubernetes-helm
|
||||
nfs-utils
|
||||
openiscsi
|
||||
tmux
|
||||
vim
|
||||
];
|
||||
|
||||
# ----------------------------------------
|
||||
# ---------- RKE2 Configuration ----------
|
||||
# ----------------------------------------
|
||||
|
||||
# RKE2 Join Token
|
||||
environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") {
|
||||
source = ../_scratch/rke2-token;
|
||||
mode = "0600";
|
||||
user = "root";
|
||||
group = "root";
|
||||
};
|
||||
|
||||
# Enable RKE2
|
||||
services.rke2 = {
|
||||
enable = true;
|
||||
role = "server";
|
||||
|
||||
disable = [
|
||||
# Disable - Utilizing Traefik
|
||||
"rke2-ingress-nginx"
|
||||
|
||||
# Disable - Utilizing Longhorn's Snapshot Controller
|
||||
"rke2-snapshot-controller"
|
||||
"rke2-snapshot-controller-crd"
|
||||
"rke2-snapshot-validation-webhook"
|
||||
];
|
||||
} // lib.optionalAttrs (config.serverAddr != "") {
|
||||
serverAddr = config.serverAddr;
|
||||
tokenFile = "/etc/rancher/rke2/node-token";
|
||||
};
|
||||
|
||||
# Enable OpeniSCSI
|
||||
services.openiscsi = {
|
||||
enable = true;
|
||||
name = "iqn.2025-01.${config.hostName}:initiator";
|
||||
};
|
||||
|
||||
# Bootstrap Kubernetes Manifests
|
||||
system.activationScripts.k8s-manifests = {
|
||||
deps = [ ];
|
||||
text = ''
|
||||
mkdir -p /var/lib/rancher/rke2/server/manifests
|
||||
|
||||
# Base Configs
|
||||
cp ${../k8s/longhorn.yaml} /var/lib/rancher/rke2/server/manifests/longhorn-base.yaml
|
||||
# cp ${../k8s/kasten.yaml} /var/lib/rancher/rke2/server/manifests/kasten-base.yaml
|
||||
'';
|
||||
};
|
||||
|
||||
# Add Symlinks Expected by Longhorn
|
||||
system.activationScripts.add-symlinks = ''
|
||||
mkdir -p /usr/bin
|
||||
ln -sf ${pkgs.openiscsi}/bin/iscsiadm /usr/bin/iscsiadm
|
||||
ln -sf ${pkgs.openiscsi}/bin/iscsid /usr/bin/iscsid
|
||||
'';
|
||||
};
|
||||
}
|
||||
@ -1,162 +0,0 @@
|
||||
{ config, pkgs, lib, ... }:
|
||||
|
||||
{
|
||||
# Node Nix Config
|
||||
options = {
|
||||
dataDiskID = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "The device ID for the data disk";
|
||||
};
|
||||
serverAddr = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "The server to join";
|
||||
default = "";
|
||||
};
|
||||
networkConfig = lib.mkOption {
|
||||
type = lib.types.submodule {
|
||||
options = {
|
||||
interface = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Network interface name";
|
||||
example = "enp0s3";
|
||||
};
|
||||
address = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Static IP address";
|
||||
example = "10.0.20.200";
|
||||
};
|
||||
defaultGateway = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Default gateway IP";
|
||||
example = "10.0.20.254";
|
||||
};
|
||||
nameservers = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
description = "List of DNS servers";
|
||||
example = [ "10.0.20.254" "8.8.8.8" ];
|
||||
default = [ "8.8.8.8" "8.8.4.4" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
description = "Network configuration";
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
# ----------------------------------------
|
||||
# ---------- Base Configuration ----------
|
||||
# ----------------------------------------
|
||||
|
||||
# OpenEBS Mayastor Requirements
|
||||
boot.kernelModules = [ "nvme_tcp" ];
|
||||
boot.kernel.sysctl = {
|
||||
"vm.nr_hugepages" = 1024;
|
||||
};
|
||||
|
||||
# Network Configuration
|
||||
networking = {
|
||||
hostName = config.hostName;
|
||||
networkmanager.enable = false;
|
||||
|
||||
# Interface Configuration
|
||||
inherit (config.networkConfig) defaultGateway nameservers;
|
||||
interfaces.${config.networkConfig.interface}.ipv4.addresses = [{
|
||||
inherit (config.networkConfig) address;
|
||||
prefixLength = 24;
|
||||
}];
|
||||
|
||||
firewall = {
|
||||
enable = true;
|
||||
|
||||
allowedTCPPorts = [
|
||||
# RKE2 Ports - https://docs.rke2.io/install/requirements#networking
|
||||
6443 # Kubernetes API
|
||||
9345 # RKE2 supervisor API
|
||||
2379 # etcd Client Port
|
||||
2380 # etcd Peer Port
|
||||
2381 # etcd Metrics Port
|
||||
10250 # kubelet metrics
|
||||
9099 # Canal CNI health checks
|
||||
|
||||
# OpenEBS Mayastor - https://openebs.io/docs/user-guides/replicated-storage-user-guide/replicated-pv-mayastor/rs-installation#network-requirements
|
||||
10124 # REST API
|
||||
8420 # NVMf
|
||||
4421 # NVMf
|
||||
];
|
||||
|
||||
allowedUDPPorts = [
|
||||
# RKE2 Ports - https://docs.rke2.io/install/requirements#networking
|
||||
8472 # Canal CNI with VXLAN
|
||||
# 51820 # Canal CNI with WireGuard IPv4 (if using encryption)
|
||||
# 51821 # Canal CNI with WireGuard IPv6 (if using encryption)
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
# System Packages
|
||||
environment.systemPackages = with pkgs; [
|
||||
htop
|
||||
k9s
|
||||
kubectl
|
||||
kubernetes-helm
|
||||
nfs-utils
|
||||
vim
|
||||
];
|
||||
|
||||
# ----------------------------------------
|
||||
# ---------- RKE2 Configuration ----------
|
||||
# ----------------------------------------
|
||||
|
||||
# RKE2 Join Token
|
||||
environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") {
|
||||
source = ../_scratch/rke2-token;
|
||||
mode = "0600";
|
||||
user = "root";
|
||||
group = "root";
|
||||
};
|
||||
|
||||
# Enable RKE2
|
||||
services.rke2 = {
|
||||
enable = true;
|
||||
role = "server";
|
||||
|
||||
disable = [
|
||||
# Disable - Utilizing Traefik
|
||||
"rke2-ingress-nginx"
|
||||
|
||||
# Distable - Utilizing OpenEBS's Snapshot Controller
|
||||
"rke2-snapshot-controller"
|
||||
"rke2-snapshot-controller-crd"
|
||||
"rke2-snapshot-validation-webhook"
|
||||
];
|
||||
|
||||
# OpenEBS Scheduleable
|
||||
nodeLabel = [
|
||||
"openebs.io/engine=mayastor"
|
||||
];
|
||||
|
||||
} // lib.optionalAttrs (config.serverAddr != "") {
|
||||
serverAddr = config.serverAddr;
|
||||
tokenFile = "/etc/rancher/rke2/node-token";
|
||||
};
|
||||
|
||||
# Bootstrap Kubernetes Manifests
|
||||
system.activationScripts.k8s-manifests = {
|
||||
deps = [ ];
|
||||
text = ''
|
||||
mkdir -p /var/lib/rancher/rke2/server/manifests
|
||||
|
||||
# Base Configs
|
||||
cp ${../k8s/openebs.yaml} /var/lib/rancher/rke2/server/manifests/openebs-base.yaml
|
||||
cp ${../k8s/kasten.yaml} /var/lib/rancher/rke2/server/manifests/kasten-base.yaml
|
||||
|
||||
# OpenEBS Disk Pool
|
||||
cp ${pkgs.substituteAll {
|
||||
src = ../k8s/openebs-disk-pool.yaml;
|
||||
hostName = config.hostName;
|
||||
dataDiskID = config.dataDiskID;
|
||||
}} /var/lib/rancher/rke2/server/manifests/openebs-disk-pool-${config.hostName}.yaml
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
215
hosts/rke2.nix
215
hosts/rke2.nix
@ -1,215 +0,0 @@
|
||||
{ config, pkgs, lib, ... }:
|
||||
|
||||
let
|
||||
formatPrivateKey = indentLevel: value:
|
||||
let
|
||||
indent = lib.strings.fixedWidthString indentLevel " " "";
|
||||
indentedLines = lib.strings.concatMapStrings
|
||||
(line: "${indent}${line}\n")
|
||||
(lib.strings.splitString "\n" value);
|
||||
in
|
||||
"|\n${indentedLines}";
|
||||
in
|
||||
{
|
||||
# Node Nix Config
|
||||
options = {
|
||||
serverAddr = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "The server to join";
|
||||
default = "";
|
||||
};
|
||||
democraticConfig = lib.mkOption {
|
||||
type = lib.types.submodule {
|
||||
options = {
|
||||
apiKeyFile = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
description = "Path to file containing the TrueNAS API key";
|
||||
};
|
||||
sshKeyFile = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
description = "Path to file containing the TrueNAS User SSH private key";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
networkConfig = lib.mkOption {
|
||||
type = lib.types.submodule {
|
||||
options = {
|
||||
interface = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Network interface name";
|
||||
example = "enp0s3";
|
||||
};
|
||||
address = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Static IP address";
|
||||
example = "10.0.20.200";
|
||||
};
|
||||
defaultGateway = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Default gateway IP";
|
||||
example = "10.0.20.254";
|
||||
};
|
||||
nameservers = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
description = "List of DNS servers";
|
||||
example = [ "10.0.20.254" "8.8.8.8" ];
|
||||
default = [ "8.8.8.8" "8.8.4.4" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
description = "Network configuration";
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
# ----------------------------------------
|
||||
# ---------- Base Configuration ----------
|
||||
# ----------------------------------------
|
||||
# Democratic Requirements
|
||||
boot.initrd = {
|
||||
availableKernelModules = [ "xen_blkfront" "xen_netfront" ];
|
||||
kernelModules = [ "xen_netfront" "xen_blkfront" ];
|
||||
supportedFilesystems = [ "ext4" "xenfs" ];
|
||||
};
|
||||
|
||||
boot.kernelModules = [
|
||||
# Xen VM Requirements
|
||||
"xen_netfront"
|
||||
"xen_blkfront"
|
||||
"xenfs"
|
||||
|
||||
# iSCSI & Multipath
|
||||
"iscsi_tcp"
|
||||
"dm_multipath"
|
||||
"dm_round_robin"
|
||||
];
|
||||
|
||||
# Network Configuration
|
||||
networking = {
|
||||
hostName = config.hostName;
|
||||
networkmanager.enable = false;
|
||||
|
||||
# Interface Configuration
|
||||
inherit (config.networkConfig) defaultGateway nameservers;
|
||||
interfaces."${config.networkConfig.interface}" = {
|
||||
mtu = 9000;
|
||||
ipv4.addresses = [{
|
||||
address = config.networkConfig.address;
|
||||
prefixLength = 24;
|
||||
}];
|
||||
};
|
||||
|
||||
firewall = {
|
||||
enable = true;
|
||||
|
||||
allowedTCPPorts = [
|
||||
# RKE2 Ports - https://docs.rke2.io/install/requirements#networking
|
||||
6443 # Kubernetes API
|
||||
9345 # RKE2 supervisor API
|
||||
2379 # etcd Client Port
|
||||
2380 # etcd Peer Port
|
||||
2381 # etcd Metrics Port
|
||||
10250 # kubelet metrics
|
||||
9099 # Canal CNI health checks
|
||||
];
|
||||
|
||||
allowedUDPPorts = [
|
||||
# RKE2 Ports - https://docs.rke2.io/install/requirements#networking
|
||||
8472 # Canal CNI with VXLAN
|
||||
# 51820 # Canal CNI with WireGuard IPv4 (if using encryption)
|
||||
# 51821 # Canal CNI with WireGuard IPv6 (if using encryption)
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
# System Packages
|
||||
environment.systemPackages = with pkgs; [
|
||||
htop
|
||||
k9s
|
||||
kubectl
|
||||
kubernetes-helm
|
||||
openiscsi
|
||||
tmux
|
||||
vim
|
||||
];
|
||||
|
||||
# ----------------------------------------
|
||||
# ---------- RKE2 Configuration ----------
|
||||
# ----------------------------------------
|
||||
|
||||
# RKE2 Join Token
|
||||
environment.etc."rancher/rke2/node-token" = lib.mkIf (config.serverAddr != "") {
|
||||
source = ../_scratch/rke2-token;
|
||||
mode = "0600";
|
||||
user = "root";
|
||||
group = "root";
|
||||
};
|
||||
|
||||
# Enable RKE2
|
||||
services.rke2 = {
|
||||
enable = true;
|
||||
role = "server";
|
||||
|
||||
disable = [
|
||||
# Disable - Utilizing Traefik
|
||||
"rke2-ingress-nginx"
|
||||
|
||||
# Disable
|
||||
# "rke2-snapshot-controller"
|
||||
# "rke2-snapshot-controller-crd"
|
||||
# "rke2-snapshot-validation-webhook"
|
||||
];
|
||||
} // lib.optionalAttrs (config.serverAddr != "") {
|
||||
serverAddr = config.serverAddr;
|
||||
tokenFile = "/etc/rancher/rke2/node-token";
|
||||
};
|
||||
|
||||
# Enable Xe Guest Utilities
|
||||
services.xe-guest-utilities.enable = true;
|
||||
|
||||
# Enable OpeniSCSI
|
||||
services.openiscsi = {
|
||||
enable = true;
|
||||
name = "iqn.2025-02.${config.hostName}:initiator";
|
||||
};
|
||||
|
||||
# Enable Multipath
|
||||
services.multipath = {
|
||||
enable = true;
|
||||
defaults = ''
|
||||
defaults {
|
||||
user_friendly_names yes
|
||||
find_multipaths yes
|
||||
}
|
||||
'';
|
||||
pathGroups = [ ];
|
||||
};
|
||||
|
||||
time.timeZone = "UTC";
|
||||
|
||||
# Add Symlinks Expected by Democratic
|
||||
system.activationScripts.add-symlinks = ''
|
||||
mkdir -p /usr/bin
|
||||
ln -sf ${pkgs.openiscsi}/bin/iscsiadm /usr/bin/iscsiadm
|
||||
ln -sf ${pkgs.openiscsi}/bin/iscsid /usr/bin/iscsid
|
||||
'';
|
||||
|
||||
# Bootstrap Kubernetes Manifests
|
||||
system.activationScripts.k8s-manifests = {
|
||||
deps = [ ];
|
||||
text = ''
|
||||
mkdir -p /var/lib/rancher/rke2/server/manifests
|
||||
|
||||
# Base Configs
|
||||
cp ${pkgs.substituteAll {
|
||||
src = ../k8s/democratic.yaml;
|
||||
apiKey = lib.strings.removeSuffix "\n" (builtins.readFile config.democraticConfig.apiKeyFile);
|
||||
privateKey = formatPrivateKey 12 (lib.strings.removeSuffix "\n" (builtins.readFile config.democraticConfig.sshKeyFile));
|
||||
}} /var/lib/rancher/rke2/server/manifests/democratic-base.yaml
|
||||
|
||||
cp ${../k8s/kasten.yaml} /var/lib/rancher/rke2/server/manifests/kasten-base.yaml
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
164
k8s/ceph.yaml
164
k8s/ceph.yaml
@ -1,164 +0,0 @@
|
||||
---
|
||||
# Namespace
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
labels:
|
||||
name: rook-ceph
|
||||
name: rook-ceph
|
||||
|
||||
---
|
||||
# HelpChart
|
||||
apiVersion: helm.cattle.io/v1
|
||||
kind: HelmChart
|
||||
metadata:
|
||||
name: ceph
|
||||
namespace: kube-system
|
||||
spec:
|
||||
repo: https://charts.rook.io/release
|
||||
chart: rook-ceph
|
||||
targetNamespace: rook-ceph
|
||||
valuesContent: |-
|
||||
enableDiscoveryDaemon: true
|
||||
|
||||
---
|
||||
# CephCluster
|
||||
apiVersion: ceph.rook.io/v1
|
||||
kind: CephCluster
|
||||
metadata:
|
||||
name: rook-ceph
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
dataDirHostPath: /var/lib/rook
|
||||
cephVersion:
|
||||
image: quay.io/ceph/ceph:v19.2
|
||||
allowUnsupported: false
|
||||
|
||||
# HA - One monitor per node
|
||||
mon:
|
||||
count: 3
|
||||
allowMultiplePerNode: false
|
||||
|
||||
# Ceph Dashboard
|
||||
dashboard:
|
||||
enabled: true
|
||||
ssl: true
|
||||
|
||||
# Network Configuration
|
||||
network:
|
||||
provider: host
|
||||
|
||||
# Storage Configuration
|
||||
storage:
|
||||
useAllNodes: true
|
||||
useAllDevices: true
|
||||
config:
|
||||
osdsPerDevice: "1"
|
||||
replicatedSize: "3"
|
||||
|
||||
# Disruption Management
|
||||
disruptionManagement:
|
||||
managePodBudgets: true
|
||||
osdMaintenanceTimeout: 30
|
||||
|
||||
# Resource Management
|
||||
# resources:
|
||||
# mgr:
|
||||
# limits:
|
||||
# cpu: "1000m"
|
||||
# memory: "1Gi"
|
||||
# requests:
|
||||
# cpu: "500m"
|
||||
# memory: "512Mi"
|
||||
# mon:
|
||||
# limits:
|
||||
# cpu: "1000m"
|
||||
# memory: "1Gi"
|
||||
# requests:
|
||||
# cpu: "500m"
|
||||
# memory: "512Mi"
|
||||
# osd:
|
||||
# limits:
|
||||
# cpu: "2000m"
|
||||
# memory: "4Gi"
|
||||
# requests:
|
||||
# cpu: "1000m"
|
||||
# memory: "2Gi"
|
||||
|
||||
---
|
||||
# BlockPool - Single Replica
|
||||
apiVersion: ceph.rook.io/v1
|
||||
kind: CephBlockPool
|
||||
metadata:
|
||||
name: ceph-block-pool-single
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
failureDomain: host
|
||||
replicated:
|
||||
size: 1
|
||||
|
||||
---
|
||||
# BlockPool - Three Replica
|
||||
apiVersion: ceph.rook.io/v1
|
||||
kind: CephBlockPool
|
||||
metadata:
|
||||
name: ceph-block-pool-triple
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
failureDomain: host
|
||||
replicated:
|
||||
size: 3
|
||||
|
||||
---
|
||||
# StorageClass - Three Replica
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: ceph-block-triple
|
||||
annotations:
|
||||
storageclass.kubernetes.io/is-default-class: "true"
|
||||
provisioner: rook-ceph.rbd.csi.ceph.com
|
||||
parameters:
|
||||
pool: ceph-block-pool-triple
|
||||
clusterID: rook-ceph
|
||||
imageFormat: "2"
|
||||
imageFeatures: layering
|
||||
|
||||
# Ceph CSI driver
|
||||
csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
|
||||
csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
|
||||
csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
|
||||
csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/fstype: ext4
|
||||
|
||||
allowVolumeExpansion: true
|
||||
volumeBindingMode: Immediate
|
||||
reclaimPolicy: Delete
|
||||
|
||||
---
|
||||
# StorageClass - Single Replica
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: ceph-block-single
|
||||
provisioner: rook-ceph.rbd.csi.ceph.com
|
||||
parameters:
|
||||
pool: ceph-block-pool-single
|
||||
clusterID: rook-ceph
|
||||
imageFormat: "2"
|
||||
imageFeatures: layering
|
||||
|
||||
# Ceph CSI driver
|
||||
csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
|
||||
csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
|
||||
csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
|
||||
csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/fstype: ext4
|
||||
|
||||
allowVolumeExpansion: true
|
||||
volumeBindingMode: Immediate
|
||||
reclaimPolicy: Delete
|
||||
@ -1,73 +0,0 @@
|
||||
---
|
||||
# Namespace
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
labels:
|
||||
name: democratic-csi
|
||||
name: democratic-csi
|
||||
|
||||
---
|
||||
# HelmChart
|
||||
apiVersion: helm.cattle.io/v1
|
||||
kind: HelmChart
|
||||
metadata:
|
||||
name: democratic-csi
|
||||
namespace: kube-system
|
||||
spec:
|
||||
repo: https://democratic-csi.github.io/charts/
|
||||
chart: democratic-csi
|
||||
targetNamespace: democratic-csi
|
||||
valuesContent: |-
|
||||
csiDriver:
|
||||
name: "org.democratic-csi.iscsi"
|
||||
|
||||
storageClasses:
|
||||
- name: truenas-iscsi
|
||||
defaultClass: true
|
||||
reclaimPolicy: Delete
|
||||
volumeBindingMode: Immediate
|
||||
allowVolumeExpansion: true
|
||||
parameters:
|
||||
fsType: xfs
|
||||
|
||||
driver:
|
||||
config:
|
||||
driver: freenas-iscsi
|
||||
instance_id: kube
|
||||
httpConnection:
|
||||
protocol: http
|
||||
host: 10.0.50.60
|
||||
port: 80
|
||||
apiKey: @apiKey@
|
||||
apiVersion: 2
|
||||
sshConnection:
|
||||
host: 10.0.50.60
|
||||
port: 22
|
||||
username: k8s-csi
|
||||
privateKey: @privateKey@
|
||||
zfs:
|
||||
cli:
|
||||
sudoEnabled: true
|
||||
paths:
|
||||
zfs: /sbin/zfs
|
||||
zpool: /sbin/zpool
|
||||
sudo: /usr/bin/sudo
|
||||
chroot: /usr/sbin/chroot
|
||||
datasetParentName: KubeStorage/pv/iscsi/v
|
||||
detachedSnapshotsDatasetParentName: KubeStorage/pv/iscsi/s
|
||||
zvolEnableReservation: false
|
||||
iscsi:
|
||||
targetPortal: "10.0.50.60:3260"
|
||||
targetPortals: []
|
||||
namePrefix: csi-
|
||||
nameSuffix: "-cluster"
|
||||
targetGroups:
|
||||
- targetGroupPortalGroup: 1
|
||||
targetGroupInitiatorGroup: 1
|
||||
targetGroupAuthType: None
|
||||
extentInsecureTpc: true
|
||||
extentXenCompat: false
|
||||
extentDisablePhysicalBlocksize: true
|
||||
extentBlocksize: 4096
|
||||
extentAvailThreshold: 0
|
||||
@ -1,83 +0,0 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: va-unraid-backup-rw
|
||||
spec:
|
||||
capacity:
|
||||
storage: 100Ti
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
storageClassName: "va-unraid-backup-rw"
|
||||
persistentVolumeReclaimPolicy: "Retain"
|
||||
mountOptions:
|
||||
- "vers=4.2,proto=tcp,port=2049"
|
||||
nfs:
|
||||
server: 10.0.20.180
|
||||
path: "/mnt/user/KubernetesBackup"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: va-unraid-backup-rw
|
||||
namespace: kasten
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
storageClassName: "va-unraid-backup-rw"
|
||||
resources:
|
||||
requests:
|
||||
storage: 100Ti
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: kasten
|
||||
labels:
|
||||
name: kasten
|
||||
---
|
||||
apiVersion: helm.cattle.io/v1
|
||||
kind: HelmChart
|
||||
metadata:
|
||||
name: k10
|
||||
namespace: kube-system
|
||||
spec:
|
||||
repo: https://charts.kasten.io/
|
||||
chart: k10
|
||||
targetNamespace: kasten
|
||||
---
|
||||
kind: Profile
|
||||
apiVersion: config.kio.kasten.io/v1alpha1
|
||||
metadata:
|
||||
name: k10-backup-profile
|
||||
namespace: kasten
|
||||
spec:
|
||||
locationSpec:
|
||||
type: FileStore
|
||||
fileStore:
|
||||
claimName: va-unraid-backup-rw
|
||||
credential:
|
||||
secretType: ""
|
||||
secret:
|
||||
apiVersion: ""
|
||||
kind: ""
|
||||
name: ""
|
||||
namespace: ""
|
||||
type: Location
|
||||
---
|
||||
apiVersion: config.kio.kasten.io/v1alpha1
|
||||
kind: TransformSet
|
||||
metadata:
|
||||
name: storage-class-rename
|
||||
namespace: kasten
|
||||
spec:
|
||||
comment: Renames cstor-r1 to truenas-iscsi
|
||||
transforms:
|
||||
- json:
|
||||
- op: replace
|
||||
path: /spec/storageClassName
|
||||
value: truenas-iscsi
|
||||
name: StorageClassRename
|
||||
subject:
|
||||
name: ""
|
||||
resource: persistentvolumeclaims
|
||||
@ -1,50 +0,0 @@
|
||||
---
|
||||
# Namespace
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
labels:
|
||||
name: longhorn
|
||||
name: longhorn
|
||||
|
||||
---
|
||||
# HelpChart
|
||||
apiVersion: helm.cattle.io/v1
|
||||
kind: HelmChart
|
||||
metadata:
|
||||
name: longhorn
|
||||
namespace: kube-system
|
||||
spec:
|
||||
repo: https://charts.longhorn.io
|
||||
chart: longhorn
|
||||
targetNamespace: longhorn
|
||||
valuesContent: |-
|
||||
persistence:
|
||||
defaultClass: true
|
||||
defaultClassReplicaCount: 3
|
||||
reclaimPolicy: Delete
|
||||
|
||||
defaultSettings:
|
||||
defaultDataPath: /storage/longhorn
|
||||
defaultReplicaCount: 3
|
||||
nodeDownPodDeletionPolicy: delete-both-statefulset-and-deployment-pod
|
||||
guaranteedEngineManagerCPU: 0.25
|
||||
guaranteedReplicaManagerCPU: 0.25
|
||||
|
||||
longhornManager:
|
||||
tolerations:
|
||||
- key: "node-role.kubernetes.io/control-plane"
|
||||
operator: "Exists"
|
||||
effect: "NoSchedule"
|
||||
---
|
||||
# StorageClass
|
||||
kind: StorageClass
|
||||
apiVersion: storage.k8s.io/v1
|
||||
metadata:
|
||||
name: longhorn-block-triple
|
||||
provisioner: driver.longhorn.io
|
||||
allowVolumeExpansion: true
|
||||
parameters:
|
||||
numberOfReplicas: "3"
|
||||
staleReplicaTimeout: "2880"
|
||||
fsType: "ext4"
|
||||
@ -1,9 +0,0 @@
|
||||
---
|
||||
apiVersion: "openebs.io/v1beta2"
|
||||
kind: DiskPool
|
||||
metadata:
|
||||
name: pool-on-@hostName@
|
||||
namespace: openebs
|
||||
spec:
|
||||
node: @hostName@
|
||||
disks: ["aio://@dataDiskID@"]
|
||||
@ -1,52 +0,0 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
labels:
|
||||
name: openebs
|
||||
name: openebs
|
||||
---
|
||||
apiVersion: helm.cattle.io/v1
|
||||
kind: HelmChart
|
||||
metadata:
|
||||
name: openebs
|
||||
namespace: kube-system
|
||||
spec:
|
||||
repo: https://openebs.github.io/openebs
|
||||
chart: openebs
|
||||
targetNamespace: openebs
|
||||
valuesContent: |-
|
||||
mayastor:
|
||||
etcd:
|
||||
replicaCount: 1
|
||||
engines:
|
||||
local:
|
||||
lvm:
|
||||
enabled: false
|
||||
zfs:
|
||||
enabled: false
|
||||
replicated:
|
||||
mayastor:
|
||||
enabled: true
|
||||
---
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: cstor-r1
|
||||
allowVolumeExpansion: true
|
||||
parameters:
|
||||
protocol: nvmf
|
||||
repl: "1"
|
||||
provisioner: io.openebs.csi-mayastor
|
||||
---
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: mayastor-r3
|
||||
annotations:
|
||||
storageclass.kubernetes.io/is-default-class: "true"
|
||||
allowVolumeExpansion: true
|
||||
parameters:
|
||||
protocol: nvmf
|
||||
repl: "3"
|
||||
provisioner: io.openebs.csi-mayastor
|
||||
Loading…
x
Reference in New Issue
Block a user