configure cluster
This commit is contained in:
1
k8s/.gitignore
vendored
1
k8s/.gitignore
vendored
@@ -1 +0,0 @@
|
||||
rke2-token
|
||||
164
k8s/ceph.yaml
Normal file
164
k8s/ceph.yaml
Normal file
@@ -0,0 +1,164 @@
|
||||
---
|
||||
# Namespace
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
labels:
|
||||
name: rook-ceph
|
||||
name: rook-ceph
|
||||
|
||||
---
|
||||
# HelpChart
|
||||
apiVersion: helm.cattle.io/v1
|
||||
kind: HelmChart
|
||||
metadata:
|
||||
name: ceph
|
||||
namespace: kube-system
|
||||
spec:
|
||||
repo: https://charts.rook.io/release
|
||||
chart: rook-ceph
|
||||
targetNamespace: rook-ceph
|
||||
valuesContent: |-
|
||||
enableDiscoveryDaemon: true
|
||||
|
||||
---
|
||||
# CephCluster
|
||||
apiVersion: ceph.rook.io/v1
|
||||
kind: CephCluster
|
||||
metadata:
|
||||
name: rook-ceph
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
dataDirHostPath: /var/lib/rook
|
||||
cephVersion:
|
||||
image: quay.io/ceph/ceph:v19.2
|
||||
allowUnsupported: false
|
||||
|
||||
# HA - One monitor per node
|
||||
mon:
|
||||
count: 3
|
||||
allowMultiplePerNode: false
|
||||
|
||||
# Ceph Dashboard
|
||||
dashboard:
|
||||
enabled: true
|
||||
ssl: true
|
||||
|
||||
# Network Configuration
|
||||
network:
|
||||
provider: host
|
||||
|
||||
# Storage Configuration
|
||||
storage:
|
||||
useAllNodes: true
|
||||
useAllDevices: true
|
||||
config:
|
||||
osdsPerDevice: "1"
|
||||
replicatedSize: "3"
|
||||
|
||||
# Disruption Management
|
||||
disruptionManagement:
|
||||
managePodBudgets: true
|
||||
osdMaintenanceTimeout: 30
|
||||
|
||||
# Resource Management
|
||||
# resources:
|
||||
# mgr:
|
||||
# limits:
|
||||
# cpu: "1000m"
|
||||
# memory: "1Gi"
|
||||
# requests:
|
||||
# cpu: "500m"
|
||||
# memory: "512Mi"
|
||||
# mon:
|
||||
# limits:
|
||||
# cpu: "1000m"
|
||||
# memory: "1Gi"
|
||||
# requests:
|
||||
# cpu: "500m"
|
||||
# memory: "512Mi"
|
||||
# osd:
|
||||
# limits:
|
||||
# cpu: "2000m"
|
||||
# memory: "4Gi"
|
||||
# requests:
|
||||
# cpu: "1000m"
|
||||
# memory: "2Gi"
|
||||
|
||||
---
|
||||
# BlockPool - Single Replica
|
||||
apiVersion: ceph.rook.io/v1
|
||||
kind: CephBlockPool
|
||||
metadata:
|
||||
name: ceph-block-pool-single
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
failureDomain: host
|
||||
replicated:
|
||||
size: 1
|
||||
|
||||
---
|
||||
# BlockPool - Three Replica
|
||||
apiVersion: ceph.rook.io/v1
|
||||
kind: CephBlockPool
|
||||
metadata:
|
||||
name: ceph-block-pool-triple
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
failureDomain: host
|
||||
replicated:
|
||||
size: 3
|
||||
|
||||
---
|
||||
# StorageClass - Three Replica
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: ceph-block-triple
|
||||
annotations:
|
||||
storageclass.kubernetes.io/is-default-class: "true"
|
||||
provisioner: rook-ceph.rbd.csi.ceph.com
|
||||
parameters:
|
||||
pool: ceph-block-pool-triple
|
||||
clusterID: rook-ceph
|
||||
imageFormat: "2"
|
||||
imageFeatures: layering
|
||||
|
||||
# Ceph CSI driver
|
||||
csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
|
||||
csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
|
||||
csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
|
||||
csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/fstype: ext4
|
||||
|
||||
allowVolumeExpansion: true
|
||||
volumeBindingMode: Immediate
|
||||
reclaimPolicy: Delete
|
||||
|
||||
---
|
||||
# StorageClass - Single Replica
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: ceph-block-single
|
||||
provisioner: rook-ceph.rbd.csi.ceph.com
|
||||
parameters:
|
||||
pool: ceph-block-pool-single
|
||||
clusterID: rook-ceph
|
||||
imageFormat: "2"
|
||||
imageFeatures: layering
|
||||
|
||||
# Ceph CSI driver
|
||||
csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
|
||||
csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
|
||||
csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
|
||||
csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/fstype: ext4
|
||||
|
||||
allowVolumeExpansion: true
|
||||
volumeBindingMode: Immediate
|
||||
reclaimPolicy: Delete
|
||||
@@ -1,34 +0,0 @@
|
||||
{ config, lib, pkgs, ... }:
|
||||
|
||||
{
|
||||
options.k8s = {
|
||||
diskPoolID = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Disk Pool ID for OpenEBS";
|
||||
};
|
||||
|
||||
manifestsDir = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
description = "Directory for Kubernetes manifests";
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
system.activationScripts.k8s-manifests = {
|
||||
deps = [ ];
|
||||
text = ''
|
||||
mkdir -p ${config.k8s.manifestsDir}
|
||||
|
||||
# Storage - OpenEBS
|
||||
cp ${pkgs.substituteAll {
|
||||
src = ./config/openebs.yaml;
|
||||
nodeName = config.networking.hostName;
|
||||
diskPoolID = config.k8s.diskPoolID;
|
||||
}} ${config.k8s.manifestsDir}/openebs.yaml
|
||||
|
||||
# Backup - Kasten
|
||||
cp ${./config/kasten.yaml} ${config.k8s.manifestsDir}/kasten.yaml
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -45,7 +45,39 @@ spec:
|
||||
repo: https://charts.kasten.io/
|
||||
chart: k10
|
||||
targetNamespace: kasten
|
||||
valuesContent: |-
|
||||
global:
|
||||
persistence:
|
||||
storageClass: mayastor-r1
|
||||
---
|
||||
kind: Profile
|
||||
apiVersion: config.kio.kasten.io/v1alpha1
|
||||
metadata:
|
||||
name: k10-backup-profile
|
||||
namespace: kasten
|
||||
spec:
|
||||
locationSpec:
|
||||
type: FileStore
|
||||
fileStore:
|
||||
claimName: va-unraid-backup-rw
|
||||
credential:
|
||||
secretType: ""
|
||||
secret:
|
||||
apiVersion: ""
|
||||
kind: ""
|
||||
name: ""
|
||||
namespace: ""
|
||||
type: Location
|
||||
---
|
||||
apiVersion: config.kio.kasten.io/v1alpha1
|
||||
kind: TransformSet
|
||||
metadata:
|
||||
name: storage-class-rename
|
||||
namespace: kasten
|
||||
spec:
|
||||
comment: Renames cstor-r1 to ceph-block-triple
|
||||
transforms:
|
||||
- json:
|
||||
- op: replace
|
||||
path: /spec/storageClassName
|
||||
value: ceph-block-triple
|
||||
name: StorageClassRename
|
||||
subject:
|
||||
name: ""
|
||||
resource: persistentvolumeclaims
|
||||
50
k8s/longhorn.yaml
Normal file
50
k8s/longhorn.yaml
Normal file
@@ -0,0 +1,50 @@
|
||||
---
|
||||
# Namespace
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
labels:
|
||||
name: longhorn
|
||||
name: longhorn
|
||||
|
||||
---
|
||||
# HelpChart
|
||||
apiVersion: helm.cattle.io/v1
|
||||
kind: HelmChart
|
||||
metadata:
|
||||
name: longhorn
|
||||
namespace: kube-system
|
||||
spec:
|
||||
repo: https://charts.longhorn.io
|
||||
chart: longhorn
|
||||
targetNamespace: longhorn
|
||||
valuesContent: |-
|
||||
persistence:
|
||||
defaultClass: true
|
||||
defaultClassReplicaCount: 3
|
||||
reclaimPolicy: Delete
|
||||
|
||||
defaultSettings:
|
||||
defaultDataPath: /storage/longhorn
|
||||
defaultReplicaCount: 3
|
||||
nodeDownPodDeletionPolicy: delete-both-statefulset-and-deployment-pod
|
||||
guaranteedEngineManagerCPU: 0.25
|
||||
guaranteedReplicaManagerCPU: 0.25
|
||||
|
||||
longhornManager:
|
||||
tolerations:
|
||||
- key: "node-role.kubernetes.io/control-plane"
|
||||
operator: "Exists"
|
||||
effect: "NoSchedule"
|
||||
---
|
||||
# StorageClass
|
||||
kind: StorageClass
|
||||
apiVersion: storage.k8s.io/v1
|
||||
metadata:
|
||||
name: longhorn-block-triple
|
||||
provisioner: driver.longhorn.io
|
||||
allowVolumeExpansion: true
|
||||
parameters:
|
||||
numberOfReplicas: "3"
|
||||
staleReplicaTimeout: "2880"
|
||||
fsType: "ext4"
|
||||
9
k8s/openebs-disk-pool.yaml
Normal file
9
k8s/openebs-disk-pool.yaml
Normal file
@@ -0,0 +1,9 @@
|
||||
---
|
||||
apiVersion: "openebs.io/v1beta2"
|
||||
kind: DiskPool
|
||||
metadata:
|
||||
name: pool-on-@hostName@
|
||||
namespace: openebs
|
||||
spec:
|
||||
node: @hostName@
|
||||
disks: ["aio://@dataDiskID@"]
|
||||
@@ -29,15 +29,6 @@ spec:
|
||||
mayastor:
|
||||
enabled: true
|
||||
---
|
||||
apiVersion: "openebs.io/v1beta2"
|
||||
kind: DiskPool
|
||||
metadata:
|
||||
name: pool-on-@nodeName@
|
||||
namespace: openebs
|
||||
spec:
|
||||
node: @nodeName@
|
||||
disks: ["aio://@diskPoolID@"]
|
||||
---
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
@@ -51,11 +42,11 @@ provisioner: io.openebs.csi-mayastor
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: mayastor-r1
|
||||
name: mayastor-r3
|
||||
annotations:
|
||||
storageclass.kubernetes.io/is-default-class: "true"
|
||||
allowVolumeExpansion: true
|
||||
parameters:
|
||||
protocol: nvmf
|
||||
repl: "1"
|
||||
repl: "3"
|
||||
provisioner: io.openebs.csi-mayastor
|
||||
Reference in New Issue
Block a user