feat: move to upstream Hcloud charts

2024-02-14 21:42:42 +01:00 · 2024-02-14 21:42:42 +01:00 · 420a6d3489
commit 420a6d3489
parent 1dde3d9c01
8 changed files with 51 additions and 513 deletions
--- a/infrastructure/configs/cp/traefik.yaml
+++ b/infrastructure/configs/cp/traefik.yaml
@ -6,24 +6,33 @@ metadata:
 spec:
  chart: traefik
  repo: https://traefik.github.io/charts
-  version: 24.0.0
+  version: 26.0.0
  valuesContent: |-
    ports:
      traefik:
        port: 9000
-        exposedPort: 9000
-        expose: true
+        expose: false
      web:
        nodePort: 32080
        forwardedHeaders:
          insecure: true
      websecure:
-        expose: false
+        expose: true
    service:
-      type: NodePort
+      type: LoadBalancer
+      annotations:
+        load-balancer.hetzner.cloud/location: "hel1"
    experimental:
      kubernetesGateway:
        enabled: true
+    providers:
+      kubernetesIngress:
+        publishedService:
+          enabled: true
+        allowExternalNameServices: true
+      kubernetesCRD:
+        enabled: true
+        allowExternalNameServices: true
    metrics:
      prometheus:
        serviceMonitor:
--- a/infrastructure/vms.auto.tfvars
+++ b/infrastructure/vms.auto.tfvars
@ -1,9 +1,9 @@
 k3s_control_plane = {
-  "cp1-cax11-hel1-gen2" = {
+  "cp1-cax11-hel1-gen3" = {
    server_type = "cax11",
-    private_ip  = "172.23.2.11"
+    private_ip  = "172.23.2.10"
    location    = "hel1"
-    alias_ips = ["172.23.2.10"]
+    alias_ips = []
  }
 }

--- a/k8s/roles/fider/files/resources/ingress.yaml
+++ b/k8s/roles/fider/files/resources/ingress.yaml
@ -3,6 +3,8 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: fider
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-production
 spec:
  rules:
    - host: fider.icb4dc0.de
@ -44,4 +46,11 @@ spec:
              service:
                name: fider
                port:
-                  number: 3000                  
+                  number: 3000
+  tls:
+    - hosts:
+        - fider.icb4dc0.de
+        - login.fider.icb4dc0.de
+        - community.buildr.icb4dc0.de
+        - community.inetmock.icb4dc0.de
+      secretName: fider-ingress-tls
--- a/k8s/roles/hcloud/tasks/main.yml
+++ b/k8s/roles/hcloud/tasks/main.yml
@ -12,12 +12,6 @@
        token: "{{ HcloudToken | b64encode }}"
        network: "{{ 'k8s-net' | b64encode }}"

- name: Deploy CSI driver
-  kubernetes.core.k8s:
-    state: present
-    definition: "{{ item }}"
-  loop: "{{ lookup('ansible.builtin.template', 'hcloud-csi.yml.j2') | ansible.builtin.from_yaml_all | list }}"
-
 - name: Add Hcloud chart repo
  kubernetes.core.helm_repository:
    name: hcloud
@ -28,24 +22,14 @@
    name: hccm
    chart_ref: hcloud/hcloud-cloud-controller-manager
    release_namespace: kube-system
-    chart_version: "1.17.0"
+    chart_version: "1.19.0"
    release_values: "{{ lookup('template', 'values.hccm.yml.j2') | from_yaml }}"

- name: Create CSI controller PodMonitor
-  kubernetes.core.k8s:
-    state: present
-    definition:
-      apiVersion: monitoring.coreos.com/v1
-      kind: PodMonitor
-      metadata:
-        name: hcloud-csi-controller
-        namespace: kube-system
-        labels:
-          prometheus: default
-      spec:
-        selector:
-          matchLabels:
-            app: hcloud-csi-controller
-        podMetricsEndpoints:
-          - port: metrics
-            path: /
+- name: Deploy hcloud CSI driver
+  kubernetes.core.helm:
+    name: hcloud-csi-driver
+    chart_ref: hcloud/hcloud-csi
+    release_namespace: kube-system
+    chart_version: "2.6.0"
+    release_values: "{{ lookup('template', 'values.csi.yml.j2') | from_yaml }}"
+
--- a/k8s/roles/hcloud/templates/cloud-controller-manager.yml.j2
+++ b/k8s/roles/hcloud/templates/cloud-controller-manager.yml.j2
@ -1,84 +0,0 @@
---
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: cloud-controller-manager
-  namespace: kube-system
---
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:cloud-controller-manager
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: cluster-admin
-subjects:
-  - kind: ServiceAccount
-    name: cloud-controller-manager
-    namespace: kube-system
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: hcloud-cloud-controller-manager
-  namespace: kube-system
-spec:
-  replicas: 1
-  revisionHistoryLimit: 2
-  selector:
-    matchLabels:
-      app: hcloud-cloud-controller-manager
-  template:
-    metadata:
-      labels:
-        app: hcloud-cloud-controller-manager
-    spec:
-      serviceAccountName: cloud-controller-manager
-      dnsPolicy: Default
-      tolerations:
-        # this taint is set by all kubelets running `--cloud-provider=external`
-        # so we should tolerate it to schedule the cloud controller manager
-        - key: "node.cloudprovider.kubernetes.io/uninitialized"
-          value: "true"
-          effect: "NoSchedule"
-        - key: "CriticalAddonsOnly"
-          operator: "Exists"
-        # cloud controller manages should be able to run on masters
-        - key: "node-role.kubernetes.io/master"
-          effect: NoSchedule
-        - key: "node-role.kubernetes.io/control-plane"
-          effect: NoSchedule
-        - key: "node.kubernetes.io/not-ready"
-          effect: "NoSchedule"
-      containers:
-        - image: hetznercloud/hcloud-cloud-controller-manager:v1.13.2
-          name: hcloud-cloud-controller-manager
-          command:
-            - "/bin/hcloud-cloud-controller-manager"
-            - "--cloud-provider=hcloud"
-            - "--leader-elect=false"
-            - "--allow-untagged-cloud"
-            - "--allocate-node-cidrs=false"
-          resources:
-            requests:
-              cpu: 100m
-              memory: 50Mi
-          env:
-            - name: HCLOUD_NETWORK_ROUTES_ENABLED
-              value: 'false'
-            - name: NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: HCLOUD_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: hcloud
-                  key: token
-            - name: HCLOUD_NETWORK
-              valueFrom:
-                secretKeyRef:
-                  name: hcloud
-                  key: network
-      priorityClassName: system-cluster-critical
--- a/k8s/roles/hcloud/templates/hcloud-csi.yml.j2
+++ b/k8s/roles/hcloud/templates/hcloud-csi.yml.j2
@ -1,394 +0,0 @@
---
-allowVolumeExpansion: true
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  annotations:
-    storageclass.kubernetes.io/is-default-class: "true"
-  name: hcloud-volumes
-provisioner: csi.hetzner.cloud
-volumeBindingMode: WaitForFirstConsumer
---
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: hcloud-csi-controller
-  namespace: kube-system
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: hcloud-csi-controller
-rules:
- apiGroups:
-  - ""
-  resources:
-  - persistentvolumes
-  verbs:
-  - get
-  - list
-  - watch
-  - update
-  - patch
- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - get
-  - list
-  - watch
- apiGroups:
-  - csi.storage.k8s.io
-  resources:
-  - csinodeinfos
-  verbs:
-  - get
-  - list
-  - watch
- apiGroups:
-  - storage.k8s.io
-  resources:
-  - csinodes
-  verbs:
-  - get
-  - list
-  - watch
- apiGroups:
-  - storage.k8s.io
-  resources:
-  - volumeattachments
-  verbs:
-  - get
-  - list
-  - watch
-  - update
-  - patch
- apiGroups:
-  - storage.k8s.io
-  resources:
-  - volumeattachments/status
-  verbs:
-  - patch
- apiGroups:
-  - ""
-  resources:
-  - secrets
-  verbs:
-  - get
-  - list
- apiGroups:
-  - ""
-  resources:
-  - persistentvolumes
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-  - patch
- apiGroups:
-  - ""
-  resources:
-  - persistentvolumeclaims
-  - persistentvolumeclaims/status
-  verbs:
-  - get
-  - list
-  - watch
-  - update
-  - patch
- apiGroups:
-  - storage.k8s.io
-  resources:
-  - storageclasses
-  verbs:
-  - get
-  - list
-  - watch
- apiGroups:
-  - ""
-  resources:
-  - events
-  verbs:
-  - list
-  - watch
-  - create
-  - update
-  - patch
- apiGroups:
-  - snapshot.storage.k8s.io
-  resources:
-  - volumesnapshots
-  verbs:
-  - get
-  - list
- apiGroups:
-  - snapshot.storage.k8s.io
-  resources:
-  - volumesnapshotcontents
-  verbs:
-  - get
-  - list
- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
- apiGroups:
-  - ""
-  resources:
-  - events
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - update
-  - patch
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: hcloud-csi-controller
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: hcloud-csi-controller
-subjects:
- kind: ServiceAccount
-  name: hcloud-csi-controller
-  namespace: kube-system
---
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: hcloud-csi-controller
-  name: hcloud-csi-controller-metrics
-  namespace: kube-system
-spec:
-  ports:
-  - name: metrics
-    port: 9189
-    targetPort: metrics
-  selector:
-    app: hcloud-csi-controller
---
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: hcloud-csi
-  name: hcloud-csi-node-metrics
-  namespace: kube-system
-spec:
-  ports:
-  - name: metrics
-    port: 9189
-    targetPort: metrics
-  selector:
-    app: hcloud-csi
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: hcloud-csi-controller
-  namespace: kube-system
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: hcloud-csi-controller
-  template:
-    metadata:
-      labels:
-        app: hcloud-csi-controller
-    spec:
-      containers:
-      - args:
-        - --default-fstype=ext4
-        image: registry.k8s.io/sig-storage/csi-attacher:v4.1.0
-        name: csi-attacher
-        volumeMounts:
-        - mountPath: /run/csi
-          name: socket-dir
-      - image: registry.k8s.io/sig-storage/csi-resizer:v1.7.0
-        name: csi-resizer
-        volumeMounts:
-        - mountPath: /run/csi
-          name: socket-dir
-      - args:
-        - --feature-gates=Topology=true
-        - --default-fstype=ext4
-        image: registry.k8s.io/sig-storage/csi-provisioner:v3.4.0
-        name: csi-provisioner
-        volumeMounts:
-        - mountPath: /run/csi
-          name: socket-dir
-      - command:
-        - /bin/hcloud-csi-driver-controller
-        env:
-        - name: CSI_ENDPOINT
-          value: unix:///run/csi/socket
-        - name: METRICS_ENDPOINT
-          value: 0.0.0.0:9189
-        - name: ENABLE_METRICS
-          value: "true"
-        - name: KUBE_NODE_NAME
-          valueFrom:
-            fieldRef:
-              apiVersion: v1
-              fieldPath: spec.nodeName
-        - name: HCLOUD_TOKEN
-          valueFrom:
-            secretKeyRef:
-              key: token
-              name: hcloud
-        image: hetznercloud/hcloud-csi-driver:v2.3.2
-        imagePullPolicy: Always
-        livenessProbe:
-          failureThreshold: 5
-          httpGet:
-            path: /healthz
-            port: healthz
-          initialDelaySeconds: 10
-          periodSeconds: 2
-          timeoutSeconds: 3
-        name: hcloud-csi-driver
-        ports:
-        - containerPort: 9189
-          name: metrics
-        - containerPort: 9808
-          name: healthz
-          protocol: TCP
-        volumeMounts:
-        - mountPath: /run/csi
-          name: socket-dir
-      - image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0
-        imagePullPolicy: Always
-        name: liveness-probe
-        volumeMounts:
-        - mountPath: /run/csi
-          name: socket-dir
-      serviceAccountName: hcloud-csi-controller
-      volumes:
-      - emptyDir: {}
-        name: socket-dir
---
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  labels:
-    app: hcloud-csi
-  name: hcloud-csi-node
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      app: hcloud-csi
-  template:
-    metadata:
-      labels:
-        app: hcloud-csi
-    spec:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-            - matchExpressions:
-              - key: instance.hetzner.cloud/is-root-server
-                operator: NotIn
-                values:
-                - "true"
-      containers:
-      - args:
-        - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.hetzner.cloud/socket
-        image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0
-        name: csi-node-driver-registrar
-        volumeMounts:
-        - mountPath: /run/csi
-          name: plugin-dir
-        - mountPath: /registration
-          name: registration-dir
-      - command:
-        - /bin/hcloud-csi-driver-node
-        env:
-        - name: CSI_ENDPOINT
-          value: unix:///run/csi/socket
-        - name: METRICS_ENDPOINT
-          value: 0.0.0.0:9189
-        - name: ENABLE_METRICS
-          value: "true"
-        image: hetznercloud/hcloud-csi-driver:v2.3.2
-        imagePullPolicy: Always
-        livenessProbe:
-          failureThreshold: 5
-          httpGet:
-            path: /healthz
-            port: healthz
-          initialDelaySeconds: 10
-          periodSeconds: 2
-          timeoutSeconds: 3
-        name: hcloud-csi-driver
-        ports:
-        - containerPort: 9189
-          name: metrics
-        - containerPort: 9808
-          name: healthz
-          protocol: TCP
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - mountPath: /var/lib/kubelet
-          mountPropagation: Bidirectional
-          name: kubelet-dir
-        - mountPath: /run/csi
-          name: plugin-dir
-        - mountPath: /dev
-          name: device-dir
-      - image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0
-        imagePullPolicy: Always
-        name: liveness-probe
-        volumeMounts:
-        - mountPath: /run/csi
-          name: plugin-dir
-      tolerations:
-      - effect: NoExecute
-        operator: Exists
-      - effect: NoSchedule
-        operator: Exists
-      - key: CriticalAddonsOnly
-        operator: Exists
-      volumes:
-      - hostPath:
-          path: /var/lib/kubelet
-          type: Directory
-        name: kubelet-dir
-      - hostPath:
-          path: /var/lib/kubelet/plugins/csi.hetzner.cloud/
-          type: DirectoryOrCreate
-        name: plugin-dir
-      - hostPath:
-          path: /var/lib/kubelet/plugins_registry/
-          type: Directory
-        name: registration-dir
-      - hostPath:
-          path: /dev
-          type: Directory
-        name: device-dir
---
-apiVersion: storage.k8s.io/v1
-kind: CSIDriver
-metadata:
-  name: csi.hetzner.cloud
-spec:
-  attachRequired: true
-  fsGroupPolicy: File
-  podInfoOnMount: true
-  volumeLifecycleModes:
-  - Persistent
--- a/k8s/roles/hcloud/templates/values.csi.yml.j2
+++ b/k8s/roles/hcloud/templates/values.csi.yml.j2
@ -0,0 +1,10 @@
+controller:
+  hcloudToken:
+    existingSecret:
+      name: hcloud
+      key: token
+
+metrics:
+  enabled: true
+  serviceMonitor:
+    enabled: true
--- a/k8s/roles/hcloud/templates/values.hccm.yml.j2
+++ b/k8s/roles/hcloud/templates/values.hccm.yml.j2
@ -4,4 +4,8 @@ monitoring:

 networking:
  enabled: true
-  clusterCIDR: 10.42.0.0/24
+  clusterCIDR: 10.42.0.0/24
+
+env:
+  HCLOUD_LOAD_BALANCERS_USE_PRIVATE_IP:
+    value: "true"