From 420a6d3489768cc47d0d6c1e3a289db86bb38946 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Wed, 14 Feb 2024 21:42:42 +0100 Subject: [PATCH] feat: move to upstream Hcloud charts --- infrastructure/configs/cp/traefik.yaml | 19 +- infrastructure/vms.auto.tfvars | 6 +- k8s/roles/fider/files/resources/ingress.yaml | 11 +- k8s/roles/hcloud/tasks/main.yml | 34 +- .../templates/cloud-controller-manager.yml.j2 | 84 ---- k8s/roles/hcloud/templates/hcloud-csi.yml.j2 | 394 ------------------ k8s/roles/hcloud/templates/values.csi.yml.j2 | 10 + k8s/roles/hcloud/templates/values.hccm.yml.j2 | 6 +- 8 files changed, 51 insertions(+), 513 deletions(-) delete mode 100644 k8s/roles/hcloud/templates/cloud-controller-manager.yml.j2 delete mode 100644 k8s/roles/hcloud/templates/hcloud-csi.yml.j2 create mode 100644 k8s/roles/hcloud/templates/values.csi.yml.j2 diff --git a/infrastructure/configs/cp/traefik.yaml b/infrastructure/configs/cp/traefik.yaml index 7544d3c..d56caa9 100644 --- a/infrastructure/configs/cp/traefik.yaml +++ b/infrastructure/configs/cp/traefik.yaml @@ -6,24 +6,33 @@ metadata: spec: chart: traefik repo: https://traefik.github.io/charts - version: 24.0.0 + version: 26.0.0 valuesContent: |- ports: traefik: port: 9000 - exposedPort: 9000 - expose: true + expose: false web: nodePort: 32080 forwardedHeaders: insecure: true websecure: - expose: false + expose: true service: - type: NodePort + type: LoadBalancer + annotations: + load-balancer.hetzner.cloud/location: "hel1" experimental: kubernetesGateway: enabled: true + providers: + kubernetesIngress: + publishedService: + enabled: true + allowExternalNameServices: true + kubernetesCRD: + enabled: true + allowExternalNameServices: true metrics: prometheus: serviceMonitor: diff --git a/infrastructure/vms.auto.tfvars b/infrastructure/vms.auto.tfvars index 1e25eb9..440e8b5 100644 --- a/infrastructure/vms.auto.tfvars +++ b/infrastructure/vms.auto.tfvars @@ -1,9 +1,9 @@ k3s_control_plane = { - "cp1-cax11-hel1-gen2" = { + "cp1-cax11-hel1-gen3" = { server_type = "cax11", - private_ip = "172.23.2.11" + private_ip = "172.23.2.10" location = "hel1" - alias_ips = ["172.23.2.10"] + alias_ips = [] } } diff --git a/k8s/roles/fider/files/resources/ingress.yaml b/k8s/roles/fider/files/resources/ingress.yaml index 77d8259..38825ac 100644 --- a/k8s/roles/fider/files/resources/ingress.yaml +++ b/k8s/roles/fider/files/resources/ingress.yaml @@ -3,6 +3,8 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: fider + annotations: + cert-manager.io/cluster-issuer: letsencrypt-production spec: rules: - host: fider.icb4dc0.de @@ -44,4 +46,11 @@ spec: service: name: fider port: - number: 3000 \ No newline at end of file + number: 3000 + tls: + - hosts: + - fider.icb4dc0.de + - login.fider.icb4dc0.de + - community.buildr.icb4dc0.de + - community.inetmock.icb4dc0.de + secretName: fider-ingress-tls \ No newline at end of file diff --git a/k8s/roles/hcloud/tasks/main.yml b/k8s/roles/hcloud/tasks/main.yml index 03eda35..f44a412 100644 --- a/k8s/roles/hcloud/tasks/main.yml +++ b/k8s/roles/hcloud/tasks/main.yml @@ -12,12 +12,6 @@ token: "{{ HcloudToken | b64encode }}" network: "{{ 'k8s-net' | b64encode }}" -- name: Deploy CSI driver - kubernetes.core.k8s: - state: present - definition: "{{ item }}" - loop: "{{ lookup('ansible.builtin.template', 'hcloud-csi.yml.j2') | ansible.builtin.from_yaml_all | list }}" - - name: Add Hcloud chart repo kubernetes.core.helm_repository: name: hcloud @@ -28,24 +22,14 @@ name: hccm chart_ref: hcloud/hcloud-cloud-controller-manager release_namespace: kube-system - chart_version: "1.17.0" + chart_version: "1.19.0" release_values: "{{ lookup('template', 'values.hccm.yml.j2') | from_yaml }}" -- name: Create CSI controller PodMonitor - kubernetes.core.k8s: - state: present - definition: - apiVersion: monitoring.coreos.com/v1 - kind: PodMonitor - metadata: - name: hcloud-csi-controller - namespace: kube-system - labels: - prometheus: default - spec: - selector: - matchLabels: - app: hcloud-csi-controller - podMetricsEndpoints: - - port: metrics - path: / +- name: Deploy hcloud CSI driver + kubernetes.core.helm: + name: hcloud-csi-driver + chart_ref: hcloud/hcloud-csi + release_namespace: kube-system + chart_version: "2.6.0" + release_values: "{{ lookup('template', 'values.csi.yml.j2') | from_yaml }}" + diff --git a/k8s/roles/hcloud/templates/cloud-controller-manager.yml.j2 b/k8s/roles/hcloud/templates/cloud-controller-manager.yml.j2 deleted file mode 100644 index 6313896..0000000 --- a/k8s/roles/hcloud/templates/cloud-controller-manager.yml.j2 +++ /dev/null @@ -1,84 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cloud-controller-manager - namespace: kube-system ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: system:cloud-controller-manager -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: - - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: hcloud-cloud-controller-manager - namespace: kube-system -spec: - replicas: 1 - revisionHistoryLimit: 2 - selector: - matchLabels: - app: hcloud-cloud-controller-manager - template: - metadata: - labels: - app: hcloud-cloud-controller-manager - spec: - serviceAccountName: cloud-controller-manager - dnsPolicy: Default - tolerations: - # this taint is set by all kubelets running `--cloud-provider=external` - # so we should tolerate it to schedule the cloud controller manager - - key: "node.cloudprovider.kubernetes.io/uninitialized" - value: "true" - effect: "NoSchedule" - - key: "CriticalAddonsOnly" - operator: "Exists" - # cloud controller manages should be able to run on masters - - key: "node-role.kubernetes.io/master" - effect: NoSchedule - - key: "node-role.kubernetes.io/control-plane" - effect: NoSchedule - - key: "node.kubernetes.io/not-ready" - effect: "NoSchedule" - containers: - - image: hetznercloud/hcloud-cloud-controller-manager:v1.13.2 - name: hcloud-cloud-controller-manager - command: - - "/bin/hcloud-cloud-controller-manager" - - "--cloud-provider=hcloud" - - "--leader-elect=false" - - "--allow-untagged-cloud" - - "--allocate-node-cidrs=false" - resources: - requests: - cpu: 100m - memory: 50Mi - env: - - name: HCLOUD_NETWORK_ROUTES_ENABLED - value: 'false' - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: HCLOUD_TOKEN - valueFrom: - secretKeyRef: - name: hcloud - key: token - - name: HCLOUD_NETWORK - valueFrom: - secretKeyRef: - name: hcloud - key: network - priorityClassName: system-cluster-critical \ No newline at end of file diff --git a/k8s/roles/hcloud/templates/hcloud-csi.yml.j2 b/k8s/roles/hcloud/templates/hcloud-csi.yml.j2 deleted file mode 100644 index bd06800..0000000 --- a/k8s/roles/hcloud/templates/hcloud-csi.yml.j2 +++ /dev/null @@ -1,394 +0,0 @@ ---- -allowVolumeExpansion: true -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - annotations: - storageclass.kubernetes.io/is-default-class: "true" - name: hcloud-volumes -provisioner: csi.hetzner.cloud -volumeBindingMode: WaitForFirstConsumer ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: hcloud-csi-controller - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: hcloud-csi-controller -rules: -- apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - watch - - update - - patch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - csi.storage.k8s.io - resources: - - csinodeinfos - verbs: - - get - - list - - watch -- apiGroups: - - storage.k8s.io - resources: - - csinodes - verbs: - - get - - list - - watch -- apiGroups: - - storage.k8s.io - resources: - - volumeattachments - verbs: - - get - - list - - watch - - update - - patch -- apiGroups: - - storage.k8s.io - resources: - - volumeattachments/status - verbs: - - patch -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list -- apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - watch - - create - - delete - - patch -- apiGroups: - - "" - resources: - - persistentvolumeclaims - - persistentvolumeclaims/status - verbs: - - get - - list - - watch - - update - - patch -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - list - - watch - - create - - update - - patch -- apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshots - verbs: - - get - - list -- apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshotcontents - verbs: - - get - - list -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - get - - list - - watch - - create - - update - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: hcloud-csi-controller -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: hcloud-csi-controller -subjects: -- kind: ServiceAccount - name: hcloud-csi-controller - namespace: kube-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: hcloud-csi-controller - name: hcloud-csi-controller-metrics - namespace: kube-system -spec: - ports: - - name: metrics - port: 9189 - targetPort: metrics - selector: - app: hcloud-csi-controller ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: hcloud-csi - name: hcloud-csi-node-metrics - namespace: kube-system -spec: - ports: - - name: metrics - port: 9189 - targetPort: metrics - selector: - app: hcloud-csi ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: hcloud-csi-controller - namespace: kube-system -spec: - replicas: 1 - selector: - matchLabels: - app: hcloud-csi-controller - template: - metadata: - labels: - app: hcloud-csi-controller - spec: - containers: - - args: - - --default-fstype=ext4 - image: registry.k8s.io/sig-storage/csi-attacher:v4.1.0 - name: csi-attacher - volumeMounts: - - mountPath: /run/csi - name: socket-dir - - image: registry.k8s.io/sig-storage/csi-resizer:v1.7.0 - name: csi-resizer - volumeMounts: - - mountPath: /run/csi - name: socket-dir - - args: - - --feature-gates=Topology=true - - --default-fstype=ext4 - image: registry.k8s.io/sig-storage/csi-provisioner:v3.4.0 - name: csi-provisioner - volumeMounts: - - mountPath: /run/csi - name: socket-dir - - command: - - /bin/hcloud-csi-driver-controller - env: - - name: CSI_ENDPOINT - value: unix:///run/csi/socket - - name: METRICS_ENDPOINT - value: 0.0.0.0:9189 - - name: ENABLE_METRICS - value: "true" - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: HCLOUD_TOKEN - valueFrom: - secretKeyRef: - key: token - name: hcloud - image: hetznercloud/hcloud-csi-driver:v2.3.2 - imagePullPolicy: Always - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - periodSeconds: 2 - timeoutSeconds: 3 - name: hcloud-csi-driver - ports: - - containerPort: 9189 - name: metrics - - containerPort: 9808 - name: healthz - protocol: TCP - volumeMounts: - - mountPath: /run/csi - name: socket-dir - - image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 - imagePullPolicy: Always - name: liveness-probe - volumeMounts: - - mountPath: /run/csi - name: socket-dir - serviceAccountName: hcloud-csi-controller - volumes: - - emptyDir: {} - name: socket-dir ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - app: hcloud-csi - name: hcloud-csi-node - namespace: kube-system -spec: - selector: - matchLabels: - app: hcloud-csi - template: - metadata: - labels: - app: hcloud-csi - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: instance.hetzner.cloud/is-root-server - operator: NotIn - values: - - "true" - containers: - - args: - - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.hetzner.cloud/socket - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0 - name: csi-node-driver-registrar - volumeMounts: - - mountPath: /run/csi - name: plugin-dir - - mountPath: /registration - name: registration-dir - - command: - - /bin/hcloud-csi-driver-node - env: - - name: CSI_ENDPOINT - value: unix:///run/csi/socket - - name: METRICS_ENDPOINT - value: 0.0.0.0:9189 - - name: ENABLE_METRICS - value: "true" - image: hetznercloud/hcloud-csi-driver:v2.3.2 - imagePullPolicy: Always - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - periodSeconds: 2 - timeoutSeconds: 3 - name: hcloud-csi-driver - ports: - - containerPort: 9189 - name: metrics - - containerPort: 9808 - name: healthz - protocol: TCP - securityContext: - privileged: true - volumeMounts: - - mountPath: /var/lib/kubelet - mountPropagation: Bidirectional - name: kubelet-dir - - mountPath: /run/csi - name: plugin-dir - - mountPath: /dev - name: device-dir - - image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 - imagePullPolicy: Always - name: liveness-probe - volumeMounts: - - mountPath: /run/csi - name: plugin-dir - tolerations: - - effect: NoExecute - operator: Exists - - effect: NoSchedule - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - volumes: - - hostPath: - path: /var/lib/kubelet - type: Directory - name: kubelet-dir - - hostPath: - path: /var/lib/kubelet/plugins/csi.hetzner.cloud/ - type: DirectoryOrCreate - name: plugin-dir - - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: Directory - name: registration-dir - - hostPath: - path: /dev - type: Directory - name: device-dir ---- -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: csi.hetzner.cloud -spec: - attachRequired: true - fsGroupPolicy: File - podInfoOnMount: true - volumeLifecycleModes: - - Persistent diff --git a/k8s/roles/hcloud/templates/values.csi.yml.j2 b/k8s/roles/hcloud/templates/values.csi.yml.j2 new file mode 100644 index 0000000..5c37545 --- /dev/null +++ b/k8s/roles/hcloud/templates/values.csi.yml.j2 @@ -0,0 +1,10 @@ +controller: + hcloudToken: + existingSecret: + name: hcloud + key: token + +metrics: + enabled: true + serviceMonitor: + enabled: true diff --git a/k8s/roles/hcloud/templates/values.hccm.yml.j2 b/k8s/roles/hcloud/templates/values.hccm.yml.j2 index b07e63d..ca8c5e1 100644 --- a/k8s/roles/hcloud/templates/values.hccm.yml.j2 +++ b/k8s/roles/hcloud/templates/values.hccm.yml.j2 @@ -4,4 +4,8 @@ monitoring: networking: enabled: true - clusterCIDR: 10.42.0.0/24 \ No newline at end of file + clusterCIDR: 10.42.0.0/24 + +env: + HCLOUD_LOAD_BALANCERS_USE_PRIVATE_IP: + value: "true" \ No newline at end of file