From 7697d2be659e0375f0c05dd5bf4293a4296663b7 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Sat, 21 Jan 2023 18:25:18 +0100 Subject: [PATCH] feat: prepare ephemeral workers --- .gitignore | 1 + infrastructure/configs/k3os-config.yml | 51 +++ infrastructure/k8s_cluster.tf | 1 - infrastructure/k8s_k3os_machines.tf | 86 ++++ infrastructure/lb.tf | 18 +- infrastructure/main.tf | 2 +- infrastructure/vars.tf | 13 + infrastructure/vms.auto.tfvars | 9 + k8s/configure_cluster.yaml | 12 +- k8s/inventory/clusters.yaml | 4 +- k8s/roles/csi/tasks/main.yml | 29 -- k8s/roles/hcloud/tasks/main.yml | 44 ++ .../templates/cloud-controller-manager.yml.j2 | 84 ++++ k8s/roles/hcloud/templates/hcloud-csi.yml.j2 | 392 ++++++++++++++++++ k8s/roles/k3s/node/templates/k3s.service.j2 | 2 +- k8s/roles/minio/tasks/main.yml | 2 +- k8s/roles/prometheus/tasks/main.yaml | 2 +- 17 files changed, 701 insertions(+), 51 deletions(-) create mode 100644 infrastructure/configs/k3os-config.yml create mode 100644 infrastructure/k8s_k3os_machines.tf delete mode 100644 k8s/roles/csi/tasks/main.yml create mode 100644 k8s/roles/hcloud/tasks/main.yml create mode 100644 k8s/roles/hcloud/templates/cloud-controller-manager.yml.j2 create mode 100644 k8s/roles/hcloud/templates/hcloud-csi.yml.j2 diff --git a/.gitignore b/.gitignore index b4544f8..ce9434e 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ .terraform/ .vaultpw .vscode/ +.ssh/ \ No newline at end of file diff --git a/infrastructure/configs/k3os-config.yml b/infrastructure/configs/k3os-config.yml new file mode 100644 index 0000000..6d0eda6 --- /dev/null +++ b/infrastructure/configs/k3os-config.yml @@ -0,0 +1,51 @@ +ssh_authorized_keys: +- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKfHZaI0F5GjAcrM8hjWqwMfULDkAZ2TOIBTQtRocg1F +hostname: ${host} +write_files: +- enconding: "" + content: |- + #!/bin/bash + write_log () { + local message="$1" + logger -t "run-cmd" "$message" + echo "$message" + } + write_log "Getting the service using eth0..." + ETH1=$(connmanctl services | awk '{ print $3 }' | while read -r s1; do connmanctl services $s1 | grep -q "eth1" && echo "$s1"; done) + write_log "eth1 is bound to: $ETH1" + write_log "Setting up manual net config..." + connmanctl config "$ETH1" --ipv4 manual ${node_ip} 255.255.254.0 172.23.2.10 + connmanctl config "$ETH1" --domains k8s.local + connmanctl config "$ETH1" --ipv6 off + write_log "Restarting connman..." + service connman restart + write_log "$(connmanctl services $ETH1)" + write_log "Network setup done." + owner: root:root + path: /etc/run-cmd.sh + permissions: '0755' +run_cmd: +- "/etc/run-cmd.sh" + +k3os: + data_sources: + - hetzner + - cdrom + modules: + - kvm + - nvme + dns_nameservers: + - 1.1.1.1 + - 1.0.0.1 + ntp_servers: + - ptbtime1.ptb.de + - ptbtime2.ptb.de + server_url: https://172.23.2.10:6443 + password: rancher + token: "${k3s_token}" + labels: + region: ${datacenter} + k3s_args: + - agent + - --node-ip=${node_ip} + - --kubelet-arg=cloud-provider=external \ No newline at end of file diff --git a/infrastructure/k8s_cluster.tf b/infrastructure/k8s_cluster.tf index 0e947a6..5a57cb8 100644 --- a/infrastructure/k8s_cluster.tf +++ b/infrastructure/k8s_cluster.tf @@ -40,7 +40,6 @@ resource "hcloud_server" "nodes" { } } - resource "hcloud_server_network" "k8s_internal" { for_each = var.vms diff --git a/infrastructure/k8s_k3os_machines.tf b/infrastructure/k8s_k3os_machines.tf new file mode 100644 index 0000000..0168244 --- /dev/null +++ b/infrastructure/k8s_k3os_machines.tf @@ -0,0 +1,86 @@ +resource "tls_private_key" "provisioning" { + algorithm = "RSA" + rsa_bits = 4096 +} + +resource "hcloud_ssh_key" "provisioning_key" { + name = "Provisioning key for hcloud cluster" + public_key = tls_private_key.provisioning.public_key_openssh +} + +resource "local_file" "provisioning_key" { + filename = "${path.module}/.ssh/provisioning_private_key.pem" + content = tls_private_key.provisioning.private_key_pem + directory_permission = "0700" + file_permission = "0400" +} + +resource "local_file" "provisioning_key_pub" { + filename = "${path.module}/.ssh/provisioning_key.pub" + content = tls_private_key.provisioning.public_key_openssh + directory_permission = "0700" + file_permission = "0440" +} + +resource "hcloud_server" "machine" { + for_each = var.k3os_workers + name = each.key + server_type = each.value.server_type + datacenter = "hel1-dc2" + image = "ubuntu-22.04" + + backups = each.value.backups + + ssh_keys = [ + hcloud_ssh_key.provisioning_key.id, + hcloud_ssh_key.default.id + ] + + labels = { + "node_type" = each.value.node_type + "cluster" = "icb4dc0.de" + } + + network { + network_id = hcloud_network.k8s_net.id + ip = each.value.private_ip + } + + public_net { + ipv4_enabled = true + ipv6_enabled = false + } + + # boot into rescue OS + rescue = "linux64" + + connection { + host = self.ipv4_address + private_key = tls_private_key.provisioning.private_key_pem + timeout = "2m" + } + + provisioner "file" { + content = templatefile( + "${path.module}/configs/k3os-config.yml", + { + "host" = "${each.key}" + "k3s_token" = "${var.k3s_token}" + "datacenter" = "hel1-dc2" + "node_ip" = "${each.value.private_ip}" + } + ) + destination = "/tmp/k3os-config.yaml" + } + + provisioner "remote-exec" { + inline = [ + "set -ex", + "apt-get install -y grub-pc", + "cat /tmp/k3os-config.yaml", + "curl -fsSL --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/rancher/k3os/master/install.sh | bash -s -- --config /tmp/k3os-config.yaml /dev/sda https://github.com/rancher/k3os/releases/download/v0.21.5-k3s2r1/k3os-amd64.iso", + "reboot" + ] + on_failure = continue + } +} \ No newline at end of file diff --git a/infrastructure/lb.tf b/infrastructure/lb.tf index 73b1984..826ea5d 100644 --- a/infrastructure/lb.tf +++ b/infrastructure/lb.tf @@ -31,15 +31,15 @@ resource "hcloud_load_balancer_service" "k8s_lb_svc_https" { health_check { protocol = "tcp" - port = 32080 + port = 32080 interval = 5 - timeout = 3 - retries = 3 + timeout = 3 + retries = 3 http { domain = "code.icb4dc0.de" - path = "/" - tls = false + path = "/" + tls = false status_codes = [ "2??", "3??" @@ -59,13 +59,13 @@ resource "hcloud_load_balancer_service" "k8s_lb_svc_ssh" { load_balancer_id = hcloud_load_balancer.k8s_lb.id protocol = "tcp" destination_port = 32022 - listen_port = 22 + listen_port = 22 health_check { protocol = "tcp" - port = 32022 + port = 32022 interval = 5 - timeout = 3 - retries = 3 + timeout = 3 + retries = 3 } } diff --git a/infrastructure/main.tf b/infrastructure/main.tf index e93a9e4..72a5ced 100644 --- a/infrastructure/main.tf +++ b/infrastructure/main.tf @@ -1,7 +1,7 @@ terraform { required_providers { hcloud = { - source = "hetznercloud/hcloud" + source = "hetznercloud/hcloud" version = "1.35.1" } } diff --git a/infrastructure/vars.tf b/infrastructure/vars.tf index dadebb4..c06b853 100644 --- a/infrastructure/vars.tf +++ b/infrastructure/vars.tf @@ -2,6 +2,10 @@ variable "hcloud_token" { sensitive = true } +variable "k3s_token" { + sensitive = true +} + variable "vms" { type = map(object({ node_type = string @@ -11,6 +15,15 @@ variable "vms" { })) } +variable "k3os_workers" { + type = map(object({ + node_type = string + server_type = string + backups = bool + private_ip = string + })) +} + variable "ci_workers" { type = map(object({ node_type = string diff --git a/infrastructure/vms.auto.tfvars b/infrastructure/vms.auto.tfvars index 9156906..9541c29 100644 --- a/infrastructure/vms.auto.tfvars +++ b/infrastructure/vms.auto.tfvars @@ -1,3 +1,12 @@ +k3os_workers = { + "worker1-gen2" = { + backups = false + node_type = "worker" + server_type = "cpx11" + private_ip = "172.23.2.41" + } +} + vms = { "cp1" = { node_type = "control-plane" diff --git a/k8s/configure_cluster.yaml b/k8s/configure_cluster.yaml index f42359b..be47a06 100644 --- a/k8s/configure_cluster.yaml +++ b/k8s/configure_cluster.yaml @@ -2,9 +2,9 @@ hosts: localhost roles: # - role: gateway-api - - role: prometheus - - role: postgres - - role: csi - - role: minio - - role: gitea - - role: concourse + - role: prometheus + - role: postgres + - role: hcloud + - role: minio + - role: gitea + - role: concourse diff --git a/k8s/inventory/clusters.yaml b/k8s/inventory/clusters.yaml index 193adc0..92f4035 100644 --- a/k8s/inventory/clusters.yaml +++ b/k8s/inventory/clusters.yaml @@ -1,8 +1,8 @@ all: vars: ansible_user: root - extra_server_args: "--node-taint=node-type=master:NoSchedule --tls-san='2a01:4f9:c012:7d4b::1' --tls-san='k8s.icb4dc0.de' --tls-san='127.0.0.1'" - extra_agent_args: "" + extra_server_args: "--node-taint=node-type=master:NoSchedule --tls-san='2a01:4f9:c012:7d4b::1' --tls-san='k8s.icb4dc0.de' --tls-san='127.0.0.1' --disable-cloud-controller --disable servicelb --kubelet-arg='--cloud-provider=external'" + extra_agent_args: "--kubelet-arg='--cloud-provider=external'" ansible_ssh_common_args: '-o StrictHostKeyChecking=no' systemd_dir: /etc/systemd/system master_ip: "172.23.2.10" diff --git a/k8s/roles/csi/tasks/main.yml b/k8s/roles/csi/tasks/main.yml deleted file mode 100644 index 63b9cd2..0000000 --- a/k8s/roles/csi/tasks/main.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -- name: Create Hcloud token secret - kubernetes.core.k8s: - state: present - definition: - apiVersion: v1 - kind: Secret - metadata: - name: hcloud-csi - namespace: kube-system - data: - token: "{{ HcloudToken | b64encode }}" - -- name: Create temporary file - ansible.builtin.tempfile: - state: file - suffix: temp - register: csi_manifest_tmp - -- name: Download CSI manifest - ansible.builtin.get_url: - url: https://raw.githubusercontent.com/hetznercloud/csi-driver/v1.6.0/deploy/kubernetes/hcloud-csi.yml - dest: "{{ csi_manifest_tmp.path }}" - mode: '0664' - -- name: Deploy CSI driver - kubernetes.core.k8s: - state: present - src: "{{ csi_manifest_tmp.path }}" diff --git a/k8s/roles/hcloud/tasks/main.yml b/k8s/roles/hcloud/tasks/main.yml new file mode 100644 index 0000000..8a625d9 --- /dev/null +++ b/k8s/roles/hcloud/tasks/main.yml @@ -0,0 +1,44 @@ +--- +- name: Create Hcloud token secret + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Secret + metadata: + name: hcloud + namespace: kube-system + data: + token: "{{ HcloudToken | b64encode }}" + network: "{{ 'k8s-net' | b64encode }}" + +- name: Deploy CSI driver + kubernetes.core.k8s: + state: present + definition: "{{ item }}" + loop: "{{ lookup('ansible.builtin.template', 'hcloud-csi.yml.j2') | ansible.builtin.from_yaml_all | list }}" + +- name: Deploy cloud-controller-manager + kubernetes.core.k8s: + state: present + definition: "{{ item }}" + loop: "{{ lookup('ansible.builtin.template', 'cloud-controller-manager.yml.j2') | ansible.builtin.from_yaml_all | list }}" + +- name: Create CSI controller PodMonitor + kubernetes.core.k8s: + state: present + definition: + apiVersion: monitoring.coreos.com/v1 + kind: PodMonitor + metadata: + name: hcloud-csi-controller + namespace: kube-system + labels: + prometheus: default + spec: + selector: + matchLabels: + app: hcloud-csi-controller + podMetricsEndpoints: + - port: metrics + path: / diff --git a/k8s/roles/hcloud/templates/cloud-controller-manager.yml.j2 b/k8s/roles/hcloud/templates/cloud-controller-manager.yml.j2 new file mode 100644 index 0000000..7dc2768 --- /dev/null +++ b/k8s/roles/hcloud/templates/cloud-controller-manager.yml.j2 @@ -0,0 +1,84 @@ +# NOTE: this release was tested against kubernetes v1.18.x + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: kube-system +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hcloud-cloud-controller-manager + namespace: kube-system +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: hcloud-cloud-controller-manager + template: + metadata: + labels: + app: hcloud-cloud-controller-manager + spec: + serviceAccountName: cloud-controller-manager + dnsPolicy: Default + tolerations: + # this taint is set by all kubelets running `--cloud-provider=external` + # so we should tolerate it to schedule the cloud controller manager + - key: "node.cloudprovider.kubernetes.io/uninitialized" + value: "true" + effect: "NoSchedule" + - key: "CriticalAddonsOnly" + operator: "Exists" + # cloud controller manages should be able to run on masters + - key: "node-role.kubernetes.io/master" + effect: NoSchedule + - key: "node-role.kubernetes.io/control-plane" + effect: NoSchedule + - key: "node.kubernetes.io/not-ready" + effect: "NoSchedule" + containers: + - image: hetznercloud/hcloud-cloud-controller-manager:v1.13.2 + name: hcloud-cloud-controller-manager + command: + - "/bin/hcloud-cloud-controller-manager" + - "--cloud-provider=hcloud" + - "--leader-elect=false" + - "--allow-untagged-cloud" + - "--allocate-node-cidrs=false" + resources: + requests: + cpu: 100m + memory: 50Mi + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: HCLOUD_TOKEN + valueFrom: + secretKeyRef: + name: hcloud + key: token + - name: HCLOUD_NETWORK + valueFrom: + secretKeyRef: + name: hcloud + key: network + priorityClassName: system-cluster-critical \ No newline at end of file diff --git a/k8s/roles/hcloud/templates/hcloud-csi.yml.j2 b/k8s/roles/hcloud/templates/hcloud-csi.yml.j2 new file mode 100644 index 0000000..cfe0307 --- /dev/null +++ b/k8s/roles/hcloud/templates/hcloud-csi.yml.j2 @@ -0,0 +1,392 @@ +--- +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "true" + name: hcloud-volumes +provisioner: csi.hetzner.cloud +volumeBindingMode: WaitForFirstConsumer +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hcloud-csi-controller + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hcloud-csi-controller +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - patch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + - persistentvolumeclaims/status + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hcloud-csi-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hcloud-csi-controller +subjects: +- kind: ServiceAccount + name: hcloud-csi-controller + namespace: kube-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: hcloud-csi + name: hcloud-csi-controller-metrics + namespace: kube-system +spec: + ports: + - name: metrics + port: 9189 + targetPort: metrics + selector: + app: hcloud-csi-controller +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: hcloud-csi + name: hcloud-csi-node-metrics + namespace: kube-system +spec: + ports: + - name: metrics + port: 9189 + targetPort: metrics + selector: + app: hcloud-csi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hcloud-csi-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: hcloud-csi-controller + template: + metadata: + labels: + app: hcloud-csi-controller + spec: + containers: + - image: k8s.gcr.io/sig-storage/csi-attacher:v3.2.1 + name: csi-attacher + volumeMounts: + - mountPath: /run/csi + name: socket-dir + - image: k8s.gcr.io/sig-storage/csi-resizer:v1.2.0 + name: csi-resizer + volumeMounts: + - mountPath: /run/csi + name: socket-dir + - args: + - --feature-gates=Topology=true + - --default-fstype=ext4 + image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.2 + name: csi-provisioner + volumeMounts: + - mountPath: /run/csi + name: socket-dir + - command: + - /bin/hcloud-csi-driver-controller + env: + - name: CSI_ENDPOINT + value: unix:///run/csi/socket + - name: METRICS_ENDPOINT + value: 0.0.0.0:9189 + - name: ENABLE_METRICS + value: "true" + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: HCLOUD_TOKEN + valueFrom: + secretKeyRef: + key: token + name: hcloud + image: hetznercloud/hcloud-csi-driver:2.1.0 + imagePullPolicy: Always + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 2 + timeoutSeconds: 3 + name: hcloud-csi-driver + ports: + - containerPort: 9189 + name: metrics + - containerPort: 9808 + name: healthz + protocol: TCP + volumeMounts: + - mountPath: /run/csi + name: socket-dir + - image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0 + imagePullPolicy: Always + name: liveness-probe + volumeMounts: + - mountPath: /run/csi + name: socket-dir + serviceAccountName: hcloud-csi-controller + volumes: + - emptyDir: {} + name: socket-dir +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: hcloud-csi + name: hcloud-csi-node + namespace: kube-system +spec: + selector: + matchLabels: + app: hcloud-csi + template: + metadata: + labels: + app: hcloud-csi + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: instance.hetzner.cloud/is-root-server + operator: NotIn + values: + - "true" + containers: + - args: + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.hetzner.cloud/socket + image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0 + name: csi-node-driver-registrar + volumeMounts: + - mountPath: /run/csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - command: + - /bin/hcloud-csi-driver-node + env: + - name: CSI_ENDPOINT + value: unix:///run/csi/socket + - name: METRICS_ENDPOINT + value: 0.0.0.0:9189 + - name: ENABLE_METRICS + value: "true" + image: hetznercloud/hcloud-csi-driver:2.1.0 + imagePullPolicy: Always + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 2 + timeoutSeconds: 3 + name: hcloud-csi-driver + ports: + - containerPort: 9189 + name: metrics + - containerPort: 9808 + name: healthz + protocol: TCP + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /run/csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0 + imagePullPolicy: Always + name: liveness-probe + volumeMounts: + - mountPath: /run/csi + name: plugin-dir + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/csi.hetzner.cloud/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: csi.hetzner.cloud +spec: + attachRequired: true + podInfoOnMount: true + volumeLifecycleModes: + - Persistent + fsGroupPolicy: File diff --git a/k8s/roles/k3s/node/templates/k3s.service.j2 b/k8s/roles/k3s/node/templates/k3s.service.j2 index d5a6cd5..02dc591 100644 --- a/k8s/roles/k3s/node/templates/k3s.service.j2 +++ b/k8s/roles/k3s/node/templates/k3s.service.j2 @@ -7,7 +7,7 @@ After=network-online.target Type=notify ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/local/bin/k3s agent --server https://{{ master_ip }}:6443 --node-ip {{ k8s_ip}} --token {{ hostvars[groups['control_plane'][0]]['token'] }} {{ extra_agent_args | default("") }} +ExecStart=/usr/local/bin/k3s agent --server https://{{ master_ip }}:6443 --node-ip {{ k8s_ip}} --kubelet-arg="provider-id=hcloud://{{ vm_id }}" --token {{ hostvars[groups['control_plane'][0]]['token'] }} {{ extra_agent_args | default("") }} KillMode=process Delegate=yes # Having non-zero Limit*s causes performance problems due to accounting overhead diff --git a/k8s/roles/minio/tasks/main.yml b/k8s/roles/minio/tasks/main.yml index 61560b6..ab46056 100644 --- a/k8s/roles/minio/tasks/main.yml +++ b/k8s/roles/minio/tasks/main.yml @@ -34,5 +34,5 @@ name: minio chart_ref: minio/minio release_namespace: minio - chart_version: 5.0.0 + chart_version: 5.0.4 release_values: "{{ lookup('ansible.builtin.file', 'values.minio.yaml') | from_yaml }}" diff --git a/k8s/roles/prometheus/tasks/main.yaml b/k8s/roles/prometheus/tasks/main.yaml index 37b24be..85d7647 100644 --- a/k8s/roles/prometheus/tasks/main.yaml +++ b/k8s/roles/prometheus/tasks/main.yaml @@ -33,5 +33,5 @@ name: prometheus chart_ref: prometheus-community/kube-prometheus-stack release_namespace: observability-system - chart_version: 43.2.0 + chart_version: 43.2.1 release_values: "{{ lookup('ansible.builtin.template', 'values.yaml.j2') | from_yaml }}" \ No newline at end of file