refactor: replace manual nodes with flatcar auto joined

infrastructure/configs/k3os-config.yml

@@ -1,51 +0,0 @@
-ssh_authorized_keys:
-- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKfHZaI0F5GjAcrM8hjWqwMfULDkAZ2TOIBTQtRocg1F
-hostname: ${host}
-write_files:
-- enconding: ""
-  content: |-
-    #!/bin/bash
-    write_log () {
-        local message="$1"
-        logger -t "run-cmd" "$message"
-        echo "$message"
-    }
-    write_log "Getting the service using eth0..."
-    ETH1=$(connmanctl services | awk '{ print $3 }' | while read -r s1; do connmanctl services $s1 | grep -q "eth1" && echo "$s1"; done)
-    write_log "eth1 is bound to: $ETH1"
-    write_log "Setting up manual net config..."
-    connmanctl config "$ETH1" --ipv4 manual ${node_ip} 255.255.254.0 172.23.2.10
-    connmanctl config "$ETH1" --domains k8s.local
-    connmanctl config "$ETH1" --ipv6 off
-    write_log "Restarting connman..."
-    service connman restart
-    write_log "$(connmanctl services $ETH1)"
-    write_log "Network setup done."
-  owner: root:root
-  path: /etc/run-cmd.sh
-  permissions: '0755'
-run_cmd:
-- "/etc/run-cmd.sh"
-
-k3os:
-  data_sources:
-  - hetzner
-  - cdrom
-  modules:
-  - kvm
-  - nvme
-  dns_nameservers:
-  - 1.1.1.1
-  - 1.0.0.1
-  ntp_servers:
-  - ptbtime1.ptb.de
-  - ptbtime2.ptb.de
-  server_url: https://172.23.2.10:6443
-  password: rancher
-  token: "${k3s_token}"
-  labels:
-    region: ${datacenter}
-  k3s_args:
-  - agent
-  - --node-ip=${node_ip}
-  - --kubelet-arg=cloud-provider=external

infrastructure/configs/k3s-flatcar.yaml

@@ -19,7 +19,7 @@ systemd:
         KillMode=process
         Environment="K3S_URL=https://172.23.2.10:6443"
         Environment="K3S_TOKEN=${k3s_token}"
-        Environment="INSTALL_K3S_EXEC='agent' '--node-ip' '${node_ip}' '--kubelet-arg' '--cloud-provider=external'"
+        Environment="INSTALL_K3S_EXEC=agent --node-ip=${node_ip} --kubelet-arg --cloud-provider=external"
         ExecStart=/usr/bin/sh -c "/opt/k3s-install.sh"
         [Install]
         WantedBy=multi-user.target
@@ -33,5 +33,4 @@ storage:
     - path: /opt/k3s-install.sh
       mode: 0777
       contents:
-        remote:
+        source: https://get.k3s.io
-          url: https://get.k3s.io

infrastructure/k8s_flatcar_machines.tf

@@ -59,17 +59,9 @@ resource "hcloud_server" "machine" {
     private_key = tls_private_key.provisioning.private_key_pem
     timeout     = "2m"
   }
-
+  
   provisioner "file" {
-    content = templatefile(
+    content     = data.ct_config.machine-ignitions[each.key].rendered
-      "${path.module}/configs/k3os-config.yml",
-      {
-        "host"       = "${each.key}"
-        "k3s_token"  = "${var.k3s_token}"
-        "datacenter" = "hel1-dc2"
-        "node_ip"    = "${each.value.private_ip}"
-      }
-    )
     destination = "/root/ignition.json"
   }
 
@@ -80,7 +72,7 @@ resource "hcloud_server" "machine" {
       "curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/flatcar/init/flatcar-master/bin/flatcar-install",
       "chmod +x flatcar-install",
       "./flatcar-install -s -i /root/ignition.json -C ${var.release_channel}",
-      "shutdown -r +1",
+      "reboot",
     ]
     on_failure = continue
   }

infrastructure/vms.auto.tfvars

@@ -2,8 +2,15 @@ k3os_workers = {
   "worker1-gen2" = {
     backups     = false
     node_type   = "worker"
-    server_type = "cpx11"
+    server_type = "cpx21"
-    private_ip  = "172.23.2.41"
+    private_ip  = "172.23.2.22"
+  }
+
+  "worker2-gen2" = {
+    backups     = false
+    node_type   = "worker"
+    server_type = "cpx21"
+    private_ip  = "172.23.2.23"
   }
 }
 
@@ -13,18 +20,6 @@ vms = {
     server_type = "cpx11",
     backups     = true,
     private_ip  = "172.23.2.10"
-  },
-  "worker1" = {
-    node_type   = "worker"
-    server_type = "cpx21"
-    backups     = false,
-    private_ip  = "172.23.2.20"
-  },
-  "worker2" = {
-    node_type   = "worker"
-    server_type = "cpx21"
-    backups     = false,
-    private_ip  = "172.23.2.21"
   }
 }
 
@@ -36,4 +31,6 @@ ci_workers = {
   }
 }
 
-ssh_keys = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKfHZaI0F5GjAcrM8hjWqwMfULDkAZ2TOIBTQtRocg1F id_ed25519"]
+ssh_keys = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKfHZaI0F5GjAcrM8hjWqwMfULDkAZ2TOIBTQtRocg1F id_ed25519"]
+
+release_channel = "stable"

k8s/roles/hcloud/templates/cloud-controller-manager.yml.j2

@@ -1,5 +1,3 @@
-# NOTE: this release was tested against kubernetes v1.18.x
-
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -67,6 +65,8 @@ spec:
               cpu: 100m
               memory: 50Mi
           env:
+            - name: HCLOUD_NETWORK_ROUTES_ENABLED
+              value: 'false'
             - name: NODE_NAME
               valueFrom:
                 fieldRef: