From 9bd3f45172e5e53f40bc574f38c53d80b4e6a726 Mon Sep 17 00:00:00 2001 From: Peter Kurfer Date: Fri, 22 Mar 2024 11:28:40 +0100 Subject: [PATCH] feat: upgrade K8s to 1.29 - replace Hetzner DNS with CloudFlare --- infrastructure/dns.tf | 115 ++++++++----------------- infrastructure/k8s_control_plane.tf | 36 +++++++- infrastructure/k8s_flatcar_machines.tf | 12 ++- infrastructure/lb.tf | 79 ----------------- infrastructure/main.tf | 4 + infrastructure/tf.sh | 12 +-- infrastructure/vars.tf | 16 +++- infrastructure/versions.tf | 10 ++- infrastructure/vms.auto.tfvars | 8 +- 9 files changed, 115 insertions(+), 177 deletions(-) delete mode 100644 infrastructure/lb.tf diff --git a/infrastructure/dns.tf b/infrastructure/dns.tf index 331387d..e48f42f 100644 --- a/infrastructure/dns.tf +++ b/infrastructure/dns.tf @@ -1,91 +1,46 @@ -resource "hetznerdns_zone" "icb4dc0de" { - name = "icb4dc0.de" - ttl = 86400 +resource "cloudflare_zone" "icb4dc0de" { + account_id = var.cloudflare_account_id + zone = "icb4dc0.de" + + lifecycle { + ignore_changes = [ account_id ] + } } -resource "hetznerdns_record" "ns_primary" { - zone_id = hetznerdns_zone.icb4dc0de.id - name = "@" - type = "NS" - value = "helium.ns.hetzner.de." +resource "cloudflare_record" "mx_primary" { + zone_id = cloudflare_zone.icb4dc0de.id + name = "@" + type = "MX" + value = "mx01.mail.icloud.com" + priority = 10 } -resource "hetznerdns_record" "ns_secondary" { - zone_id = hetznerdns_zone.icb4dc0de.id - name = "@" - type = "NS" - value = "oxygen.ns.hetzner.com." + +resource "cloudflare_record" "mx_secondary" { + zone_id = cloudflare_zone.icb4dc0de.id + name = "@" + type = "MX" + value = "mx02.mail.icloud.com" + priority = 10 } -resource "hetznerdns_record" "ns_ternary" { - zone_id = hetznerdns_zone.icb4dc0de.id - name = "@" - type = "NS" - value = "hydrogen.ns.hetzner.com." +resource "cloudflare_record" "apple_proof" { + zone_id = cloudflare_zone.icb4dc0de.id + name = "@" + type = "TXT" + value = "apple-domain=chwbVvzH8hWIgg1l" } -resource "hetznerdns_record" "soa" { - zone_id = hetznerdns_zone.icb4dc0de.id - name = "@" - type = "SOA" - value = "hydrogen.ns.hetzner.com. dns.hetzner.com. 2023120305 86400 10800 3600000 3600" +resource "cloudflare_record" "apple_spf" { + zone_id = cloudflare_zone.icb4dc0de.id + name = "@" + type = "TXT" + value = "\"v=spf1 include:icloud.com ~all\"" } -resource "hetznerdns_record" "mx_primary" { - zone_id = hetznerdns_zone.icb4dc0de.id - name = "@" - type = "MX" - value = "10 mx01.mail.icloud.com." -} - -resource "hetznerdns_record" "mx_secondary" { - zone_id = hetznerdns_zone.icb4dc0de.id - name = "@" - type = "MX" - value = "10 mx02.mail.icloud.com." -} - -resource "hetznerdns_record" "apple_proof" { - zone_id = hetznerdns_zone.icb4dc0de.id - name = "@" - type = "TXT" - value = "apple-domain=chwbVvzH8hWIgg1l" -} - -resource "hetznerdns_record" "apple_spf" { - zone_id = hetznerdns_zone.icb4dc0de.id - name = "@" - type = "TXT" - value = "\"v=spf1 include:icloud.com ~all\"" -} - -resource "hetznerdns_record" "apple_sig_domainkey" { - zone_id = hetznerdns_zone.icb4dc0de.id - name = "sig1._domainkey" - type = "CNAME" - value = "sig1.dkim.icb4dc0.de.at.icloudmailadmin.com." -} - -resource "hetznerdns_record" "wildcard_ipv4" { - zone_id = hetznerdns_zone.icb4dc0de.id - name = "*" - type = "A" - value = "65.109.42.5" - ttl = 300 -} - -resource "hetznerdns_record" "wildcard_ipv6" { - zone_id = hetznerdns_zone.icb4dc0de.id - name = "*" - type = "AAAA" - value = "2a01:4f9:c01d:4f1::1" - ttl = 300 -} - -resource "hetznerdns_record" "k8s" { - zone_id = hetznerdns_zone.icb4dc0de.id - name = "k8s" - type = "AAAA" - value = "2a01:4f9:c012:7d4b::1" - ttl = 60 +resource "cloudflare_record" "apple_sig_domainkey" { + zone_id = cloudflare_zone.icb4dc0de.id + name = "sig1._domainkey" + type = "CNAME" + value = "sig1.dkim.icb4dc0.de.at.icloudmailadmin.com" } \ No newline at end of file diff --git a/infrastructure/k8s_control_plane.tf b/infrastructure/k8s_control_plane.tf index de7d43f..a6be94c 100644 --- a/infrastructure/k8s_control_plane.tf +++ b/infrastructure/k8s_control_plane.tf @@ -1,3 +1,9 @@ +resource "null_resource" "cp-config" { + triggers = { + version = var.control_plane_k3s_version + } +} + resource "hcloud_server" "control-plane" { for_each = var.k3s_control_plane name = each.key @@ -7,6 +13,10 @@ resource "hcloud_server" "control-plane" { backups = false + lifecycle { + replace_triggered_by = [ null_resource.cp-config ] + } + ssh_keys = [ hcloud_ssh_key.provisioning_key.id, hcloud_ssh_key.default.id @@ -25,7 +35,7 @@ resource "hcloud_server" "control-plane" { public_net { ipv4_enabled = true - ipv6_enabled = false + ipv6_enabled = true } # boot into rescue OS @@ -34,7 +44,7 @@ resource "hcloud_server" "control-plane" { connection { host = self.ipv4_address private_key = tls_private_key.provisioning.private_key_pem - timeout = "2m" + timeout = "5m" } provisioner "file" { @@ -68,6 +78,28 @@ resource "hcloud_server" "control-plane" { } } +resource "cloudflare_record" "cp-host-ipv4" { + for_each = var.k3s_control_plane + + depends_on = [ hcloud_server.control-plane ] + + zone_id = cloudflare_zone.icb4dc0de.id + name = "${each.key}.k8s" + type = "A" + value = hcloud_server.control-plane[each.key].ipv4_address +} + +resource "cloudflare_record" "cp-host-ipv6" { + for_each = var.k3s_control_plane + + depends_on = [ hcloud_server.control-plane ] + + zone_id = cloudflare_zone.icb4dc0de.id + name = "${each.key}.k8s" + type = "AAAA" + value = hcloud_server.control-plane[each.key].ipv6_address +} + data "ct_config" "machine-ignitions-cp" { for_each = var.k3s_control_plane strict = true diff --git a/infrastructure/k8s_flatcar_machines.tf b/infrastructure/k8s_flatcar_machines.tf index f3e97aa..73ecf3c 100644 --- a/infrastructure/k8s_flatcar_machines.tf +++ b/infrastructure/k8s_flatcar_machines.tf @@ -1,3 +1,9 @@ +resource "null_resource" "worker-config" { + triggers = { + version = var.worker_k3s_version + } +} + resource "tls_private_key" "provisioning" { algorithm = "RSA" rsa_bits = 4096 @@ -31,6 +37,10 @@ resource "hcloud_server" "machine" { backups = false + lifecycle { + replace_triggered_by = [ null_resource.worker-config ] + } + ssh_keys = [ hcloud_ssh_key.provisioning_key.id, hcloud_ssh_key.default.id @@ -57,7 +67,7 @@ resource "hcloud_server" "machine" { connection { host = self.ipv4_address private_key = tls_private_key.provisioning.private_key_pem - timeout = "2m" + timeout = "5m" } provisioner "file" { diff --git a/infrastructure/lb.tf b/infrastructure/lb.tf deleted file mode 100644 index ad99f4a..0000000 --- a/infrastructure/lb.tf +++ /dev/null @@ -1,79 +0,0 @@ -resource "hcloud_load_balancer" "k8s_lb" { - name = "k8s-lb" - load_balancer_type = "lb11" - location = "hel1" -} - -resource "hcloud_load_balancer_network" "k8s_lb_net" { - load_balancer_id = hcloud_load_balancer.k8s_lb.id - network_id = hcloud_network.k8s_net.id - ip = "172.23.2.5" -} - -resource "hcloud_load_balancer_target" "k8s_lb_target" { - type = "label_selector" - label_selector = "node_type=worker" - load_balancer_id = hcloud_load_balancer.k8s_lb.id - use_private_ip = true -} - -resource "hcloud_managed_certificate" "icb4dc0de_20230613_001" { - name = "icb4dc0de_20230613_001" - domain_names = [ - "icb4dc0.de", - "*.icb4dc0.de", - "*.inetmock.icb4dc0.de", - "*.buildr.icb4dc0.de", - "*.prskr.icb4dc0.de", - "*.fider.icb4dc0.de", - "*.ide.icb4dc0.de", - ] - labels = { - } -} - -resource "hcloud_load_balancer_service" "k8s_lb_svc_https" { - load_balancer_id = hcloud_load_balancer.k8s_lb.id - protocol = "https" - destination_port = 32080 - - health_check { - protocol = "tcp" - port = 32080 - interval = 5 - timeout = 3 - retries = 3 - - http { - domain = "code.icb4dc0.de" - path = "/" - tls = false - status_codes = [ - "2??", - "3??" - ] - } - } - - http { - redirect_http = true - certificates = [ - hcloud_managed_certificate.icb4dc0de_20230613_001.id - ] - } -} - -resource "hcloud_load_balancer_service" "k8s_lb_svc_ssh" { - load_balancer_id = hcloud_load_balancer.k8s_lb.id - protocol = "tcp" - destination_port = 32022 - listen_port = 22 - - health_check { - protocol = "tcp" - port = 32022 - interval = 5 - timeout = 3 - retries = 3 - } -} diff --git a/infrastructure/main.tf b/infrastructure/main.tf index 706ed68..9e65eb6 100644 --- a/infrastructure/main.tf +++ b/infrastructure/main.tf @@ -1,3 +1,7 @@ provider "hcloud" { token = var.hcloud_token } + +provider "cloudflare" { + api_token = var.cloudflare_api_token +} \ No newline at end of file diff --git a/infrastructure/tf.sh b/infrastructure/tf.sh index 81dc10d..a148fd6 100755 --- a/infrastructure/tf.sh +++ b/infrastructure/tf.sh @@ -1,12 +1,14 @@ #!/usr/bin/env bash -export AWS_ACCESS_KEY=$(rbw get --raw "CloudFlare TFState" | jq -r ".data.username") -export AWS_SECRET_KEY=$(rbw get --raw "CloudFlare TFState" | jq -r ".data.password") -export HETZNER_DNS_API_TOKEN=$(rbw get --raw "Hetzner DNS" | jq -r '.fields[0].value') +export AWS_ACCESS_KEY=$(rbw get -f username "CloudFlare TFState") +export AWS_SECRET_KEY=$(rbw get "CloudFlare TFState") +export HETZNER_DNS_API_TOKEN=$(rbw get -f "API Token" "Hetzner DNS") export TF_VAR_hcloud_token="$(rbw get "HCloud API")" export TF_VAR_k3s_token="$(rbw get "K3s Token")" -export TF_VAR_litestream_access_key="$(rbw get --raw "Litestream" | jq -r ".data.username")" +export TF_VAR_litestream_access_key="$(rbw get -f username "Litestream")" export TF_VAR_litestream_secret_key="$(rbw get "Litestream")" -export TF_VAR_litestream_endpoint="$(rbw get --raw "Litestream" | jq -r ".fields[0].value")" +export TF_VAR_litestream_endpoint="$(rbw get -f Endpoint "Litestream")" +export TF_VAR_cloudflare_api_token="$(rbw get -f "DNS API Token" "CloudFlare")" +export TF_VAR_cloudflare_account_id="$(rbw get -f "Account ID" "CloudFlare")" tofu $@ \ No newline at end of file diff --git a/infrastructure/vars.tf b/infrastructure/vars.tf index 5255dbe..ea5906d 100644 --- a/infrastructure/vars.tf +++ b/infrastructure/vars.tf @@ -1,8 +1,20 @@ variable "hcloud_token" { + type = string + sensitive = true +} + +variable "cloudflare_api_token" { + type = string + sensitive = true +} + +variable "cloudflare_account_id" { + type = string sensitive = true } variable "k3s_token" { + type = string sensitive = true } @@ -27,12 +39,12 @@ variable "litestream_version" { variable "control_plane_k3s_version" { type = string - default = "v1.28.4+k3s2" + default = "v1.29.2+k3s1" } variable "worker_k3s_version" { type = string - default = "v1.28.4+k3s2" + default = "v1.29.2+k3s1" } variable "k3s_sans" { diff --git a/infrastructure/versions.tf b/infrastructure/versions.tf index 41e3140..911c1d9 100644 --- a/infrastructure/versions.tf +++ b/infrastructure/versions.tf @@ -17,12 +17,14 @@ terraform { required_providers { hcloud = { source = "hetznercloud/hcloud" - version = "1.44.1" + version = "1.45.0" } - hetznerdns = { - source = "timohirt/hetznerdns" - version = "2.2.0" + + cloudflare = { + source = "cloudflare/cloudflare" + version = "4.26.0" } + ct = { source = "poseidon/ct" version = "0.13.0" diff --git a/infrastructure/vms.auto.tfvars b/infrastructure/vms.auto.tfvars index 440e8b5..8f31b32 100644 --- a/infrastructure/vms.auto.tfvars +++ b/infrastructure/vms.auto.tfvars @@ -1,5 +1,5 @@ k3s_control_plane = { - "cp1-cax11-hel1-gen3" = { + "cp1-cax11-hel1-gen4" = { server_type = "cax11", private_ip = "172.23.2.10" location = "hel1" @@ -14,19 +14,19 @@ k3s_sans = [ ] k3s_workers = { - "w1-cx21-hel1-gen1" = { + "w1-cx21-hel1-gen2" = { server_type = "cx21" private_ip = "172.23.2.20" location = "hel1" } - "w2-cax21-hel1-gen6" = { + "w2-cax21-hel1-gen7" = { server_type = "cax21" private_ip = "172.23.2.21" location = "hel1" } - "w3-cax21-hel1-gen6" = { + "w3-cax21-hel1-gen7" = { server_type = "cax21" private_ip = "172.23.2.22" location = "hel1"