feat: add ci minions

configs/ci-runner/runner-flatcar.yaml

@@ -0,0 +1,104 @@
+variant: flatcar
+version: 1.1.0
+
+systemd:
+  units:
+    - name: forgejo-runner-install.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Run Forgejo runner install script
+        Wants = network-online.target
+        After = network.target network-online.target
+        ConditionPathExists=/opt/forgejo-runner-install.sh
+        ConditionPathExists=!/opt/bin/forgejo-runner
+        [Service]
+        Type=oneshot
+        TimeoutStartSec=180
+        RemainAfterExit=yes
+        KillMode=process
+        ExecStart=/usr/bin/sh -c "/opt/forgejo-runner-install.sh"
+        [Install]
+        WantedBy=multi-user.target
+
+    - name: forgejo-runner.service
+      enabled: false
+      contents: |
+        [Unit]
+        Description=Run Forgejo runner
+        Wants = network-online.target
+        After = network.target network-online.target
+        ConditionPathExists=/opt/bin/forgejo-runner
+        [Service]
+        Type=simple
+        TimeoutStartSec=180
+        KillMode=process
+        ExecStart=/opt/bin/forgejo-runner daemon --config /etc/act/config.yaml
+        [Install]
+        WantedBy=multi-user.target
+
+storage:
+  files:
+    - path: /etc/hostname
+      mode: 0644
+      contents:
+        inline: ${host}
+    - path: /opt/forgejo-runner-install.sh
+      mode: 0777
+      contents:
+        inline: |
+          #!/bin/bash
+          set -ex
+          wget -O /opt/bin/forgejo-runner https://data.forgejo.org/forgejo/runner/releases/download/v${runner_version}/forgejo-runner-${runner_version}-linux-${arch}
+          chmod +x /opt/bin/forgejo-runner
+          sudo -u runner /opt/bin/forgejo-runner register --config /etc/act/config.yaml --no-interactive --token ${runner_secret} --name ${host} --instance ${forgejo_instance_url} --labels docker:docker://code.icb4dc0.de/infrastructure/images/act_runtime:arm64,ubuntu-latest:docker://code.icb4dc0.de/infrastructure/images/act_runtime:arm64,ubuntu-22.04:docker://code.icb4dc0.de/infrastructure/images/act_runtime:arm64,ubuntu-20.04:docker://code.icb4dc0.de/infrastructure/images/act_runtime:20.04-arm64
+
+    - path: /etc/act/config.yaml
+      mode: 0644
+      contents:
+        inline: |
+          # You don't have to copy this file to your instance,
+          # just run `forgejo-runner generate-config > config.yaml` to generate a config file.
+          log:
+            # The level of logging, can be trace, debug, info, warn, error, fatal
+            level: info
+
+          runner:
+            file: /home/runner/.runner
+            capacity: 1
+            timeout: 30m
+            fetch_timeout: 5s
+            fetch_interval: 2s
+            labels:
+              - "docker:docker://code.icb4dc0.de/infrastructure/images/act_runtime:arm64"
+              - "ubuntu-latest:docker://code.icb4dc0.de/infrastructure/images/act_runtime:arm64"
+              - "ubuntu-22.04:docker://code.icb4dc0.de/infrastructure/images/act_runtime:arm64"
+              - "ubuntu-20.04:docker://code.icb4dc0.de/infrastructure/images/act_runtime:20.04-arm64"
+
+          cache:
+            enabled: true
+            dir: ""
+            host: ""
+            port: 0
+            external_server: ""
+
+          container:
+            network: ""
+            enable_ipv6: false
+            privileged: false
+            options:
+            workdir_parent:
+            valid_volumes: []
+            docker_host: "unix:///var/run/docker.sock"
+            force_pull: true
+
+          host:
+            workdir_parent:
+
+passwd:
+  users:
+    - name: runner
+      ssh_authorized_keys: ${ssh_keys}
+      home_dir: /home/runner
+      groups:
+        - docker

dns.tf

@@ -11,7 +11,7 @@ resource "cloudflare_record" "mx_primary" {
   zone_id  = cloudflare_zone.icb4dc0de.id
   name     = "@"
   type     = "MX"
-  content    = "mx01.mail.icloud.com"
+  content  = "mx01.mail.icloud.com"
   priority = 10
 }
 
@@ -20,7 +20,7 @@ resource "cloudflare_record" "mx_secondary" {
   zone_id  = cloudflare_zone.icb4dc0de.id
   name     = "@"
   type     = "MX"
-  content    = "mx02.mail.icloud.com"
+  content  = "mx02.mail.icloud.com"
   priority = 10
 }
 
@@ -28,26 +28,26 @@ resource "cloudflare_record" "apple_proof" {
   zone_id = cloudflare_zone.icb4dc0de.id
   name    = "@"
   type    = "TXT"
-  content   = "apple-domain=chwbVvzH8hWIgg1l"
+  content = "apple-domain=chwbVvzH8hWIgg1l"
 }
 
 resource "cloudflare_record" "keybase_proof" {
   zone_id = cloudflare_zone.icb4dc0de.id
   name    = "@"
   type    = "TXT"
-  content   = "keybase-site-verification=WDQoLtW22epD7eQnts6rPKJBGA0lD6jSI6m0bGMYWag"
+  content = "keybase-site-verification=WDQoLtW22epD7eQnts6rPKJBGA0lD6jSI6m0bGMYWag"
 }
 
 resource "cloudflare_record" "apple_spf" {
   zone_id = cloudflare_zone.icb4dc0de.id
   name    = "@"
   type    = "TXT"
-  content   = "\"v=spf1 include:icloud.com ~all\""
+  content = "\"v=spf1 include:icloud.com ~all\""
 }
 
 resource "cloudflare_record" "apple_sig_domainkey" {
   zone_id = cloudflare_zone.icb4dc0de.id
   name    = "sig1._domainkey"
   type    = "CNAME"
-  content   = "sig1.dkim.icb4dc0.de.at.icloudmailadmin.com"
+  content = "sig1.dkim.icb4dc0.de.at.icloudmailadmin.com"
 }

forgejo-runner_machines.tf

@@ -0,0 +1,158 @@
+resource "null_resource" "runner-config" {
+  triggers = {
+    version = var.forgejo_runner_version
+  }
+}
+
+resource "null_resource" "runner_generation" {
+  for_each = var.forgejo_runners
+  triggers = {
+    timestamp = "${each.value.generation}"
+  }
+}
+
+resource "hcloud_placement_group" "forgejo_runners" {
+  name = "forgejo-runners"
+  type = "spread"
+  labels = {
+    "cluster" = "forgejo.icb4dc0.de"
+  }
+}
+
+
+resource "hcloud_server" "forgejo_runner" {
+  for_each           = var.forgejo_runners
+  name               = each.key
+  server_type        = each.value.server_type
+  location           = each.value.location
+  image              = "ubuntu-22.04"
+  placement_group_id = hcloud_placement_group.k3s_machines.id
+
+  backups = false
+
+  lifecycle {
+    replace_triggered_by = [
+      null_resource.runner-config,
+      null_resource.runner_generation[each.key]
+    ]
+  }
+
+  ssh_keys = [
+    hcloud_ssh_key.provisioning_key.id,
+    hcloud_ssh_key.default.id
+  ]
+
+  labels = {
+    "node_type" = "forgejo_runner"
+    "cluster"   = "forgejo.icb4dc0.de"
+  }
+
+  network {
+    network_id = hcloud_network.k8s_net.id
+    ip         = each.value.private_ip
+  }
+
+  public_net {
+    ipv4_enabled = true
+    ipv6_enabled = true
+  }
+
+  # boot into rescue OS
+  rescue = "linux64"
+
+  connection {
+    host        = self.ipv4_address
+    agent       = false
+    private_key = tls_private_key.provisioning.private_key_pem
+    timeout     = "5m"
+  }
+
+  provisioner "file" {
+    content     = data.ct_config.forgejo-machine-ignitions[each.key].rendered
+    destination = "/root/ignition.json"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "set -ex",
+      "apt-get install -y gawk",
+      "curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/flatcar/init/flatcar-master/bin/flatcar-install",
+      "chmod +x flatcar-install",
+      "./flatcar-install -s -i /root/ignition.json -C ${var.flatcar_release_channel}",
+      "reboot",
+    ]
+    on_failure = continue
+  }
+
+  provisioner "remote-exec" {
+    connection {
+      host        = self.ipv4_address
+      private_key = tls_private_key.provisioning.private_key_pem
+      timeout     = "3m"
+      user        = "core"
+    }
+    inline = [
+      "sudo hostnamectl set-hostname ${self.name}",
+    ]
+  }
+
+  provisioner "file" {
+    connection {
+      host        = self.ipv4_address
+      private_key = tls_private_key.provisioning.private_key_pem
+      timeout     = "3m"
+      user        = "core"
+    }
+    destination = "/tmp/00-eth0.network"
+    content     = <<EOT
+      [Match]
+      Name=eth0
+      
+      [Network]
+      DHCP=ipv4
+      Address=${self.ipv6_address}
+      Gateway=fe80::1
+      DNS=2a01:4ff:ff00::add:2
+      DNS=2a01:4ff:ff00::add:1
+      EOT
+  }
+
+  provisioner "remote-exec" {
+    connection {
+      host        = self.ipv4_address
+      private_key = tls_private_key.provisioning.private_key_pem
+      timeout     = "3m"
+      user        = "core"
+    }
+    inline = [
+      "sudo mv /tmp/00-eth0.network /etc/systemd/network/00-eth0.network",
+      "sudo systemctl restart systemd-networkd",
+      "sudo systemctl enable --now forgejo-runner.service",
+    ]
+  }
+}
+
+data "ct_config" "forgejo-machine-ignitions" {
+  for_each = var.forgejo_runners
+  strict   = true
+  content = templatefile(
+    "${path.module}/configs/ci-runner/runner-flatcar.yaml",
+    {
+      "host"                 = each.key
+      "node_ip"              = each.value.private_ip
+      "runner_version"       = var.forgejo_runner_version
+      "forgejo_instance_url" = var.forgejo_instance_url
+      "runner_secret"        = var.forgejo_runner_secret
+      "arch"                 = startswith(each.value.server_type, "cax") ? "arm64" : "amd64"
+      "ssh_keys"             = jsonencode(concat(var.ssh_keys, [tls_private_key.provisioning.public_key_openssh]))
+    }
+  )
+  snippets = [
+    templatefile(
+      "${path.module}/configs/core-user.yaml.tmpl",
+      {
+        ssh_keys = jsonencode(concat(var.ssh_keys, [tls_private_key.provisioning.public_key_openssh]))
+      }
+    )
+  ]
+}

k8s_flatcar_machines.tf

@@ -4,30 +4,6 @@ resource "null_resource" "worker-config" {
   }
 }
 
-resource "tls_private_key" "provisioning" {
-  algorithm = "RSA"
-  rsa_bits  = 4096
-}
-
-resource "hcloud_ssh_key" "provisioning_key" {
-  name       = "Provisioning key for hcloud cluster"
-  public_key = tls_private_key.provisioning.public_key_openssh
-}
-
-resource "local_file" "provisioning_key" {
-  filename             = "${path.module}/.ssh/provisioning_private_key.pem"
-  content              = tls_private_key.provisioning.private_key_pem
-  directory_permission = "0700"
-  file_permission      = "0400"
-}
-
-resource "local_file" "provisioning_key_pub" {
-  filename             = "${path.module}/.ssh/provisioning_key.pub"
-  content              = tls_private_key.provisioning.public_key_openssh
-  directory_permission = "0700"
-  file_permission      = "0440"
-}
-
 resource "null_resource" "machine_generation" {
   for_each = var.k3s_workers
   triggers = {
@@ -123,7 +99,6 @@ resource "hcloud_server" "machine" {
       "sudo hostnamectl set-hostname ${self.name}",
     ]
   }
-
 }
 
 data "ct_config" "machine-ignitions" {
@@ -132,12 +107,12 @@ data "ct_config" "machine-ignitions" {
   content = templatefile(
     "${path.module}/configs/workers/k3s-flatcar.yaml",
     {
-      "host"         = each.key
+      "host"              = each.key
-      "k3s_token"    = var.k3s_token
+      "k3s_token"         = var.k3s_token
-      "node_ip"      = each.value.private_ip
+      "node_ip"           = each.value.private_ip
-      "k3s_version"  = var.worker_k3s_version
+      "k3s_version"       = var.worker_k3s_version
       "spin_shim_version" = var.spin_shim_version
-      "arch" = startswith(each.value.server_type, "cax") ? "aarch64" : "x86_64"
+      "arch"              = startswith(each.value.server_type, "cax") ? "aarch64" : "x86_64"
     }
   )
   snippets = [

provisioning.tf

@@ -0,0 +1,23 @@
+resource "tls_private_key" "provisioning" {
+  algorithm = "RSA"
+  rsa_bits  = 4096
+}
+
+resource "hcloud_ssh_key" "provisioning_key" {
+  name       = "Provisioning key for hcloud cluster"
+  public_key = tls_private_key.provisioning.public_key_openssh
+}
+
+resource "local_file" "provisioning_key" {
+  filename             = "${path.module}/.ssh/provisioning_private_key.pem"
+  content              = tls_private_key.provisioning.private_key_pem
+  directory_permission = "0700"
+  file_permission      = "0400"
+}
+
+resource "local_file" "provisioning_key_pub" {
+  filename             = "${path.module}/.ssh/provisioning_key.pub"
+  content              = tls_private_key.provisioning.public_key_openssh
+  directory_permission = "0700"
+  file_permission      = "0440"
+}

tf.sh

@@ -5,10 +5,11 @@ export AWS_SECRET_KEY=$(rbw get "CloudFlare TFState")
 export HETZNER_DNS_API_TOKEN=$(rbw get -f "API Token" "Hetzner DNS")
 export TF_VAR_hcloud_token="$(rbw get "HCloud API")"
 export TF_VAR_k3s_token="$(rbw get "K3s Token")"
+export TF_VAR_forgejo_runner_secret="$(rbw get "Forgejo Runner Secret")"
 export TF_VAR_k3s_backup_access_key="$(rbw get -f username "K3s Backup")"
 export TF_VAR_k3s_backup_secret_key="$(rbw get "K3s Backup")"
 export TF_VAR_k3s_backup_endpoint="$(rbw get -f Endpoint "K3s Backup")"
 export TF_VAR_cloudflare_api_token="$(rbw get -f "DNS API Token" "CloudFlare")"
 export TF_VAR_cloudflare_account_id="$(rbw get -f "Account ID" "CloudFlare")"
 
-tofu $@
+tofu $@

vars.tf

@@ -52,6 +52,21 @@ variable "worker_k3s_version" {
   default = "v1.31.5+k3s1"
 }
 
+variable "forgejo_runner_secret" {
+  type      = string
+  sensitive = true
+}
+
+variable "forgejo_runner_version" {
+  type    = string
+  default = "6.2.2"
+}
+
+variable "forgejo_instance_url" {
+  type    = string
+  default = "https://code.icb4dc0.de"
+}
+
 variable "k3s_sans" {
   type = list(string)
 }
@@ -68,10 +83,19 @@ variable "k3s_control_plane" {
 
 variable "k3s_workers" {
   type = map(object({
-    server_type  = string
+    server_type = string
-    generation   = number
+    generation  = number
-    private_ip   = string
+    private_ip  = string
-    location     = string
+    location    = string
+  }))
+}
+
+variable "forgejo_runners" {
+  type = map(object({
+    server_type = string
+    generation  = number
+    private_ip  = string
+    location    = string
   }))
 }
 

vms.auto.tfvars

@@ -37,6 +37,22 @@ k3s_workers = {
   }
 }
 
+forgejo_runners = {
+  "ci-minion-bob" = {
+    server_type = "cax21"
+    generation  = 2
+    private_ip  = "172.23.2.30"
+    location    = "hel1"
+  }
+
+  "ci-minion-stuart" = {
+    server_type = "cax21"
+    generation  = 2
+    private_ip  = "172.23.2.31"
+    location    = "hel1"
+  }
+}
+
 ssh_keys = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDQoNCLuHgcaDn4JTjCeQKJsIsYU0Jmub5PUMzIIZbUBb+TGMh6mCAY/UbYaq/n4jVnskXopzPGJbx4iPBG5HrNzqYZqMjkk8uIeeT0mdIcNv9bXxuCxCH1iHZF8LlzIZCmQ0w3X6VQ1izcJgvjrAYzbHN3gqCHOXtNkqIUkwaadIWCEjg33OVSlM4yrIDElr6+LHzv84VNh/PhemixCVVEMJ83GjhDtpApMg9WWW3es6rpJn4TlYEMV+aPNU4ZZEWFen/DFBKoX+ulkiJ8CwpY3eJxSzlBijs5ZKH89OOk/MXN1lnREElFqli+jE8EbZKQzi59Zmx8ZOb52qVNot8XZT0Un4EttAIEeE8cETqUC4jK+6RbUrsXtclVbU9i57LWRpl65LYSIJEFmkTxvYdkPXqGbvlW024IjgSo8kds121w95+Rpo6419cSYsQWowS8+aXfEv2Q8SE81QH7ObYfWFXsPBAmmNleQNN3E5HOoaxpWQjv3aTUGuxm4PCzKLdP0LsHmTfGJB7Priaj+9i8xLjDWe7zXDde2Gp9FmdedDr06uEkVSRFnS35Dwfd7M7xP6NsilfMOdWzJWWy/BAYxtnWcrEFxhaEr4vgs8Ub+KBtKhr740x3Mr8up+mythConAs4LOj37lWK4kJ8cI7TXjcSJi9nTIPd39us7tp3Aw== cardno:24_781_961"]
 
 flatcar_release_channel = "stable"