feat: prepare custom images for Forgejo runners

configs/ci-runner/10-runcwd

@@ -1 +0,0 @@
-Defaults    runcwd=*

configs/ci-runner/runner-flatcar.yaml

@@ -1,104 +0,0 @@
-variant: flatcar
-version: 1.1.0
-
-systemd:
-  units:
-    - name: forgejo-runner-install.service
-      enabled: true
-      contents: |
-        [Unit]
-        Description=Run Forgejo runner install script
-        Wants = network-online.target
-        After = network.target network-online.target
-        ConditionPathExists=/opt/forgejo-runner-install.sh
-        ConditionPathExists=!/opt/bin/forgejo-runner
-        [Service]
-        Type=oneshot
-        TimeoutStartSec=180
-        RemainAfterExit=yes
-        KillMode=process
-        ExecStart=/usr/bin/sh -c "/opt/forgejo-runner-install.sh"
-        [Install]
-        WantedBy=multi-user.target
-
-    - name: forgejo-runner.service
-      enabled: false
-      contents: |
-        [Unit]
-        Description=Run Forgejo runner
-        Wants = network-online.target
-        After = network.target network-online.target
-        ConditionPathExists=/opt/bin/forgejo-runner
-        [Service]
-        Type=simple
-        TimeoutStartSec=180
-        KillMode=process
-        ExecStart=/opt/bin/forgejo-runner daemon --config /etc/act/config.yaml
-        [Install]
-        WantedBy=multi-user.target
-
-storage:
-  files:
-    - path: /etc/hostname
-      mode: 0644
-      contents:
-        inline: ${host}
-    - path: /opt/forgejo-runner-install.sh
-      mode: 0777
-      contents:
-        inline: |
-          #!/bin/bash
-          set -ex
-          wget -O /opt/bin/forgejo-runner https://data.forgejo.org/forgejo/runner/releases/download/v${runner_version}/forgejo-runner-${runner_version}-linux-${arch}
-          chmod +x /opt/bin/forgejo-runner
-          sudo -u runner /opt/bin/forgejo-runner register --config /etc/act/config.yaml --no-interactive --token ${runner_secret} --name ${host} --instance ${forgejo_instance_url} --labels docker:docker://code.icb4dc0.de/infrastructure/images/act_runtime:arm64,ubuntu-latest:docker://code.icb4dc0.de/infrastructure/images/act_runtime:arm64,ubuntu-22.04:docker://code.icb4dc0.de/infrastructure/images/act_runtime:arm64,ubuntu-20.04:docker://code.icb4dc0.de/infrastructure/images/act_runtime:20.04-arm64
-
-    - path: /etc/act/config.yaml
-      mode: 0644
-      contents:
-        inline: |
-          # You don't have to copy this file to your instance,
-          # just run `forgejo-runner generate-config > config.yaml` to generate a config file.
-          log:
-            # The level of logging, can be trace, debug, info, warn, error, fatal
-            level: info
-
-          runner:
-            file: /home/runner/.runner
-            capacity: 1
-            timeout: 30m
-            fetch_timeout: 5s
-            fetch_interval: 2s
-            labels:
-              - "docker:docker://code.icb4dc0.de/infrastructure/images/act_runtime:arm64"
-              - "ubuntu-latest:docker://code.icb4dc0.de/infrastructure/images/act_runtime:arm64"
-              - "ubuntu-22.04:docker://code.icb4dc0.de/infrastructure/images/act_runtime:arm64"
-              - "ubuntu-20.04:docker://code.icb4dc0.de/infrastructure/images/act_runtime:20.04-arm64"
-
-          cache:
-            enabled: true
-            dir: ""
-            host: ""
-            port: 0
-            external_server: ""
-
-          container:
-            network: ""
-            enable_ipv6: false
-            privileged: false
-            options:
-            workdir_parent:
-            valid_volumes: []
-            docker_host: "unix:///var/run/docker.sock"
-            force_pull: true
-
-          host:
-            workdir_parent:
-
-passwd:
-  users:
-    - name: runner
-      ssh_authorized_keys: ${ssh_keys}
-      home_dir: /home/runner
-      groups:
-        - docker

dns.tf

@@ -4,7 +4,7 @@ resource "cloudflare_zone" "icb4dc0de" {
   }
   name = "icb4dc0.de"
   type = "full"
-  
+
   lifecycle {
     ignore_changes = [account.id]
   }

forgejo-runner_machines.tf

@@ -19,13 +19,21 @@ resource "hcloud_placement_group" "forgejo_runners" {
   }
 }
 
+data "hcloud_image" "forgejo_runner_snapshot_arm64" {
+  id = "228451454"
+}
+
+data "hcloud_image" "forgejo_runner_snapshot_amd64" {
+  id = "228451463"
+}
+
 resource "hcloud_server" "forgejo_runner" {
   for_each           = var.forgejo_runners
   name               = each.key
   server_type        = each.value.server_type
   location           = each.value.location
-  image              = "ubuntu-24.04"
-  placement_group_id = hcloud_placement_group.k3s_machines.id
+  image              = startswith(each.value.server_type, "cax") ? data.hcloud_image.forgejo_runner_snapshot_arm64.id : data.hcloud_image.forgejo_runner_snapshot_amd64.id
+  placement_group_id = hcloud_placement_group.forgejo_runners.id
 
   backups = false
 
@@ -88,62 +96,10 @@ data "cloudinit_config" "runner_config" {
   gzip          = true
   base64_encode = true
 
-  part {
-    content_type = "text/cloud-config"
-    content      = <<-EOF
-      groups:
-        - docker
-      users:
-        - name: runner
-          homedir: /var/lib/runner
-          groups: docker
-      package_update: true
-      package_upgrade: true
-      package_reboot_if_required: true
-      packages:
-        - git
-        - uidmap
-        - dbus-user-session
-        - ca-certificates
-        - curl
-        - gnupg
-        - lsb-release
-        - docker-ce
-        - docker-ce-cli
-        - docker-ce-rootless-extras
-        - containerd.io
-        - docker-compose-plugin
-      apt:
-        sources:
-          docker.list:
-            source: "deb [arch=${startswith(each.value.server_type, "cax") ? "arm64" : "amd64"} signed-by=$KEY_FILE] https://download.docker.com/linux/ubuntu $RELEASE stable"
-            keyid: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88
-    EOF
-  }
-
   part {
     content_type = "text/cloud-config"
     content = <<-EOF
         write_files:
-          - encoding: gzip+base64
-            content: ${base64gzip(file("configs/ci-runner/10-runcwd"))}
-            path: /etc/sudoers.d/10-runcwd
-            owner: root:root
-            permissions: "0644"
-
-          - encoding: gzip+base64
-            content: ${base64gzip(file("configs/ci-runner/50unattended-upgrades"))}
-            path: /etc/apt/apt.conf.d/50unattended-upgrades
-            owner: root:root
-            permissions: "0644"
-
-          - encoding: gzip+base64
-            content: ${base64gzip(file("configs/ci-runner/forgejo-runner.service"))}
-            path: /etc/systemd/user/forgejo-runner.service
-            owner: runner:runner
-            permissions: "0640"
-            defer: true
-
           - encoding: gzip+base64
             content: ${base64gzip(file("configs/ci-runner/docker-buildx-cleanup.service"))}
             path: /lib/systemd/system/docker-buildx-cleanup.service
@@ -156,22 +112,6 @@ data "cloudinit_config" "runner_config" {
             path: /lib/systemd/system/docker-buildx-cleanup.timer
             owner: root:root
             permissions: "0640"
-            defer: true            
-
-          - encoding: gzip+base64
-            content: ${base64gzip(templatefile("configs/ci-runner/runner-config.yaml", {
-    arch = startswith(each.value.server_type, "cax") ? "arm64" : "amd64"
-}))}
-            path: /etc/act/config.yaml
-            owner: runner:runner
-            permissions: "0640"
-            defer: true
-
-          - encoding: gzip+base64
-            content: ${base64gzip(file("configs/ci-runner/daemon.json"))}
-            path: /etc/docker/daemon.json
-            owner: root:root
-            permissions: "0640"
             defer: true
 
           - encoding: gzip+base64
@@ -180,17 +120,10 @@ data "cloudinit_config" "runner_config" {
             owner: runner:runner
             permissions: "0640"
             defer: true
-            
-          - encoding: gzip+base64
-            content: ${base64gzip(file("configs/ci-runner/daemon.json"))}
-            path: /var/lib/runner/.config/docker/daemon.json
-            owner: runner:runner
-            permissions: "0640"
-            defer: true
 
           - encoding: gzip+base64
             content: ${base64gzip(templatefile("configs/ci-runner/docker-rootless-config.json", {
-              registry_auth: base64encode("${data.azurerm_key_vault_secret.harbor_minion_username.value}:${data.azurerm_key_vault_secret.harbor_minion_token.value}")
+    registry_auth : base64encode("${data.azurerm_key_vault_secret.harbor_minion_username.value}:${data.azurerm_key_vault_secret.harbor_minion_token.value}")
 }))}
             path: /var/lib/runner/.docker/config.json
             owner: runner:runner
@@ -205,23 +138,8 @@ part {
       runcmd:
         - |
           set -e
-          loginctl enable-linger runner
-          
-          docker run --privileged --rm tonistiigi/binfmt --install all
-          
-          sleep 10
-          
-          sudo -u runner DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus XDG_RUNTIME_DIR=/run/user/1000 /usr/bin/dockerd-rootless-setuptool.sh install --force
-          
-          curl -L -o /usr/local/bin/forgejo-runner https://data.forgejo.org/forgejo/runner/releases/download/v${var.forgejo_runner_version}/forgejo-runner-${var.forgejo_runner_version}-linux-${startswith(each.value.server_type, "cax") ? "arm64" : "amd64"}
-          curl -L -o /tmp/forgejo-runner.asc https://data.forgejo.org/forgejo/runner/releases/download/v${var.forgejo_runner_version}/forgejo-runner-${var.forgejo_runner_version}-linux-${startswith(each.value.server_type, "cax") ? "arm64" : "amd64"}.asc
-          gpg --keyserver keys.openpgp.org --recv EB114F5E6C0DC2BCDD183550A4B61A2DC5923710
-          gpg --verify /tmp/forgejo-runner.asc /usr/local/bin/forgejo-runner
-          chmod +x /usr/local/bin/forgejo-runner
-          
-          sudo -u runner DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus XDG_RUNTIME_DIR=/run/user/1000 systemctl --user enable --now forgejo-runner.service
-          systemctl enable --now docker-buildx-cleanup.timer
-          systemctl restart unattended-upgrades.service
+          systemctl daemon-reload
+          systemctl enable --now forgejo-runner.service
     EOF
 }
 }

images/forgejo-runner/configs/cloud-init.yaml

@@ -0,0 +1,104 @@
+#cloud-config
+
+growpart:
+  mode: "off"
+resize_rootfs: false
+
+groups:
+  - docker
+users:
+  - name: runner
+    homedir: /var/lib/runner
+    groups: docker
+package_update: true
+package_upgrade: true
+package_reboot_if_required: false
+packages:
+  - git
+  - uidmap
+  - dbus-user-session
+  - ca-certificates
+  - curl
+  - gnupg
+  - lsb-release
+  - docker-ce
+  - docker-ce-cli
+  - docker-ce-rootless-extras
+  - containerd.io
+  - docker-compose-plugin
+apt:
+  sources:
+    docker.list:
+      source: "deb [arch=${arch} signed-by=$KEY_FILE] https://download.docker.com/linux/ubuntu $RELEASE stable"
+      keyid: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88
+
+write_files:
+  - content: |
+      Defaults    runcwd=*
+    path: /etc/sudoers.d/10-runcwd
+    owner: root:root
+    permissions: "0644"
+
+  - encoding: gzip+base64
+    content: ${unattended_upgrades_config}
+    path: /etc/apt/apt.conf.d/50unattended-upgrades
+    owner: root:root
+    permissions: "0644"
+
+  - encoding: gzip+base64
+    content: ${forgejo_runner_service}
+    path: /lib/systemd/system/forgejo-runner.service
+    owner: runner:runner
+    permissions: "0640"
+    defer: true
+
+  - encoding: gzip+base64
+    content: ${forgejo_runner_config}
+    path: /etc/act/config.yaml
+    owner: runner:runner
+    permissions: "0640"
+    defer: true
+
+  - content: |
+      {
+        "features": {
+          "containerd-snapshotter": true
+        }
+      }
+
+    path: /etc/docker/daemon.json
+    owner: root:root
+    permissions: "0640"
+    defer: true
+
+  - content: |
+      {
+        "features": {
+          "containerd-snapshotter": true
+        }
+      }
+    path: /var/lib/runner/.config/docker/daemon.json
+    owner: runner:runner
+    permissions: "0640"
+    defer: true
+
+runcmd:
+  - |
+    set -e
+    loginctl enable-linger runner
+
+    docker run --privileged --rm tonistiigi/binfmt --install all
+
+    sleep 10
+
+    sudo -u runner DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus XDG_RUNTIME_DIR=/run/user/1000 /usr/bin/dockerd-rootless-setuptool.sh install --force
+
+    curl -L -o /usr/local/bin/forgejo-runner https://code.forgejo.org/forgejo/runner/releases/download/v${forgejo_runner_version}/forgejo-runner-${forgejo_runner_version}-linux-${arch}
+
+    curl -L -o /tmp/forgejo-runner.asc https://code.forgejo.org/forgejo/runner/releases/download/v${forgejo_runner_version}/forgejo-runner-${forgejo_runner_version}-linux-${arch}.asc
+
+    gpg --keyserver keys.openpgp.org --recv EB114F5E6C0DC2BCDD183550A4B61A2DC5923710
+    gpg --verify /tmp/forgejo-runner.asc /usr/local/bin/forgejo-runner
+    chmod +x /usr/local/bin/forgejo-runner
+
+    systemctl restart unattended-upgrades.service

images/forgejo-runner/configs/forgejo-runner.service

@@ -7,6 +7,7 @@ ConditionPathExists=/run/user/1000/docker.sock
 [Service]
 TimeoutStartSec=180
 KillMode=process
+User=runner
 ExecStart=forgejo-runner daemon --config /etc/act/config.yaml
 WorkingDirectory=/var/lib/runner
 

images/forgejo-runner/packer.pkr.hcl

@@ -0,0 +1,84 @@
+packer {
+  required_plugins {
+    hcloud = {
+      source  = "github.com/hetznercloud/hcloud"
+      version = ">= 1.6.0"
+    }
+  }
+}
+
+variable "hcloud_location" {
+  type    = string
+  default = "hel1"
+}
+
+variable "forgejo_runner_version" {
+  type    = string
+  default = "6.3.1"
+}
+
+source "hcloud" "base-arm64" {
+  image         = "ubuntu-24.04"
+  location      = var.hcloud_location
+  server_type   = "cax11"
+  ssh_username  = "root"
+  snapshot_name = "forgejo-runner-${var.forgejo_runner_version}"
+  snapshot_labels = {
+    name           = "forgejo_runner"
+    base           = "ubuntu-24.04",
+    arch           = "arm64"
+    runner_version = var.forgejo_runner_version
+  }
+
+  user_data = templatefile("configs/cloud-init.yaml", {
+    arch                       = "arm64"
+    unattended_upgrades_config = base64gzip(file("configs/50unattended-upgrades"))
+    forgejo_runner_service     = base64gzip(file("configs/forgejo-runner.service"))
+    forgejo_runner_version     = var.forgejo_runner_version
+    forgejo_runner_config = base64gzip(templatefile("configs/runner-config.yaml", {
+      arch = "arm64"
+    }))
+  })
+}
+
+source "hcloud" "base-amd64" {
+  image         = "ubuntu-24.04"
+  location      = var.hcloud_location
+  server_type   = "cx22"
+  ssh_keys      = ["Yubikey", "Default Management"]
+  ssh_username  = "root"
+  snapshot_name = "forgejo-runner-${var.forgejo_runner_version}"
+  snapshot_labels = {
+    name           = "forgejo_runner"
+    base           = "ubuntu-24.04",
+    version        = "v1.0.0",
+    arch           = "amd64"
+    runner_version = var.forgejo_runner_version
+  }
+
+  user_data = templatefile("configs/cloud-init.yaml", {
+    arch                       = "amd64"
+    unattended_upgrades_config = base64gzip(file("configs/50unattended-upgrades"))
+    forgejo_runner_service     = base64gzip(file("configs/forgejo-runner.service"))
+    forgejo_runner_version     = var.forgejo_runner_version
+    forgejo_runner_config = base64gzip(templatefile("configs/runner-config.yaml", {
+      arch = "amd64"
+    }))
+  })
+}
+
+build {
+  sources = ["source.hcloud.base-arm64", "source.hcloud.base-amd64"]
+
+  provisioner "shell" {
+    inline           = ["cloud-init status --wait --long"]
+    valid_exit_codes = [0, 2]
+  }
+
+  provisioner "shell" {
+    scripts = [
+      "scripts/upgrade.sh",
+      "scripts/cleanup.sh",
+    ]
+  }
+}

images/forgejo-runner/scripts/cleanup.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+set -eux
+
+clean_cloud_init() {
+  cloud-init clean --logs --machine-id --seed --configs all
+
+  rm -rf /run/cloud-init/*
+  rm -rf /var/lib/cloud/*
+}
+
+clean_apt() {
+  export DEBIAN_FRONTEND=noninteractive
+
+  apt-get -y autopurge
+  apt-get -y clean
+
+  rm -rf /var/lib/apt/lists/*
+}
+
+clean_ssh_keys() {
+  rm -f /etc/ssh/ssh_host_*_key /etc/ssh/ssh_host_*_key.pub
+}
+
+clean_logs() {
+  journalctl --flush
+  journalctl --rotate --vacuum-time=0
+
+  find /var/log -type f -exec truncate --size 0 {} \; # truncate system logs
+  find /var/log -type f -name '*.[1-9]' -delete # remove archived logs
+  find /var/log -type f -name '*.gz' -delete # remove compressed archived logs
+}
+
+clean_root() {
+  unset HISTFILE
+
+  rm -rf /root/.cache
+  rm -rf /root/.ssh
+  rm -f /root/.bash_history
+  rm -f /root/.lesshst
+  rm -f /root/.viminfo
+}
+
+flush_disk() {
+  dd if=/dev/zero of=/zero bs=4M || true
+  sync
+  rm -f /zero
+}
+
+clean_cloud_init
+clean_apt
+clean_ssh_keys
+clean_logs
+clean_root
+
+flush_disk

images/forgejo-runner/scripts/upgrade.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -eux
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get autoremove -y --purge