feat: prepare Prometheus monitoring and upgrade Postgres DB

k8s/configure_cluster.yaml

@@ -1,6 +1,8 @@
 - name: Configure cluster
   hosts: localhost
   roles:
+  # - role: gateway-api
+    - role: prometheus
     - role: postgres
     - role: csi
     - role: minio

k8s/configure_postgres.yaml

@@ -0,0 +1,4 @@
+- name: Configure postgres
+  hosts: localhost
+  roles:
+    - role: postgres-config

k8s/hack/postgres-client.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: psql-client
+  namespace: postgres
+spec:
+  containers:
+  - name: psql
+    image: docker.io/alpine
+    command:
+      - "/bin/ash"
+      - -c
+      - sleep 7200

k8s/inventory/clusters.yaml

@@ -1,20 +1,20 @@
 all:
   vars:
     ansible_user: root
-    k3s_version: v1.24.3+k3s1
     extra_server_args: "--node-taint=node-type=master:NoSchedule --tls-san='2a01:4f9:c012:7d4b::1' --tls-san='k8s.icb4dc0.de' --tls-san='127.0.0.1'"
     extra_agent_args: ""
     ansible_ssh_common_args: '-o StrictHostKeyChecking=no'
     systemd_dir: /etc/systemd/system
     master_ip: "172.23.2.10"
     domain: icb4dc0.de
-    agola_image: code.icb4dc0.de/prskr/agola:latest
   children:
     control_plane:
       hosts:
         cp01:
           ansible_host: "95.216.168.169"
           k8s_ip: "172.23.2.10"
+      vars:
+        k3s_version: v1.25.4+k3s1
     worker_nodes:
       hosts:
         worker01:
@@ -24,3 +24,5 @@ all:
         worker02:
           ansible_host: "95.217.184.201"
           k8s_ip: "172.23.2.21"
+      vars:
+        k3s_version: v1.25.4+k3s1

k8s/roles/concourse/tasks/main.yml

@@ -5,6 +5,10 @@
     api_version: v1
     kind: Namespace
     state: present
+    definition:
+      metadata:
+        labels:
+          prometheus: default
 
 - name: Add Concourse chart repo
   kubernetes.core.helm_repository:

k8s/roles/concourse/templates/values.concourse.yml.j2

@@ -33,7 +33,7 @@ concourse:
         disableGroups: true
         skipEmailVerifiedValidation: true
     postgres:
-      host: default-postgres-postgresql.postgres.svc.cluster.local
+      host: postgres-15-postgresql.postgres.svc.cluster.local
       port: "5432"
       database: concourse
     kubernetes:

k8s/roles/download/tasks/main.yml

@@ -1,23 +1,8 @@
 ---
-- name: Create temporary file
-  ansible.builtin.tempfile:
-    state: file
-    suffix: temp
-  register: k3s_binary_tmp
-  delegate_to: localhost
-  run_once: true
-
 - name: Download k3s binary
-  get_url:
+  ansible.builtin.get_url:
     url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s
     checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-amd64.txt
-    dest: "{{ k3s_binary_tmp.path }}"
-  delegate_to: localhost
-  run_once: true
-
-- name: Copy k3s binary
-  ansible.builtin.copy:
-    src: "{{ k3s_binary_tmp.path }}"
     dest: /usr/local/bin/k3s
     owner: root
     group: root

k8s/roles/gitea/tasks/main.yml

@@ -5,6 +5,10 @@
     api_version: v1
     kind: Namespace
     state: present
+    definition:
+      metadata:
+        labels:
+          prometheus: default
 
 - name: Create Gitea admin credentials
   kubernetes.core.k8s:

k8s/roles/gitea/templates/values.gitea.yml.j2

@@ -27,6 +27,12 @@ persistence:
 gitea:
   admin:
     existingSecret: gitea-admin-credentials
+  metrics:
+    enabled: true
+    serviceMonitor:
+      enabled: true
+      additionalLabels:
+        prometheus: default
   config:
     git.timeout:
       migrate: 3600
@@ -50,11 +56,13 @@ gitea:
       MINIO_USE_SSL: 'false'
     database:
       DB_TYPE: postgres
-      HOST: default-postgres-postgresql.postgres.svc.cluster.local:5432
+      HOST: postgres-15-postgresql.postgres.svc.cluster.local:5432
       NAME: gitea
       USER: gitea
       PASSWD: "{{ gitea.dbPassword }}"
       log_sql: "false"
+    metrics:
+      ENABLED: true
 
 
 postgresql:

k8s/roles/k3s/master/files/traefik.yaml

@@ -4,6 +4,9 @@ metadata:
   name: traefik
   namespace: kube-system
 spec:
+  chart: traefik
+  repo: https://traefik.github.io/charts
+  version: 20.8.0
   valuesContent: |-
     ports:
       web:
@@ -12,3 +15,15 @@ spec:
         expose: false
     service:
       type: NodePort
+    experimental:
+      kubernetesGateway:
+        enabled: true
+    metrics:
+      prometheus:
+        serviceMonitor:
+          interval: 30s
+          scrapeTimeout: 5s
+          additionalLabels:
+            prometheus: default
+        service:
+          enabled: true

k8s/roles/minio/tasks/main.yml

@@ -5,6 +5,10 @@
     api_version: v1
     kind: Namespace
     state: present
+    definition:
+      metadata:
+        labels:
+          prometheus: default
 
 - name: Create MinIO secret
   kubernetes.core.k8s:

k8s/roles/postgres-config/tasks/main.yaml

@@ -0,0 +1,29 @@
+---
+- name: Create users
+  community.postgresql.postgresql_user:
+    name: "{{ item.name }}"
+    password: "{{ item.password }}"
+    login_host: "127.0.0.1"
+    login_password: "{{ PostgresPassword }}"
+  loop: 
+    - name: gitea
+      password: "{{ gitea.dbPassword }}"
+    - name: "{{ grafana.db.user }}"
+      password: "{{ grafana.db.password }}"
+    - name: "{{ concourse.db.user }}"
+      password: "{{ concourse.db.password }}"
+
+- name: Create databases
+  community.postgresql.postgresql_db:
+    name: "{{ item.name }}"
+    owner: "{{ item.owner }}"
+    encoding: UTF-8
+    login_host: "127.0.0.1"
+    login_password: "{{ PostgresPassword }}"
+  loop: 
+    - name: gitea
+      owner: gitea
+    - name: concourse
+      owner: "{{ concourse.db.user }}"
+    - name: grafana
+      owner: "{{ grafana.db.user }}"

k8s/roles/postgres/files/values.postgres15.yaml

@@ -0,0 +1,7 @@
+auth:
+  existingSecret: postgres-credentials
+
+primary:
+  persistence:
+    storageClass: hcloud-volumes
+    size: 8Gi

k8s/roles/postgres/tasks/main.yml

@@ -5,6 +5,10 @@
     api_version: v1
     kind: Namespace
     state: present
+    definition:
+      metadata:
+        labels:
+          prometheus: default
 
 - name: Create Postgres secret
   kubernetes.core.k8s:
@@ -30,3 +34,11 @@
     release_namespace: postgres
     chart_version: 11.9.13
     release_values: "{{ lookup('ansible.builtin.file', 'values.postgres.yaml') | from_yaml }}"
+
+- name: Deploy Postgres 15
+  kubernetes.core.helm:
+    name: postgres-15
+    chart_ref: bitnami/postgresql
+    release_namespace: postgres
+    chart_version: 12.1.6
+    release_values: "{{ lookup('ansible.builtin.file', 'values.postgres15.yaml') | from_yaml }}"

k8s/roles/prometheus/tasks/main.yaml

@@ -0,0 +1,37 @@
+---
+- name: Add Prometheus chart repo
+  kubernetes.core.helm_repository:
+    name: prometheus-community
+    repo_url: https://prometheus-community.github.io/helm-charts
+
+- name: Create observability namespace
+  kubernetes.core.k8s:
+    name: observability-system
+    api_version: v1
+    kind: Namespace
+    state: present
+    definition:
+      metadata:
+        labels:
+          prometheus: default
+
+- name: Create Grafana admin credentials secret
+  kubernetes.core.k8s:
+    state: present
+    definition:
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: grafana-admin-credentials
+        namespace: observability-system
+      data:
+        user: "{{ grafana.admin.user | b64encode }}"
+        password: "{{ grafana.admin.password | b64encode }}"
+
+- name: Deploy Prometheus chart
+  kubernetes.core.helm:
+    name: prometheus
+    chart_ref: prometheus-community/kube-prometheus-stack
+    release_namespace: observability-system
+    chart_version: 43.2.0
+    release_values: "{{ lookup('ansible.builtin.template', 'values.yaml.j2') | from_yaml }}"

k8s/roles/prometheus/templates/values.yaml.j2

@@ -0,0 +1,66 @@
+commonLabels:
+  prometheus: default
+
+admin:
+  existingSecret: grafana-admin-credentials
+  userKey: user
+  passwordKey: password
+
+defaultRules:
+  rules:
+    etcd: false
+
+prometheus:
+  prometheusSpec:
+    retention: 7d
+    serviceMonitorNamespaceSelector:
+      matchLabels:
+        prometheus: default
+    serviceMonitorSelector:
+      matchLabels:
+        prometheus: default
+    storageSpec:
+      volumeClaimTemplate:
+        spec:
+          storageClassName: hcloud-volumes
+          resources:
+            requests:
+              storage: 15Gi
+
+alertmanager:
+  enabled: false
+
+kubeEtcd:
+  enabled: false
+
+grafana:
+  ingress:
+    enabled: true
+    hosts:
+      - grafana.icb4dc0.de
+  grafana.ini:
+    server:
+      domain: grafana.icb4dc0.de
+      root_url: "https://%(domain)s"
+    database:
+      type: postgres
+      host: postgres-15-postgresql.postgres.svc.cluster.local:5432
+      name: grafana
+      user: "{{ grafana.db.user }}"
+      password: "{{ grafana.db.password }}"
+      ssl_mode: disable
+    auth:
+      disable_login_form: true
+    auth.generic_oauth:
+        name: Gitea
+        icon: signin
+        enabled: "true"
+        client_id: {{ grafana.auth.clientId }}
+        client_secret: {{ grafana.auth.clientSecret }}
+        empty_scopes: true
+        auth_url: https://code.icb4dc0.de/login/oauth/authorize
+        token_url: https://code.icb4dc0.de/login/oauth/access_token
+        api_url: https://code.icb4dc0.de/login/oauth/userinfo
+  persistence:
+    enabled: false
+    storageClassName: hcloud-volumes

k8s/roles/requirements.yaml

@@ -2,3 +2,4 @@
 collections:
 - kubernetes.core
 - hetzner.hcloud
+- community.postgresql