fix(metrics): collect metrics from kube components

k8s/roles/concourse/templates/rbac/deploy-role.yml.j2

@@ -31,3 +31,11 @@ rules:
       - "ingresses"
     verbs:
       - "*"
+
+  - apiGroups:
+      - "monitoring.coreos.com"
+    resources:
+      - "podmonitors"
+      - "servicemonitors"
+    verbs:
+      - "*"

k8s/roles/k3s/master/tasks/main.yml

@@ -16,6 +16,14 @@
     group: root
     mode: 0644
 
+- name: Render control plain config
+  ansible.builtin.template:
+    src: cp-config.yaml.j2
+    dest: /etc/rancher/k3s/config.yaml
+    owner: root
+    group: root
+    mode: 0644
+
 - name: Enable and check K3s service
   systemd:
     name: k3s

k8s/roles/k3s/master/templates/cp-config.yaml.j2

@@ -0,0 +1,10 @@
+kube-controller-manager-arg:
+- "bind-address=0.0.0.0"
+- "authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics"
+kube-proxy-arg:
+- "metrics-bind-address=0.0.0.0:10249"
+kube-scheduler-arg:
+- "bind-address=0.0.0.0"
+- "authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics"
+# Controller Manager exposes etcd sqllite metrics
+etcd-expose-metrics: true

k8s/roles/prometheus/templates/values.yaml.j2

@@ -19,6 +19,18 @@ prometheus:
     serviceMonitorSelector:
       matchLabels:
         prometheus: default
+    ruleSelector:
+      matchLabels:
+        prometheus: default
+    ruleNamespaceSelector:
+      matchLabels:
+        prometheus: default
+    podMonitorSelector:
+      matchLabels:
+        prometheus: default
+    podMonitorNamespaceSelector:
+      matchLabels:
+        prometheus: default
     storageSpec:
       volumeClaimTemplate:
         spec:
@@ -33,6 +45,36 @@ alertmanager:
 kubeEtcd:
   enabled: false
 
+kubeControllerManager:
+  enabled: true
+  endpoints: ['{{ master_ip }}']
+  service:
+    enabled: true
+    port: 10257
+    targetPort: 10257
+  serviceMonitor:
+    enabled: true
+    https: true
+
+kubeScheduler:
+  enabled: true
+  endpoints: ['{{ master_ip }}']
+  service:
+    enabled: true
+    port: 10259
+    targetPort: 10259
+  serviceMonitor:
+    enabled: true
+    https: true
+
+kubeProxy:
+  enabled: true
+  endpoints: ['{{ master_ip }}']
+  service:
+    enabled: true
+    port: 10249
+    targetPort: 10249
+
 grafana:
   ingress:
     enabled: true
@@ -64,3 +106,15 @@ grafana:
   persistence:
     enabled: false
     storageClassName: hcloud-volumes
+
+prometheus-node-exporter:
+  prometheus:
+    monitor:
+      additionalLabels:
+        prometheus: default
+
+kube-state-metrics:
+  prometheus:
+    monitor:
+      additionalLabels:
+        prometheus: default