From e9302c51beca666d4ba4129675b8641e0f46e4e5 Mon Sep 17 00:00:00 2001
From: Peter Kurfer <peter.kurfer@rwe.com>
Date: Tue, 4 Feb 2025 16:42:17 +0100
Subject: [PATCH] feat(apigateway): allow to enable debug logging
---
api/v1alpha1/apigateway_types.go | 38 ++++++++++++++-
api/v1alpha1/zz_generated.deepcopy.go | 40 ++++++++++++++++
.../20221207154255_create_vault.sql | 2 +-
.../supabase.k8s.icb4dc0.de_apigateways.yaml | 27 +++++++++++
config/dev/apigateway.yaml | 5 ++
docs/api/supabase.k8s.icb4dc0.de.md | 47 +++++++++++++++++++
internal/controller/apigateway_controller.go | 29 ++++++++----
.../envoy_control_plane_config.yaml.tmpl | 2 +-
8 files changed, 177 insertions(+), 13 deletions(-)
diff --git a/api/v1alpha1/apigateway_types.go b/api/v1alpha1/apigateway_types.go
index 567b6fe..b1bd84a 100644
--- a/api/v1alpha1/apigateway_types.go
+++ b/api/v1alpha1/apigateway_types.go
@@ -19,6 +19,7 @@ package v1alpha1
import (
"iter"
"maps"
+ "strings"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -37,6 +38,40 @@ type ControlPlaneSpec struct {
Port uint16 `json:"port"`
}
+type EnvoyLogLevel string
+
+type EnvoyComponentLogLevel struct {
+ // Component - the component to set the log level for
+ // the component IDs can be found [here](https://github.com/envoyproxy/envoy/blob/main/source/common/common/logger.h#L36)
+ Component string `json:"component"`
+ // Level - the log level to set for the component
+ // +kubebuilder:validation:Enum=trace;debug;info;warning;error;critical;off
+ Level EnvoyLogLevel `json:"level"`
+}
+
+type EnvoyDebuggingOptions struct {
+ ComponentLogLevels []EnvoyComponentLogLevel `json:"componentLogLevels,omitempty"`
+}
+
+func (o *EnvoyDebuggingOptions) DebugLogging() string {
+ if o == nil || len(o.ComponentLogLevels) == 0 {
+ return ""
+ }
+
+ var builder strings.Builder
+ for i, lvl := range o.ComponentLogLevels {
+ if i > 0 {
+ builder.WriteString(",")
+ }
+
+ builder.WriteString(lvl.Component)
+ builder.WriteRune(':')
+ builder.WriteString(string(lvl.Level))
+ }
+
+ return builder.String()
+}
+
type EnvoySpec struct {
// NodeName - identifies the Envoy cluster within the current namespace
// if not set, the name of the APIGateway resource will be used
@@ -48,7 +83,8 @@ type EnvoySpec struct {
WorkloadTemplate *WorkloadTemplate `json:"workloadTemplate,omitempty"`
// DisableIPv6 - disable IPv6 for the Envoy instance
// this will force Envoy to use IPv4 for upstream hosts (mostly for the OAuth2 token endpoint)
- DisableIPv6 bool `json:"disableIPv6,omitempty"`
+ DisableIPv6 bool `json:"disableIPv6,omitempty"`
+ Debugging *EnvoyDebuggingOptions `json:"debugging,omitempty"`
}
type TlsCertRef struct {
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 029e1e8..8746d9e 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -873,6 +873,41 @@ func (in *EndpointTlsSpec) DeepCopy() *EndpointTlsSpec {
return out
}
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EnvoyComponentLogLevel) DeepCopyInto(out *EnvoyComponentLogLevel) {
+ *out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvoyComponentLogLevel.
+func (in *EnvoyComponentLogLevel) DeepCopy() *EnvoyComponentLogLevel {
+ if in == nil {
+ return nil
+ }
+ out := new(EnvoyComponentLogLevel)
+ in.DeepCopyInto(out)
+ return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EnvoyDebuggingOptions) DeepCopyInto(out *EnvoyDebuggingOptions) {
+ *out = *in
+ if in.ComponentLogLevels != nil {
+ in, out := &in.ComponentLogLevels, &out.ComponentLogLevels
+ *out = make([]EnvoyComponentLogLevel, len(*in))
+ copy(*out, *in)
+ }
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvoyDebuggingOptions.
+func (in *EnvoyDebuggingOptions) DeepCopy() *EnvoyDebuggingOptions {
+ if in == nil {
+ return nil
+ }
+ out := new(EnvoyDebuggingOptions)
+ in.DeepCopyInto(out)
+ return out
+}
+
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *EnvoySpec) DeepCopyInto(out *EnvoySpec) {
*out = *in
@@ -886,6 +921,11 @@ func (in *EnvoySpec) DeepCopyInto(out *EnvoySpec) {
*out = new(WorkloadTemplate)
(*in).DeepCopyInto(*out)
}
+ if in.Debugging != nil {
+ in, out := &in.Debugging, &out.Debugging
+ *out = new(EnvoyDebuggingOptions)
+ (*in).DeepCopyInto(*out)
+ }
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvoySpec.
diff --git a/assets/migrations/migrations/20221207154255_create_vault.sql b/assets/migrations/migrations/20221207154255_create_vault.sql
index f6d5012..d4f7141 100644
--- a/assets/migrations/migrations/20221207154255_create_vault.sql
+++ b/assets/migrations/migrations/20221207154255_create_vault.sql
@@ -9,7 +9,7 @@ BEGIN
-- for some reason extension custom scripts aren't run during AMI build, so
-- we manually run it here
grant usage on schema vault to postgres with grant option;
- grant select on vault.secrets, vault.decrypted_secrets to postgres with grant option;
+ grant select, delete on vault.secrets, vault.decrypted_secrets to postgres with grant option;
grant execute on function vault.create_secret, vault.update_secret, vault._crypto_aead_det_decrypt to postgres with grant option;
END IF;
END $$;
diff --git a/config/crd/bases/supabase.k8s.icb4dc0.de_apigateways.yaml b/config/crd/bases/supabase.k8s.icb4dc0.de_apigateways.yaml
index 4003876..1a2c783 100644
--- a/config/crd/bases/supabase.k8s.icb4dc0.de_apigateways.yaml
+++ b/config/crd/bases/supabase.k8s.icb4dc0.de_apigateways.yaml
@@ -234,6 +234,33 @@ spec:
- host
- port
type: object
+ debugging:
+ properties:
+ componentLogLevels:
+ items:
+ properties:
+ component:
+ description: |-
+ Component - the component to set the log level for
+ the component IDs can be found [here](https://github.com/envoyproxy/envoy/blob/main/source/common/common/logger.h#L36)
+ type: string
+ level:
+ description: Level - the log level to set for the component
+ enum:
+ - trace
+ - debug
+ - info
+ - warning
+ - error
+ - critical
+ - "off"
+ type: string
+ required:
+ - component
+ - level
+ type: object
+ type: array
+ type: object
disableIPv6:
description: |-
DisableIPv6 - disable IPv6 for the Envoy instance
diff --git a/config/dev/apigateway.yaml b/config/dev/apigateway.yaml
index eda5e3c..2cb1c78 100644
--- a/config/dev/apigateway.yaml
+++ b/config/dev/apigateway.yaml
@@ -8,6 +8,11 @@ metadata:
name: gateway-sample
namespace: supabase-demo
spec:
+ envoy:
+ debugging:
+ componentLogLevels:
+ - component: oauth2
+ level: debug
apiEndpoint:
jwks:
name: core-sample-jwt
diff --git a/docs/api/supabase.k8s.icb4dc0.de.md b/docs/api/supabase.k8s.icb4dc0.de.md
index b81bd05..be0cf47 100644
--- a/docs/api/supabase.k8s.icb4dc0.de.md
+++ b/docs/api/supabase.k8s.icb4dc0.de.md
@@ -594,6 +594,52 @@ _Appears in:_
| `cert` _[TlsCertRef](#tlscertref)_ | | | |
+#### EnvoyComponentLogLevel
+
+
+
+
+
+
+
+_Appears in:_
+- [EnvoyDebuggingOptions](#envoydebuggingoptions)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `component` _string_ | Component - the component to set the log level for<br />the component IDs can be found [here](https://github.com/envoyproxy/envoy/blob/main/source/common/common/logger.h#L36) | | |
+| `level` _[EnvoyLogLevel](#envoyloglevel)_ | Level - the log level to set for the component | | Enum: [trace debug info warning error critical off] <br /> |
+
+
+#### EnvoyDebuggingOptions
+
+
+
+
+
+
+
+_Appears in:_
+- [EnvoySpec](#envoyspec)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `componentLogLevels` _[EnvoyComponentLogLevel](#envoycomponentloglevel) array_ | | | |
+
+
+#### EnvoyLogLevel
+
+_Underlying type:_ _string_
+
+
+
+
+
+_Appears in:_
+- [EnvoyComponentLogLevel](#envoycomponentloglevel)
+
+
+
#### EnvoySpec
@@ -611,6 +657,7 @@ _Appears in:_
| `controlPlane` _[ControlPlaneSpec](#controlplanespec)_ | ControlPlane - configure the control plane where Envoy will retrieve its configuration from | | |
| `workloadTemplate` _[WorkloadTemplate](#workloadtemplate)_ | WorkloadTemplate - customize the Envoy deployment | | |
| `disableIPv6` _boolean_ | DisableIPv6 - disable IPv6 for the Envoy instance<br />this will force Envoy to use IPv4 for upstream hosts (mostly for the OAuth2 token endpoint) | | |
+| `debugging` _[EnvoyDebuggingOptions](#envoydebuggingoptions)_ | | | |
#### EnvoyStatus
diff --git a/internal/controller/apigateway_controller.go b/internal/controller/apigateway_controller.go
index 95f4d40..0b7807e 100644
--- a/internal/controller/apigateway_controller.go
+++ b/internal/controller/apigateway_controller.go
@@ -348,12 +348,15 @@ func (r *APIGatewayReconciler) reconcileEnvoyConfig(
ctx context.Context,
gateway *supabasev1alpha1.APIGateway,
) (configHash string, err error) {
- configMap := &corev1.ConfigMap{
- ObjectMeta: metav1.ObjectMeta{
- Name: supabase.ServiceConfig.Envoy.ObjectName(gateway),
- Namespace: gateway.Namespace,
- },
- }
+ var (
+ envoySpec = gateway.Spec.Envoy
+ configMap = &corev1.ConfigMap{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: supabase.ServiceConfig.Envoy.ObjectName(gateway),
+ Namespace: gateway.Namespace,
+ },
+ }
+ )
_, err = controllerutil.CreateOrUpdate(ctx, r.Client, configMap, func() error {
configMap.Labels = MergeLabels(objectLabels(gateway, "envoy", "api-gateway", supabase.Images.Envoy.Tag), gateway.Labels)
@@ -369,7 +372,7 @@ func (r *APIGatewayReconciler) reconcileEnvoyConfig(
Port uint16
}
- instance := fmt.Sprintf("%s:%s", gateway.Spec.Envoy.NodeName, gateway.Namespace)
+ instance := fmt.Sprintf("%s:%s", envoySpec.NodeName, gateway.Namespace)
tmplData := struct {
Node nodeSpec
@@ -381,8 +384,8 @@ func (r *APIGatewayReconciler) reconcileEnvoyConfig(
},
ControlPlane: controlPlaneSpec{
Name: "supabase-control-plane",
- Host: gateway.Spec.Envoy.ControlPlane.Host,
- Port: gateway.Spec.Envoy.ControlPlane.Port,
+ Host: envoySpec.ControlPlane.Host,
+ Port: envoySpec.ControlPlane.Port,
},
}
@@ -446,6 +449,12 @@ func (r *APIGatewayReconciler) reconileEnvoyDeployment(
envoyDeployment.Spec.Replicas = envoySpec.WorkloadTemplate.ReplicaCount()
+ envoyArgs := []string{"-c /etc/envoy/config.yaml"}
+
+ if componentLogLevels := envoySpec.Debugging.DebugLogging(); len(componentLogLevels) > 0 {
+ envoyArgs = append(envoyArgs, "--component-log-level", componentLogLevels)
+ }
+
envoyDeployment.Spec.Template = corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
@@ -462,7 +471,7 @@ func (r *APIGatewayReconciler) reconileEnvoyDeployment(
Name: "envoy-proxy",
Image: envoySpec.WorkloadTemplate.Image(supabase.Images.Envoy.String()),
ImagePullPolicy: envoySpec.WorkloadTemplate.ImagePullPolicy(),
- Args: []string{"-c /etc/envoy/config.yaml"}, // , "--component-log-level", "upstream:debug,connection:debug"
+ Args: envoyArgs,
Ports: []corev1.ContainerPort{
{
Name: serviceCfg.Defaults.StudioPortName,
diff --git a/internal/controller/templates/envoy_control_plane_config.yaml.tmpl b/internal/controller/templates/envoy_control_plane_config.yaml.tmpl
index 4a797a8..9d35f49 100644
--- a/internal/controller/templates/envoy_control_plane_config.yaml.tmpl
+++ b/internal/controller/templates/envoy_control_plane_config.yaml.tmpl
@@ -47,13 +47,13 @@ static_resources:
trusted_ca:
filename: /etc/envoy/certs/cp/ca.crt
-
admin:
address:
socket_address:
address: 0.0.0.0
port_value: 19000
+
application_log_config:
log_format:
json_format: