From 0fccef973fc9ee2a97b9e8d6609ec037113a4742 Mon Sep 17 00:00:00 2001 From: Peter Kurfer <peter.kurfer@rwe.com> Date: Mon, 3 Feb 2025 09:57:05 +0100 Subject: [PATCH] feat(dashboard): PoC Oauth2 auth --- Tiltfile | 4 +- api/v1alpha1/apigateway_types.go | 99 +++- api/v1alpha1/zz_generated.deepcopy.go | 124 ++++- cmd/control_plane.go | 167 +++++- cmd/flags.go | 41 ++ cmd/manager.go | 10 + config/control-plane/cert-ca.yaml | 33 ++ config/control-plane/control-plane.yaml | 59 +- config/control-plane/kustomization.yaml | 4 + config/control-plane/kustomizeconfig.yaml | 8 + .../supabase.k8s.icb4dc0.de_apigateways.yaml | 131 +++++ config/default/kustomization.yaml | 22 +- config/dev/.gitignore | 1 + config/dev/apigateway.yaml | 37 ++ config/{samples => dev}/cnpg-cluster.yaml | 4 + config/dev/core.yaml | 2 + config/dev/dashboard.yaml | 1 + config/dev/kustomization.yaml | 3 + config/dev/minio-operator.yaml | 3 - config/dev/minio.yaml | 2 +- config/dev/namespace.yaml | 4 + config/dev/storage.yaml | 2 + config/dev/studio-credentials-secret.yaml | 8 + config/manager/manager.yaml | 33 +- config/rbac/control-plane-role.yaml | 13 + config/samples/kustomization.yaml | 1 - dev/cluster.yaml | 3 + go.mod | 23 +- go.sum | 44 +- internal/certs/format.go | 60 +++ internal/certs/generate.go | 122 +++++ internal/controller/apigateway_controller.go | 341 ++++++++++-- internal/controller/core_gotrue_controller.go | 4 +- .../controller/core_postgrest_controller.go | 2 +- .../dashboard_pg-meta_controller.go | 4 +- .../controller/dashboard_studio_controller.go | 4 +- internal/controller/storage_api_controller.go | 4 +- .../controller/storage_imgproxy_controller.go | 2 +- .../envoy_control_plane_config.yaml.tmpl | 39 +- .../controlplane/apigateway_controller.go | 41 +- internal/controlplane/endpoints.go | 19 - internal/controlplane/filters.go | 2 + internal/controlplane/logging.go | 62 +++ internal/controlplane/snapshot.go | 504 ++++++++++++++---- internal/controlplane/studio.go | 44 ++ internal/health/cert_valid.go | 42 ++ internal/supabase/auth.go | 3 +- internal/supabase/env.go | 19 +- internal/supabase/envoy.go | 29 +- internal/supabase/pg_meta.go | 3 +- internal/supabase/postgrest.go | 3 +- internal/supabase/storage.go | 3 +- internal/supabase/studio.go | 3 +- 53 files changed, 1914 insertions(+), 331 deletions(-) create mode 100644 cmd/flags.go create mode 100644 config/control-plane/cert-ca.yaml create mode 100644 config/control-plane/kustomizeconfig.yaml create mode 100644 config/dev/.gitignore rename config/{samples => dev}/cnpg-cluster.yaml (95%) delete mode 100644 config/dev/minio-operator.yaml create mode 100644 config/dev/namespace.yaml create mode 100644 config/dev/studio-credentials-secret.yaml create mode 100644 internal/certs/format.go create mode 100644 internal/certs/generate.go create mode 100644 internal/controlplane/logging.go create mode 100644 internal/health/cert_valid.go diff --git a/Tiltfile b/Tiltfile index 0dd859c..91f534d 100644 --- a/Tiltfile +++ b/Tiltfile @@ -61,7 +61,7 @@ k8s_resource( k8s_resource( objects=["gateway-sample:APIGateway:supabase-demo"], extra_pod_selectors={"app.kubernetes.io/component": "api-gateway"}, - port_forwards=[8000, 19000], + port_forwards=[3000, 8000, 19000], new_name='API Gateway', resource_deps=[ 'supabase-controller-manager' @@ -72,7 +72,7 @@ k8s_resource( objects=["dashboard-sample:Dashboard:supabase-demo"], extra_pod_selectors={"app.kubernetes.io/component": "dashboard", "app.kubernetes.io/name": "studio"}, discovery_strategy="selectors-only", - port_forwards=[3000], + # port_forwards=[3000], new_name='Dashboard', resource_deps=[ 'supabase-controller-manager' diff --git a/api/v1alpha1/apigateway_types.go b/api/v1alpha1/apigateway_types.go index 4c03dbb..177ed52 100644 --- a/api/v1alpha1/apigateway_types.go +++ b/api/v1alpha1/apigateway_types.go @@ -46,14 +46,111 @@ type EnvoySpec struct { ControlPlane *ControlPlaneSpec `json:"controlPlane"` // WorkloadTemplate - customize the Envoy deployment WorkloadTemplate *WorkloadTemplate `json:"workloadTemplate,omitempty"` + // DisableIPv6 - disable IPv6 for the Envoy instance + // this will force Envoy to use IPv4 for upstream hosts (mostly for the OAuth2 token endpoint) + DisableIPv6 bool `json:"disableIPv6,omitempty"` +} + +type TlsCertRef struct { + SecretName string `json:"secretName"` + // ServerCertKey - key in the secret that contains the server certificate + // +kubebuilder:default="tls.crt" + ServerCertKey string `json:"serverCertKey"` + // ServerKeyKey - key in the secret that contains the server private key + // +kubebuilder:default="tls.key" + ServerKeyKey string `json:"serverKeyKey"` + // CaCertKey - key in the secret that contains the CA certificate + // +kubebuilder:default="ca.crt" + CaCertKey string `json:"caCertKey,omitempty"` +} + +type EndpointTlsSpec struct { + Cert *TlsCertRef `json:"cert"` } type ApiEndpointSpec struct { // JWKSSelector - selector where the JWKS can be retrieved from to enable the API gateway to validate JWTs JWKSSelector *corev1.SecretKeySelector `json:"jwks"` + // TLS - enable and configure TLS for the API endpoint + TLS *EndpointTlsSpec `json:"tls,omitempty"` } -type DashboardEndpointSpec struct{} +func (s *ApiEndpointSpec) TLSSpec() *EndpointTlsSpec { + if s == nil { + return nil + } + + return s.TLS +} + +type DashboardAuthType string + +const ( + DashboardAuthTypeNone DashboardAuthType = "none" + DashboardAuthTypeOAuth2 DashboardAuthType = "oauth2" + DashboardAuthTypeBasic DashboardAuthType = "basic" +) + +type DashboardOAuth2Spec struct { + // TokenEndpoint - endpoint where Envoy will retrieve the OAuth2 access and identity token from + TokenEndpoint string `json:"tokenEndpoint"` + // AuthorizationEndpoint - endpoint where the user will be redirected to authenticate + AuthorizationEndpoint string `json:"authorizationEndpoint"` + // ClientID - client ID to authenticate with the OAuth2 provider + ClientID string `json:"clientId"` + // Scopes - scopes to request from the OAuth2 provider (e.g. "openid", "profile", ...) - optional + Scopes []string `json:"scopes,omitempty"` + // Resources - resources to request from the OAuth2 provider (e.g. "user", "email", ...) - optional + Resources []string `json:"resources,omitempty"` + // ClientSecretRef - reference to the secret that contains the client secret + ClientSecretRef *corev1.SecretKeySelector `json:"clientSecretRef"` +} + +type DashboardBasicAuthSpec struct{} + +type DashboardAuthSpec struct { + OAuth2 *DashboardOAuth2Spec `json:"oauth2,omitempty"` + Basic *DashboardBasicAuthSpec `json:"basic,omitempty"` +} + +type DashboardEndpointSpec struct { + // Auth - configure authentication for the dashboard endpoint + Auth *DashboardAuthSpec `json:"auth,omitempty"` + // TLS - enable and configure TLS for the Dashboard endpoint + TLS *EndpointTlsSpec `json:"tls,omitempty"` +} + +func (s *DashboardEndpointSpec) TLSSpec() *EndpointTlsSpec { + if s == nil { + return nil + } + + return s.TLS +} + +func (s *DashboardEndpointSpec) AuthType() DashboardAuthType { + if s == nil || s.Auth == nil { + return DashboardAuthTypeNone + } + + if s.Auth.OAuth2 != nil { + return DashboardAuthTypeOAuth2 + } + + if s.Auth.Basic != nil { + return DashboardAuthTypeBasic + } + + return DashboardAuthTypeNone +} + +func (s *DashboardEndpointSpec) OAuth2() *DashboardOAuth2Spec { + if s == nil || s.Auth == nil { + return nil + } + + return s.Auth.OAuth2 +} // APIGatewaySpec defines the desired state of APIGateway. type APIGatewaySpec struct { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6aa26ea..029e1e8 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1alpha1 import ( - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -101,7 +101,7 @@ func (in *APIGatewaySpec) DeepCopyInto(out *APIGatewaySpec) { if in.DashboardEndpoint != nil { in, out := &in.DashboardEndpoint, &out.DashboardEndpoint *out = new(DashboardEndpointSpec) - **out = **in + (*in).DeepCopyInto(*out) } if in.ServiceSelector != nil { in, out := &in.ServiceSelector, &out.ServiceSelector @@ -160,6 +160,11 @@ func (in *ApiEndpointSpec) DeepCopyInto(out *ApiEndpointSpec) { *out = new(v1.SecretKeySelector) (*in).DeepCopyInto(*out) } + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(EndpointTlsSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ApiEndpointSpec. @@ -490,6 +495,46 @@ func (in *Dashboard) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DashboardAuthSpec) DeepCopyInto(out *DashboardAuthSpec) { + *out = *in + if in.OAuth2 != nil { + in, out := &in.OAuth2, &out.OAuth2 + *out = new(DashboardOAuth2Spec) + (*in).DeepCopyInto(*out) + } + if in.Basic != nil { + in, out := &in.Basic, &out.Basic + *out = new(DashboardBasicAuthSpec) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DashboardAuthSpec. +func (in *DashboardAuthSpec) DeepCopy() *DashboardAuthSpec { + if in == nil { + return nil + } + out := new(DashboardAuthSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DashboardBasicAuthSpec) DeepCopyInto(out *DashboardBasicAuthSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DashboardBasicAuthSpec. +func (in *DashboardBasicAuthSpec) DeepCopy() *DashboardBasicAuthSpec { + if in == nil { + return nil + } + out := new(DashboardBasicAuthSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DashboardDbSpec) DeepCopyInto(out *DashboardDbSpec) { *out = *in @@ -513,6 +558,16 @@ func (in *DashboardDbSpec) DeepCopy() *DashboardDbSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DashboardEndpointSpec) DeepCopyInto(out *DashboardEndpointSpec) { *out = *in + if in.Auth != nil { + in, out := &in.Auth, &out.Auth + *out = new(DashboardAuthSpec) + (*in).DeepCopyInto(*out) + } + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(EndpointTlsSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DashboardEndpointSpec. @@ -557,6 +612,36 @@ func (in *DashboardList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DashboardOAuth2Spec) DeepCopyInto(out *DashboardOAuth2Spec) { + *out = *in + if in.Scopes != nil { + in, out := &in.Scopes, &out.Scopes + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ClientSecretRef != nil { + in, out := &in.ClientSecretRef, &out.ClientSecretRef + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DashboardOAuth2Spec. +func (in *DashboardOAuth2Spec) DeepCopy() *DashboardOAuth2Spec { + if in == nil { + return nil + } + out := new(DashboardOAuth2Spec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DashboardSpec) DeepCopyInto(out *DashboardSpec) { *out = *in @@ -768,6 +853,26 @@ func (in *EmailAuthSmtpSpec) DeepCopy() *EmailAuthSmtpSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EndpointTlsSpec) DeepCopyInto(out *EndpointTlsSpec) { + *out = *in + if in.Cert != nil { + in, out := &in.Cert, &out.Cert + *out = new(TlsCertRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EndpointTlsSpec. +func (in *EndpointTlsSpec) DeepCopy() *EndpointTlsSpec { + if in == nil { + return nil + } + out := new(EndpointTlsSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EnvoySpec) DeepCopyInto(out *EnvoySpec) { *out = *in @@ -1261,6 +1366,21 @@ func (in *StudioSpec) DeepCopy() *StudioSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TlsCertRef) DeepCopyInto(out *TlsCertRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TlsCertRef. +func (in *TlsCertRef) DeepCopy() *TlsCertRef { + if in == nil { + return nil + } + out := new(TlsCertRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *UploadTempSpec) DeepCopyInto(out *UploadTempSpec) { *out = *in diff --git a/cmd/control_plane.go b/cmd/control_plane.go index aada234..bf936d5 100644 --- a/cmd/control_plane.go +++ b/cmd/control_plane.go @@ -19,8 +19,10 @@ package main import ( "context" "crypto/tls" + "crypto/x509" "fmt" "net" + "strings" "time" clusterservice "github.com/envoyproxy/go-control-plane/envoy/service/cluster/v3" @@ -32,30 +34,52 @@ import ( secretservice "github.com/envoyproxy/go-control-plane/envoy/service/secret/v3" cachev3 "github.com/envoyproxy/go-control-plane/pkg/cache/v3" "github.com/envoyproxy/go-control-plane/pkg/server/v3" + "google.golang.org/grpc/credentials" + + "github.com/go-logr/logr" + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" "google.golang.org/grpc" grpchealth "google.golang.org/grpc/health" "google.golang.org/grpc/health/grpc_health_v1" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/reflection" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/healthz" mgr "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "code.icb4dc0.de/prskr/supabase-operator/internal/certs" "code.icb4dc0.de/prskr/supabase-operator/internal/controlplane" + "code.icb4dc0.de/prskr/supabase-operator/internal/health" ) //nolint:lll // flag declaration with struct tags is as long as it is type controlPlane struct { - ListenAddr string `name:"listen-address" default:":18000" help:"The address the control plane binds to."` + caCert tls.Certificate `kong:"-"` + + ListenAddr string `name:"listen-address" default:":18000" help:"The address the control plane binds to."` + Tls struct { + CA struct { + Cert FileContent `env:"CERT" name:"server-cert" required:"" help:"The path to the server certificate file."` + Key FileContent `env:"KEY" name:"server-key" required:"" help:"The path to the server key file."` + } `embed:"" prefix:"ca." envprefix:"CA_"` + ServerSecretName string `name:"server-secret-name" help:"The name of the secret containing the server certificate and key." default:"control-plane-xds-tls"` + } `embed:"" prefix:"tls." envprefix:"TLS_"` MetricsAddr string `name:"metrics-bind-address" default:"0" help:"The address the metrics endpoint binds to. Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service."` EnableLeaderElection bool `name:"leader-elect" default:"false" help:"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager."` ProbeAddr string `name:"health-probe-bind-address" default:":8081" help:"The address the probe endpoint binds to."` SecureMetrics bool `name:"metrics-secure" default:"true" help:"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead."` EnableHTTP2 bool `name:"enable-http2" default:"false" help:"If set, HTTP/2 will be enabled for the metrics and webhook servers"` + ServiceName string `name:"service-name" env:"CONTROL_PLANE_SERVICE_NAME" default:"" required:"" help:"The name of the control plane service."` + Namespace string `name:"namespace" env:"CONTROL_PLANE_NAMESPACE" default:"" required:"" help:"Namespace where the controller is running, ideally set via downward API"` } -func (cp controlPlane) Run(ctx context.Context) error { +func (cp *controlPlane) Run(ctx context.Context) error { var tlsOpts []func(*tls.Config) // if the enable-http2 flag is false (the default), http/2 should be disabled @@ -91,6 +115,11 @@ func (cp controlPlane) Run(ctx context.Context) error { metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization } + bootstrapClient, err := client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme}) + if err != nil { + return fmt.Errorf("unable to create bootstrap client: %w", err) + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, Metrics: metricsServerOptions, @@ -106,7 +135,12 @@ func (cp controlPlane) Run(ctx context.Context) error { envoySnapshotCache := cachev3.NewSnapshotCache(false, cachev3.IDHash{}, nil) - envoySrv, err := cp.envoyServer(ctx, envoySnapshotCache) + serverCert, err := cp.ensureControlPlaneTlsCert(ctx, bootstrapClient) + if err != nil { + return fmt.Errorf("failed to ensure control plane TLS cert: %w", err) + } + + envoySrv, err := cp.envoyServer(ctx, envoySnapshotCache, serverCert) if err != nil { return err } @@ -123,6 +157,18 @@ func (cp controlPlane) Run(ctx context.Context) error { return fmt.Errorf("unable to create controller Core DB: %w", err) } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + return fmt.Errorf("unable to set up health check: %w", err) + } + + if err := mgr.AddHealthzCheck("server-cert", health.CertValidCheck(serverCert)); err != nil { + return fmt.Errorf("unable to set up health check: %w", err) + } + + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + return fmt.Errorf("unable to set up ready check: %w", err) + } + setupLog.Info("starting manager") if err := mgr.Start(ctx); err != nil { return fmt.Errorf("problem running manager: %w", err) @@ -131,9 +177,19 @@ func (cp controlPlane) Run(ctx context.Context) error { return nil } -func (cp controlPlane) envoyServer( +func (cp *controlPlane) AfterApply() (err error) { + cp.caCert, err = tls.X509KeyPair(cp.Tls.CA.Cert, cp.Tls.CA.Key) + if err != nil { + return fmt.Errorf("failed to parse server certificate: %w", err) + } + + return nil +} + +func (cp *controlPlane) envoyServer( ctx context.Context, cache cachev3.SnapshotCache, + serverCert tls.Certificate, ) (runnable mgr.Runnable, err error) { const ( grpcKeepaliveTime = 30 * time.Second @@ -153,7 +209,17 @@ func (cp controlPlane) envoyServer( // availability problems. Keepalive timeouts based on connection_keepalive parameter // https://www.envoyproxy.io/docs/envoy/latest/configuration/overview/examples#dynamic + tlsCfg, err := cp.tlsConfig(serverCert) + if err != nil { + return nil, fmt.Errorf("failed to create TLS config: %w", err) + } + + loggingOpts := []logging.Option{ + logging.WithLogOnEvents(logging.StartCall, logging.FinishCall), + } + grpcOptions := append(make([]grpc.ServerOption, 0, 4), + grpc.Creds(credentials.NewTLS(tlsCfg)), grpc.MaxConcurrentStreams(grpcMaxConcurrentStreams), grpc.KeepaliveParams(keepalive.ServerParameters{ Time: grpcKeepaliveTime, @@ -163,6 +229,12 @@ func (cp controlPlane) envoyServer( MinTime: grpcKeepaliveMinTime, PermitWithoutStream: true, }), + grpc.ChainUnaryInterceptor( + logging.UnaryServerInterceptor(InterceptorLogger(ctrl.Log), loggingOpts...), + ), + grpc.ChainStreamInterceptor( + logging.StreamServerInterceptor(InterceptorLogger(ctrl.Log), loggingOpts...), + ), ) grpcServer := grpc.NewServer(grpcOptions...) @@ -195,3 +267,90 @@ func (cp controlPlane) envoyServer( return grpcServer.Serve(lis) }), nil } + +func (cp *controlPlane) ensureControlPlaneTlsCert(ctx context.Context, k8sClient client.Client) (tls.Certificate, error) { + var ( + controlPlaneServerCert = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: cp.Tls.ServerSecretName, + Namespace: cp.Namespace, + }, + } + serverCert tls.Certificate + ) + + _, err := controllerutil.CreateOrUpdate(ctx, k8sClient, controlPlaneServerCert, func() (err error) { + controlPlaneServerCert.Type = corev1.SecretTypeTLS + + if controlPlaneServerCert.Data == nil { + controlPlaneServerCert.Data = make(map[string][]byte, 3) + } + + var ( + cert = controlPlaneServerCert.Data[corev1.TLSCertKey] + privateKey = controlPlaneServerCert.Data[corev1.TLSPrivateKeyKey] + ) + + var requireRenewal bool + if cert != nil && privateKey != nil { + if serverCert, err = tls.X509KeyPair(cert, privateKey); err != nil { + return fmt.Errorf("failed to parse server certificate: %w", err) + } + + renewGracePeriod := time.Duration(float64(serverCert.Leaf.NotAfter.Sub(serverCert.Leaf.NotBefore)) * 0.1) + if serverCert.Leaf.NotAfter.Before(time.Now().Add(-renewGracePeriod)) { + requireRenewal = true + } + } else { + requireRenewal = true + } + + if requireRenewal { + dnsNames := []string{ + strings.Join([]string{cp.ServiceName, cp.Namespace, "svc"}, "."), + strings.Join([]string{cp.ServiceName, cp.Namespace, "svc", "cluster", "local"}, "."), + } + if certResult, err := certs.ServerCert("supabase-control-plane", dnsNames, cp.caCert); err != nil { + return fmt.Errorf("failed to generate server certificate: %w", err) + } else { + serverCert = certResult.ServerCert + controlPlaneServerCert.Data[corev1.TLSCertKey] = certResult.PublicKey + controlPlaneServerCert.Data[corev1.TLSPrivateKeyKey] = certResult.PrivateKey + } + } + + return nil + }) + if err != nil { + return tls.Certificate{}, fmt.Errorf("failed to create or update control plane server certificate: %w", err) + } + + return serverCert, nil +} + +func (cp *controlPlane) tlsConfig(serverCert tls.Certificate) (*tls.Config, error) { + tlsCfg := &tls.Config{ + RootCAs: x509.NewCertPool(), + ClientCAs: x509.NewCertPool(), + ClientAuth: tls.RequireAndVerifyClientCert, + } + + tlsCfg.Certificates = append(tlsCfg.Certificates, serverCert) + if !tlsCfg.RootCAs.AppendCertsFromPEM(cp.Tls.CA.Cert) { + return nil, fmt.Errorf("failed to parse CA certificate") + } + + if !tlsCfg.ClientCAs.AppendCertsFromPEM(cp.Tls.CA.Cert) { + return nil, fmt.Errorf("failed to parse client CA certificate") + } + + return tlsCfg, nil +} + +// InterceptorLogger adapts slog logger to interceptor logger. +// This code is simple enough to be copied and not imported. +func InterceptorLogger(l logr.Logger) logging.Logger { + return logging.LoggerFunc(func(ctx context.Context, lvl logging.Level, msg string, fields ...any) { + l.Info(msg, fields...) + }) +} diff --git a/cmd/flags.go b/cmd/flags.go new file mode 100644 index 0000000..04dc09e --- /dev/null +++ b/cmd/flags.go @@ -0,0 +1,41 @@ +/* +Copyright 2025 Peter Kurfer. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + + "github.com/alecthomas/kong" +) + +var _ kong.MapperValue = (*FileContent)(nil) + +type FileContent []byte + +func (f *FileContent) Decode(ctx *kong.DecodeContext) (err error) { + var filePath string + if err := ctx.Scan.PopValueInto("file-content", &filePath); err != nil { + return err + } + + if *f, err = os.ReadFile(filePath); err != nil { + return fmt.Errorf("failed to read file: %w", err) + } + + return nil +} diff --git a/cmd/manager.go b/cmd/manager.go index ac4e3aa..292c0b9 100644 --- a/cmd/manager.go +++ b/cmd/manager.go @@ -40,6 +40,10 @@ type manager struct { SecureMetrics bool `name:"metrics-secure" default:"true" help:"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead."` EnableHTTP2 bool `name:"enable-http2" default:"false" help:"If set, HTTP/2 will be enabled for the metrics and webhook servers"` Namespace string `name:"controller-namespace" env:"CONTROLLER_NAMESPACE" default:"" help:"Namespace where the controller is running, ideally set via downward API"` + Tls struct { + CACert FileContent `env:"CA_CERT" name:"ca-cert" required:"" help:"The path to the CA certificate file."` + CAKey FileContent `env:"CA_KEY" name:"ca-key" required:"" help:"The path to the CA key file."` + } `embed:"" prefix:"tls." envprefix:"TLS_"` } func (m manager) Run(ctx context.Context) error { @@ -68,6 +72,11 @@ func (m manager) Run(ctx context.Context) error { TLSOpts: tlsOpts, }) + caCert, err := tls.X509KeyPair(m.Tls.CACert, m.Tls.CAKey) + if err != nil { + return fmt.Errorf("unable to load CA cert: %w", err) + } + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. // More info: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server @@ -145,6 +154,7 @@ func (m manager) Run(ctx context.Context) error { if err = (&controller.APIGatewayReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), + CACert: caCert, }).SetupWithManager(ctx, mgr); err != nil { return fmt.Errorf("unable to create controller APIGateway: %w", err) } diff --git a/config/control-plane/cert-ca.yaml b/config/control-plane/cert-ca.yaml new file mode 100644 index 0000000..4e0113d --- /dev/null +++ b/config/control-plane/cert-ca.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: supabase-operator + app.kubernetes.io/managed-by: kustomize + name: cp-selfsigned-issuer + namespace: system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: certificate + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: supabase-operator + app.kubernetes.io/part-of: supabase-operator + app.kubernetes.io/managed-by: kustomize + name: cp-ca-cert + namespace: supabase-system +spec: + commonName: control-plane-ca + privateKey: + algorithm: ECDSA + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: control-plane-ca-cert-tls + isCA: true diff --git a/config/control-plane/control-plane.yaml b/config/control-plane/control-plane.yaml index 9997476..c55d2b7 100644 --- a/config/control-plane/control-plane.yaml +++ b/config/control-plane/control-plane.yaml @@ -18,40 +18,26 @@ spec: labels: app.kubernetes.io/name: control-plane spec: - # TODO(user): Uncomment the following code to configure the nodeAffinity expression - # according to the platforms which are supported by your solution. - # It is considered best practice to support multiple architectures. You can - # build your manager image using the makefile target docker-buildx. - # affinity: - # nodeAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: kubernetes.io/arch - # operator: In - # values: - # - amd64 - # - arm64 - # - ppc64le - # - s390x - # - key: kubernetes.io/os - # operator: In - # values: - # - linux securityContext: runAsNonRoot: true - # TODO(user): For common cases that do not require escalating privileges - # it is recommended to ensure that all your Pods/Containers are restrictive. - # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted - # Please uncomment the following code if your project does NOT have to work on old Kubernetes - # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). - # seccompProfile: - # type: RuntimeDefault + seccompProfile: + type: RuntimeDefault containers: - args: - control-plane image: supabase-operator:latest name: control-plane + env: + - name: CONTROL_PLANE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CONTROL_PLANE_SERVICE_NAME + value: control-plane + - name: TLS_CA_CERT + value: /etc/supabase/control-plane/certs/tls.crt + - name: TLS_CA_KEY + value: /etc/supabase/control-plane/certs/tls.key ports: - containerPort: 18000 name: grpc @@ -62,17 +48,17 @@ spec: drop: - "ALL" livenessProbe: - grpc: - port: 18000 + httpGet: + path: /healthz + port: 8081 initialDelaySeconds: 15 periodSeconds: 20 readinessProbe: - grpc: - port: 18000 + httpGet: + path: /readyz + port: 8081 initialDelaySeconds: 5 periodSeconds: 10 - # TODO(user): Configure the resources accordingly based on the project requirements. - # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ resources: limits: cpu: 150m @@ -80,5 +66,12 @@ spec: requests: cpu: 50m memory: 64Mi + volumeMounts: + - name: tls-certs + mountPath: /etc/supabase/control-plane/certs + volumes: + - name: tls-certs + secret: + secretName: control-plane-ca-cert-tls serviceAccountName: control-plane terminationGracePeriodSeconds: 10 diff --git a/config/control-plane/kustomization.yaml b/config/control-plane/kustomization.yaml index 0233292..bc86d33 100644 --- a/config/control-plane/kustomization.yaml +++ b/config/control-plane/kustomization.yaml @@ -2,5 +2,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - cert-ca.yaml - control-plane.yaml - service.yaml + +configurations: + - kustomizeconfig.yaml diff --git a/config/control-plane/kustomizeconfig.yaml b/config/control-plane/kustomizeconfig.yaml new file mode 100644 index 0000000..cf6f89e --- /dev/null +++ b/config/control-plane/kustomizeconfig.yaml @@ -0,0 +1,8 @@ +# This configuration is for teaching kustomize how to update name ref substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name diff --git a/config/crd/bases/supabase.k8s.icb4dc0.de_apigateways.yaml b/config/crd/bases/supabase.k8s.icb4dc0.de_apigateways.yaml index ba7c81b..c5e5cc0 100644 --- a/config/crd/bases/supabase.k8s.icb4dc0.de_apigateways.yaml +++ b/config/crd/bases/supabase.k8s.icb4dc0.de_apigateways.yaml @@ -73,6 +73,36 @@ spec: - key type: object x-kubernetes-map-type: atomic + tls: + description: TLS - enable and configure TLS for the API endpoint + properties: + cert: + properties: + caCertKey: + default: ca.crt + description: CaCertKey - key in the secret that contains + the CA certificate + type: string + secretName: + type: string + serverCertKey: + default: tls.crt + description: ServerCertKey - key in the secret that contains + the server certificate + type: string + serverKeyKey: + default: tls.key + description: ServerKeyKey - key in the secret that contains + the server private key + type: string + required: + - secretName + - serverCertKey + - serverKeyKey + type: object + required: + - cert + type: object required: - jwks type: object @@ -85,6 +115,102 @@ spec: description: |- DashboardEndpoint - Configure the endpoint for the Supabase dashboard (studio) this includes optional authentication (basic or Oauth2) for the dashboard + properties: + auth: + description: Auth - configure authentication for the dashboard + endpoint + properties: + basic: + type: object + oauth2: + properties: + authorizationEndpoint: + description: AuthorizationEndpoint - endpoint where the + user will be redirected to authenticate + type: string + clientId: + description: ClientID - client ID to authenticate with + the OAuth2 provider + type: string + clientSecretRef: + description: ClientSecretRef - reference to the secret + that contains the client secret + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + resources: + description: Resources - resources to request from the + OAuth2 provider (e.g. "user", "email", ...) - optional + items: + type: string + type: array + scopes: + description: Scopes - scopes to request from the OAuth2 + provider (e.g. "openid", "profile", ...) - optional + items: + type: string + type: array + tokenEndpoint: + description: TokenEndpoint - endpoint where Envoy will + retrieve the OAuth2 access and identity token from + type: string + required: + - authorizationEndpoint + - clientId + - clientSecretRef + - tokenEndpoint + type: object + type: object + tls: + description: TLS - enable and configure TLS for the Dashboard + endpoint + properties: + cert: + properties: + caCertKey: + default: ca.crt + description: CaCertKey - key in the secret that contains + the CA certificate + type: string + secretName: + type: string + serverCertKey: + default: tls.crt + description: ServerCertKey - key in the secret that contains + the server certificate + type: string + serverKeyKey: + default: tls.key + description: ServerKeyKey - key in the secret that contains + the server private key + type: string + required: + - secretName + - serverCertKey + - serverKeyKey + type: object + required: + - cert + type: object type: object envoy: description: Envoy - configure the envoy instance and most importantly @@ -108,6 +234,11 @@ spec: - host - port type: object + disableIPv6: + description: |- + DisableIPv6 - disable IPv6 for the Envoy instance + this will force Envoy to use IPv4 for upstream hosts (mostly for the OAuth2 token endpoint) + type: boolean nodeName: description: |- NodeName - identifies the Envoy cluster within the current namespace diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index e48da28..2fde352 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -23,7 +23,6 @@ resources: - ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus - # [METRICS] Expose the controller manager metrics service. - metrics_service.yaml # [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy. # Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics. @@ -44,19 +43,31 @@ patches: # crd/kustomization.yaml - path: manager_webhook_patch.yaml -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. -# Uncomment the following replacements to add the cert-manager CA injection annotations replacements: - - source: # Uncomment the following block if you have any webhook + - source: + kind: Service + version: v1 + name: control-plane + fieldPath: .metadata.name + targets: + - select: + kind: Deployment + group: apps + version: v1 + name: control-plane + fieldPaths: + - .spec.template.spec.containers.*.env.[name=CONTROL_PLANE_SERVICE_NAME].value + - source: kind: Service version: v1 name: webhook-service - fieldPath: .metadata.name # Name of the service + fieldPath: .metadata.name targets: - select: kind: Certificate group: cert-manager.io version: v1 + name: serving-cert fieldPaths: - .spec.dnsNames.0 - .spec.dnsNames.1 @@ -74,6 +85,7 @@ replacements: kind: Certificate group: cert-manager.io version: v1 + name: serving-cert fieldPaths: - .spec.dnsNames.0 - .spec.dnsNames.1 diff --git a/config/dev/.gitignore b/config/dev/.gitignore new file mode 100644 index 0000000..a795acf --- /dev/null +++ b/config/dev/.gitignore @@ -0,0 +1 @@ +studio-credentials-secret.yaml \ No newline at end of file diff --git a/config/dev/apigateway.yaml b/config/dev/apigateway.yaml index ddd8a64..eda5e3c 100644 --- a/config/dev/apigateway.yaml +++ b/config/dev/apigateway.yaml @@ -1,3 +1,4 @@ +--- apiVersion: supabase.k8s.icb4dc0.de/v1alpha1 kind: APIGateway metadata: @@ -5,8 +6,44 @@ metadata: app.kubernetes.io/name: supabase-operator app.kubernetes.io/managed-by: kustomize name: gateway-sample + namespace: supabase-demo spec: apiEndpoint: jwks: name: core-sample-jwt key: jwks.json + dashboardEndpoint: + auth: + oauth2: + tokenEndpoint: "https://login.microsoftonline.com/f4e80111-1571-477a-b56d-c5fe517676b7/oauth2/token" + authorizationEndpoint: "https://login.microsoftonline.com/f4e80111-1571-477a-b56d-c5fe517676b7/oauth2/authorize" + clientId: 3528016b-f6e3-49be-8fb3-f9a9a2ab6c3f + scopes: + - openid + - profile + - email + clientSecretRef: + name: studio-sample-oauth2 + key: clientSecret +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: certificate + app.kubernetes.io/instance: dashboard-tls + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: supabase-operator + app.kubernetes.io/part-of: supabase-operator + app.kubernetes.io/managed-by: kustomize + name: dashboard-tls + namespace: supabase-demo +spec: + dnsNames: + - gateway-sample-envoy.supabase-demo.svc + - gateway-sample-envoy.supabase-demo.svc.cluster.local + - localhost:3000 + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: dashboard-tls-cert diff --git a/config/samples/cnpg-cluster.yaml b/config/dev/cnpg-cluster.yaml similarity index 95% rename from config/samples/cnpg-cluster.yaml rename to config/dev/cnpg-cluster.yaml index 74aa01d..cdbd22e 100644 --- a/config/samples/cnpg-cluster.yaml +++ b/config/dev/cnpg-cluster.yaml @@ -3,6 +3,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: pgsodium-config + namespace: supabase-demo data: pgsodium_getkey.sh: | #!/bin/bash @@ -18,6 +19,7 @@ apiVersion: v1 kind: Secret metadata: name: pgsodium-key + namespace: supabase-demo data: # Generate a 32-byte key # head -c 32 /dev/urandom | od -A n -t x1 | tr -d ' \n' | base64 @@ -27,6 +29,7 @@ apiVersion: v1 kind: Secret metadata: name: supabase-admin-credentials + namespace: supabase-demo labels: cnpg.io/reload: "true" type: kubernetes.io/basic-auth @@ -38,6 +41,7 @@ apiVersion: postgresql.cnpg.io/v1 kind: Cluster metadata: name: cluster-example + namespace: supabase-demo spec: instances: 1 imageName: ghcr.io/supabase/postgres:15.8.1.021 diff --git a/config/dev/core.yaml b/config/dev/core.yaml index f1484a4..6dfedfd 100644 --- a/config/dev/core.yaml +++ b/config/dev/core.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: Secret metadata: name: supabase-demo-credentials + namespace: supabase-demo stringData: url: postgresql://supabase_admin:1n1t-R00t!@cluster-example-rw.supabase-demo:5432/app --- @@ -12,6 +13,7 @@ metadata: app.kubernetes.io/name: supabase-operator app.kubernetes.io/managed-by: kustomize name: core-sample + namespace: supabase-demo spec: # public URL of the Supabase instance (API) # normally the Ingress/HTTPRoute endpoint diff --git a/config/dev/dashboard.yaml b/config/dev/dashboard.yaml index 48f907b..d8789bb 100644 --- a/config/dev/dashboard.yaml +++ b/config/dev/dashboard.yaml @@ -6,6 +6,7 @@ metadata: app.kubernetes.io/name: supabase-operator app.kubernetes.io/managed-by: kustomize name: dashboard-sample + namespace: supabase-demo spec: db: host: cluster-example-rw.supabase-demo.svc diff --git a/config/dev/kustomization.yaml b/config/dev/kustomization.yaml index b30cc8a..2526b76 100644 --- a/config/dev/kustomization.yaml +++ b/config/dev/kustomization.yaml @@ -4,8 +4,11 @@ kind: Kustomization resources: - https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml - https://github.com/cloudnative-pg/cloudnative-pg/releases/download/v1.25.0/cnpg-1.25.0.yaml + - namespace.yaml + - cnpg-cluster.yaml - minio.yaml - ../default + - studio-credentials-secret.yaml - core.yaml - apigateway.yaml - dashboard.yaml diff --git a/config/dev/minio-operator.yaml b/config/dev/minio-operator.yaml deleted file mode 100644 index d8ca5c5..0000000 --- a/config/dev/minio-operator.yaml +++ /dev/null @@ -1,3 +0,0 @@ -- op: replace - path: /spec/replicas - value: 1 diff --git a/config/dev/minio.yaml b/config/dev/minio.yaml index a417580..184f8c3 100644 --- a/config/dev/minio.yaml +++ b/config/dev/minio.yaml @@ -1,4 +1,4 @@ -# Deploys a new Namespace for the MinIO Pod +--- apiVersion: v1 kind: Namespace metadata: diff --git a/config/dev/namespace.yaml b/config/dev/namespace.yaml new file mode 100644 index 0000000..c8d3ce3 --- /dev/null +++ b/config/dev/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: supabase-demo diff --git a/config/dev/storage.yaml b/config/dev/storage.yaml index 32a1f95..beae391 100644 --- a/config/dev/storage.yaml +++ b/config/dev/storage.yaml @@ -3,6 +3,7 @@ apiVersion: v1 kind: Secret metadata: name: storage-s3-credentials + namespace: supabase-demo stringData: accessKeyId: FPxTAFL7NaubjPgIGBo3 secretAccessKey: 7F437pPe84QcoocD3MWdAIVBU3oXonhVHxK645tm @@ -14,6 +15,7 @@ metadata: app.kubernetes.io/name: supabase-operator app.kubernetes.io/managed-by: kustomize name: storage-sample + namespace: supabase-demo spec: api: s3Backend: diff --git a/config/dev/studio-credentials-secret.yaml b/config/dev/studio-credentials-secret.yaml new file mode 100644 index 0000000..bb78898 --- /dev/null +++ b/config/dev/studio-credentials-secret.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: studio-sample-oauth2 + namespace: supabase-demo +stringData: + clientSecret: "G9r8Q~o4LJRlTQwPpdCBaZLsWdhUxM_02Y_XBcEr" \ No newline at end of file diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 81414b4..09b3ff4 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -20,26 +20,6 @@ spec: labels: control-plane: controller-manager spec: - # TODO(user): Uncomment the following code to configure the nodeAffinity expression - # according to the platforms which are supported by your solution. - # It is considered best practice to support multiple architectures. You can - # build your manager image using the makefile target docker-buildx. - # affinity: - # nodeAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: kubernetes.io/arch - # operator: In - # values: - # - amd64 - # - arm64 - # - ppc64le - # - s390x - # - key: kubernetes.io/os - # operator: In - # values: - # - linux securityContext: runAsNonRoot: true seccompProfile: @@ -56,6 +36,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: TLS_CA_CERT + value: /etc/supabase/operator/certs/tls.crt + - name: TLS_CA_KEY + value: /etc/supabase/operator/certs/tls.key securityContext: allowPrivilegeEscalation: false capabilities: @@ -73,8 +57,6 @@ spec: port: 8081 initialDelaySeconds: 5 periodSeconds: 10 - # TODO(user): Configure the resources accordingly based on the project requirements. - # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ resources: limits: cpu: 150m @@ -82,5 +64,12 @@ spec: requests: cpu: 10m memory: 64Mi + volumeMounts: + - name: tls-certs + mountPath: /etc/supabase/operator/certs + volumes: + - name: tls-certs + secret: + secretName: control-plane-ca-cert-tls serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 diff --git a/config/rbac/control-plane-role.yaml b/config/rbac/control-plane-role.yaml index 9bccf5c..8dc1eb5 100644 --- a/config/rbac/control-plane-role.yaml +++ b/config/rbac/control-plane-role.yaml @@ -4,6 +4,17 @@ kind: ClusterRole metadata: name: control-plane-role rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - update + - get + - list + - watch + - apiGroups: - supabase.k8s.icb4dc0.de resources: @@ -12,6 +23,8 @@ rules: - get - list - watch + - update + - apiGroups: - supabase.k8s.icb4dc0.de resources: diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 74dba7c..48d6520 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -4,7 +4,6 @@ namespace: supabase-demo resources: - namespace.yaml - - cnpg-cluster.yaml - supabase_v1alpha1_core.yaml - supabase_v1alpha1_apigateway.yaml - supabase_v1alpha1_dashboard.yaml diff --git a/dev/cluster.yaml b/dev/cluster.yaml index 30d89f7..30355bb 100644 --- a/dev/cluster.yaml +++ b/dev/cluster.yaml @@ -7,3 +7,6 @@ apiVersion: ctlptl.dev/v1alpha1 kind: Cluster product: kind registry: ctlptl-registry +kindV1Alpha4Cluster: + networking: + ipFamily: dual diff --git a/go.mod b/go.mod index f5360b5..94c7bd4 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,8 @@ go 1.23.5 require ( github.com/alecthomas/kong v1.6.0 github.com/envoyproxy/go-control-plane v0.13.1 + github.com/go-logr/logr v1.4.2 + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.2.0 github.com/jackc/pgx/v5 v5.7.1 github.com/lestrrat-go/jwx/v2 v2.1.3 github.com/magefile/mage v1.15.0 @@ -12,8 +14,8 @@ require ( github.com/onsi/gomega v1.35.1 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 - google.golang.org/grpc v1.65.0 - google.golang.org/protobuf v1.36.3 + google.golang.org/grpc v1.70.0 + google.golang.org/protobuf v1.36.4 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.32.1 k8s.io/apimachinery v0.32.1 @@ -22,7 +24,7 @@ require ( ) require ( - cel.dev/expr v0.18.0 // indirect + cel.dev/expr v0.19.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -30,7 +32,7 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b // indirect + github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect @@ -39,7 +41,6 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect @@ -84,12 +85,12 @@ require ( github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect - go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel v1.32.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.28.0 // indirect - go.opentelemetry.io/otel/sdk v1.28.0 // indirect - go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.32.0 // indirect + go.opentelemetry.io/otel/sdk v1.32.0 // indirect + go.opentelemetry.io/otel/trace v1.32.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.32.0 // indirect @@ -102,8 +103,8 @@ require ( golang.org/x/time v0.9.0 // indirect golang.org/x/tools v0.26.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect k8s.io/apiextensions-apiserver v0.32.1 // indirect diff --git a/go.sum b/go.sum index d7e3669..0afea64 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -cel.dev/expr v0.18.0 h1:CJ6drgk+Hf96lkLikr4rFf19WrU0BOWEihyZnI2TAzo= -cel.dev/expr v0.18.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= +cel.dev/expr v0.19.0 h1:lXuo+nDhpyJSpWxpPVi5cPUwzKb+dsdOiw6IreM5yt0= +cel.dev/expr v0.19.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/kong v1.6.0 h1:mwOzbdMR7uv2vul9J0FU3GYxE7ls/iX1ieMg5WIM6gE= @@ -20,8 +20,8 @@ github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMr github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw= -github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI= +github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -82,6 +82,8 @@ github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgY github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.2.0 h1:kQ0NI7W1B3HwiN5gAYtY+XFItDPbLBwYRxAqbFTyDes= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.2.0/go.mod h1:zrT2dxOAjNFPRGjTUe2Xmb4q4YdUwVvQFV6xiCSf+z0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -180,18 +182,20 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= -go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= -go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= +go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= -go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= -go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= -go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= -go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= +go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= +go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= +go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= +go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= +go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= +go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= +go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -247,14 +251,14 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw= -google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 h1:2035KHhUv+EpyB+hWgJnaWKJOdX1E95w2S8Rr4uWKTs= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= -google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= -google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU= -google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a h1:OAiGFfOiA0v9MRYsSidp3ubZaBnteRUyn3xB2ZQ5G/E= +google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a/go.mod h1:jehYqy3+AhJU9ve55aNOaSml7wUXjF9x6z2LcCfpAhY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287 h1:J1H9f+LEdWAfHcez/4cvaVBox7cOYT+IU6rgqj5x++8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287/go.mod h1:8BS3B93F/U1juMFq9+EDk+qOT5CO1R9IzXxG3PTqiRk= +google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= +google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= +google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= +google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/certs/format.go b/internal/certs/format.go new file mode 100644 index 0000000..890e1b7 --- /dev/null +++ b/internal/certs/format.go @@ -0,0 +1,60 @@ +/* +Copyright 2025 Peter Kurfer. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certs + +import ( + "crypto/tls" + "encoding/pem" + "fmt" +) + +const ( + certificateBlockType = "CERTIFICATE" + privateKeyBlockType = "PRIVATE KEY" +) + +func EncodePublicKeyToPEM(derBytes []byte) []byte { + return pem.EncodeToMemory(&pem.Block{Type: certificateBlockType, Bytes: derBytes}) +} + +func EncodePrivateKeyToPEM(privateKeyBytes []byte) []byte { + return pem.EncodeToMemory(&pem.Block{Type: privateKeyBlockType, Bytes: privateKeyBytes}) +} + +func newCertResult(derBytes, privateKeyBytes []byte) (CertResult, error) { + pemEncodedPublicKey := EncodePublicKeyToPEM(derBytes) + pemEncodedPrivateKey := EncodePrivateKeyToPEM(privateKeyBytes) + + cert, err := tls.X509KeyPair(pemEncodedPublicKey, pemEncodedPrivateKey) + if err != nil { + return CertResult{}, fmt.Errorf("failed to create TLS cert based on x509 key pair: %w", err) + } + + result := CertResult{ + ServerCert: cert, + PublicKey: pemEncodedPublicKey, + PrivateKey: pemEncodedPrivateKey, + } + + return result, nil +} + +type CertResult struct { + ServerCert tls.Certificate + PublicKey []byte + PrivateKey []byte +} diff --git a/internal/certs/generate.go b/internal/certs/generate.go new file mode 100644 index 0000000..71156f7 --- /dev/null +++ b/internal/certs/generate.go @@ -0,0 +1,122 @@ +/* +Copyright 2025 Peter Kurfer. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certs + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "math/big" + "time" +) + +func ServerCert( + commonName string, + dnsNames []string, + ca tls.Certificate, +) (result CertResult, err error) { + serial, err := generateSerialNumber() + if err != nil { + return result, err + } + + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return result, fmt.Errorf("failed to generate private key: %w", err) + } + + certTemplate := x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: commonName, + }, + DNSNames: dnsNames, + NotBefore: time.Now().Add(-1 * time.Hour), + NotAfter: time.Now().AddDate(0, 1, 0), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageDataEncipherment | x509.KeyUsageKeyEncipherment | x509.KeyUsageContentCommitment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + + var caCrt *x509.Certificate + if caCrt, err = x509.ParseCertificate(ca.Certificate[0]); err != nil { + return result, fmt.Errorf("failed parse CA cert: %w", err) + } + + var derBytes []byte + if derBytes, err = x509.CreateCertificate(rand.Reader, &certTemplate, caCrt, &privateKey.PublicKey, ca.PrivateKey); err != nil { + return result, fmt.Errorf("failed to create signed certificate: %w", err) + } + + var privateKeyBytes []byte + if privateKeyBytes, err = x509.MarshalPKCS8PrivateKey(privateKey); err != nil { + return result, fmt.Errorf("failed to marshal private key: %w", err) + } + + return newCertResult(derBytes, privateKeyBytes) +} + +func ClientCert( + commonName string, + ca tls.Certificate, +) (result CertResult, err error) { + serial, err := generateSerialNumber() + if err != nil { + return result, err + } + + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return result, fmt.Errorf("failed to generate private key: %w", err) + } + + certTemplate := x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: commonName, + }, + NotBefore: time.Now().Add(-1 * time.Hour), + NotAfter: time.Now().AddDate(0, 0, 7), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageDataEncipherment | x509.KeyUsageKeyEncipherment | x509.KeyUsageContentCommitment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + + var caCrt *x509.Certificate + if caCrt, err = x509.ParseCertificate(ca.Certificate[0]); err != nil { + return result, fmt.Errorf("failed parse CA cert: %w", err) + } + + var derBytes []byte + if derBytes, err = x509.CreateCertificate(rand.Reader, &certTemplate, caCrt, &privateKey.PublicKey, ca.PrivateKey); err != nil { + return result, fmt.Errorf("failed to create signed certificate: %w", err) + } + + var privateKeyBytes []byte + if privateKeyBytes, err = x509.MarshalPKCS8PrivateKey(privateKey); err != nil { + return result, fmt.Errorf("failed to marshal private key: %w", err) + } + + return newCertResult(derBytes, privateKeyBytes) +} + +func generateSerialNumber() (*big.Int, error) { + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + return rand.Int(rand.Reader, serialNumberLimit) +} diff --git a/internal/controller/apigateway_controller.go b/internal/controller/apigateway_controller.go index d460d64..14c7911 100644 --- a/internal/controller/apigateway_controller.go +++ b/internal/controller/apigateway_controller.go @@ -19,10 +19,13 @@ package controller import ( "bytes" "context" + "crypto/rand" + "crypto/tls" "embed" "encoding/hex" "errors" "fmt" + "strings" "text/template" "time" @@ -41,6 +44,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" supabasev1alpha1 "code.icb4dc0.de/prskr/supabase-operator/api/v1alpha1" + "code.icb4dc0.de/prskr/supabase-operator/internal/certs" "code.icb4dc0.de/prskr/supabase-operator/internal/meta" "code.icb4dc0.de/prskr/supabase-operator/internal/supabase" ) @@ -64,6 +68,7 @@ func init() { type APIGatewayReconciler struct { client.Client Scheme *runtime.Scheme + CACert tls.Certificate } // Reconcile is part of the main kubernetes reconciliation loop which aims to @@ -85,6 +90,14 @@ func (r *APIGatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } + if err := r.reconcileHmacSecret(ctx, &gateway); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to ensure HMAC token secret: %w", err) + } + + if err := r.reconcileClientCertSecret(ctx, &gateway); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to ensure client certificate: %w", err) + } + if jwksHash, err = r.reconcileJwksSecret(ctx, &gateway); err != nil { return ctrl.Result{}, err } @@ -105,7 +118,8 @@ func (r *APIGatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - return ctrl.Result{}, nil + // requeue after 15 minutes to watch for expiring certificates + return ctrl.Result{RequeueAfter: 15 * time.Minute}, nil } // SetupWithManager sets up the controller with the Manager. @@ -144,6 +158,8 @@ func (r *APIGatewayReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Ma return ctrl.NewControllerManagedBy(mgr). For(&supabasev1alpha1.APIGateway{}). Named("apigateway"). + // watch for the HMAC & client cert secrets + Owns(new(corev1.Secret)). Owns(new(corev1.ConfigMap)). Owns(new(appsv1.Deployment)). Owns(new(corev1.Service)). @@ -219,6 +235,115 @@ func (r *APIGatewayReconciler) reconcileJwksSecret( return hex.EncodeToString(HashBytes(jwksRaw)), nil } +func (r *APIGatewayReconciler) reconcileHmacSecret( + ctx context.Context, + gateway *supabasev1alpha1.APIGateway, +) error { + const hmacSecretLength = 32 + + serviceCfg := supabase.ServiceConfig.Envoy + + hmacSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceCfg.HmacSecretName(gateway), + Namespace: gateway.Namespace, + }, + } + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, hmacSecret, func() error { + if hmacSecret.Data == nil { + hmacSecret.Data = make(map[string][]byte) + } + + if _, ok := hmacSecret.Data[serviceCfg.Defaults.HmacSecretKey]; !ok { + secret := make([]byte, hmacSecretLength) + if n, err := rand.Read(secret); err != nil { + return fmt.Errorf("failed to generate HMAC token secret: %w", err) + } else if n != hmacSecretLength { + return fmt.Errorf("failed to generate HMAC token secret: not enough bytes generated") + } + + hmacSecret.Data[serviceCfg.Defaults.HmacSecretKey] = secret + } + + if err := controllerutil.SetControllerReference(gateway, hmacSecret, r.Scheme); err != nil { + return fmt.Errorf("failed to set controller reference: %w", err) + } + + return nil + }) + + return err +} + +func (r *APIGatewayReconciler) reconcileClientCertSecret( + ctx context.Context, + gateway *supabasev1alpha1.APIGateway, +) error { + var ( + logger = log.FromContext(ctx) + clientCertSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: supabase.ServiceConfig.Envoy.ControlPlaneClientCertSecretName(gateway), + Namespace: gateway.Namespace, + }, + } + ) + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, clientCertSecret, func() (err error) { + clientCertSecret.Type = corev1.SecretTypeTLS + if clientCertSecret.Data == nil { + clientCertSecret.Data = make(map[string][]byte, 3) + } + + caCertBytes := certs.EncodePublicKeyToPEM(r.CACert.Certificate[0]) + clientCertSecret.Data["ca.crt"] = caCertBytes + + var ( + cert = clientCertSecret.Data[corev1.TLSCertKey] + privateKey = clientCertSecret.Data[corev1.TLSPrivateKeyKey] + clientCert tls.Certificate + ) + + var requireRenewal bool + if cert != nil && privateKey != nil { + if clientCert, err = tls.X509KeyPair(cert, privateKey); err != nil { + return fmt.Errorf("failed to parse server certificate: %w", err) + } + + renewGracePeriod := time.Duration(float64(clientCert.Leaf.NotAfter.Sub(clientCert.Leaf.NotBefore)) * 0.1) + if clientCert.Leaf.NotAfter.Before(time.Now().Add(-renewGracePeriod)) { + logger.Info("Envoy control-plane client certificate requires renewal", + "not_after", clientCert.Leaf.NotAfter, + "renew_grace_period", renewGracePeriod, + ) + requireRenewal = true + } + } else { + logger.Info("Client cert is not set creating a new one") + requireRenewal = true + } + + if requireRenewal { + if certResult, err := certs.ClientCert(strings.Join([]string{gateway.Name, gateway.Namespace}, ":"), r.CACert); err != nil { + return fmt.Errorf("failed to generate server certificate: %w", err) + } else { + clientCert = certResult.ServerCert + clientCertSecret.Data[corev1.TLSCertKey] = certResult.PublicKey + clientCertSecret.Data[corev1.TLSPrivateKeyKey] = certResult.PrivateKey + } + } + + if err := controllerutil.SetControllerReference(gateway, clientCertSecret, r.Scheme); err != nil { + return fmt.Errorf("failed to set controller reference: %w", err) + } + + return nil + }) + + return err +} + func (r *APIGatewayReconciler) reconcileEnvoyConfig( ctx context.Context, gateway *supabasev1alpha1.APIGateway, @@ -291,6 +416,12 @@ func (r *APIGatewayReconciler) reconileEnvoyDeployment( gateway *supabasev1alpha1.APIGateway, configHash, jwksHash string, ) error { + const ( + configVolumeName = "config" + controlPlaneTlsVolumeName = "cp-tls" + dashboardTlsVolumeName = "dashboard-tls" + apiTlsVolumeName = "api-tls" + ) envoyDeployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: supabase.ServiceConfig.Envoy.ObjectName(gateway), @@ -317,6 +448,131 @@ func (r *APIGatewayReconciler) reconileEnvoyDeployment( envoyDeployment.Spec.Replicas = envoySpec.WorkloadTemplate.ReplicaCount() + configVolumeProjectionSources := []corev1.VolumeProjection{ + { + ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: supabase.ServiceConfig.Envoy.ObjectName(gateway), + }, + Items: []corev1.KeyToPath{ + { + Key: "config.yaml", + Path: "config.yaml", + }, + }, + }, + }, + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: gateway.Spec.ApiEndpoint.JWKSSelector.Name, + }, + Items: []corev1.KeyToPath{{ + Key: gateway.Spec.ApiEndpoint.JWKSSelector.Key, + Path: "jwks.json", + }}, + }, + }, + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: serviceCfg.ControlPlaneClientCertSecretName(gateway), + }, + Items: []corev1.KeyToPath{ + { + Key: "ca.crt", + Path: "certs/cp/ca.crt", + }, + { + Key: "tls.crt", + Path: "certs/cp/tls.crt", + }, + { + Key: "tls.key", + Path: "certs/cp/tls.key", + }, + }, + }, + }, + } + + if oauth2Spec := gateway.Spec.DashboardEndpoint.OAuth2(); oauth2Spec != nil { + configVolumeProjectionSources = append(configVolumeProjectionSources, corev1.VolumeProjection{ + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: oauth2Spec.ClientSecretRef.Name, + }, + Items: []corev1.KeyToPath{{ + Key: oauth2Spec.ClientSecretRef.Key, + Path: serviceCfg.Defaults.OAuth2ClientSecretKey, + }}, + }, + }) + } + + volumeMounts := []corev1.VolumeMount{ + { + Name: configVolumeName, + ReadOnly: true, + MountPath: "/etc/envoy", + }, + } + + volumes := []corev1.Volume{ + { + Name: configVolumeName, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: configVolumeProjectionSources, + }, + }, + }, + { + Name: controlPlaneTlsVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceCfg.ControlPlaneClientCertSecretName(gateway), + }, + }, + }, + } + + if tlsSpec := gateway.Spec.ApiEndpoint.TLSSpec(); tlsSpec != nil { + volumes = append(volumes, corev1.Volume{ + Name: apiTlsVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tlsSpec.Cert.SecretName, + }, + }, + }) + + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: dashboardTlsVolumeName, + ReadOnly: true, + MountPath: "/etc/envoy/certs/api", + SubPath: "certs/api", + }) + } + + if tlsSpec := gateway.Spec.DashboardEndpoint.TLSSpec(); tlsSpec != nil { + volumes = append(volumes, corev1.Volume{ + Name: dashboardTlsVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tlsSpec.Cert.SecretName, + }, + }, + }) + + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: dashboardTlsVolumeName, + ReadOnly: true, + MountPath: "/etc/envoy/certs/dashboard", + SubPath: "certs/dashboard", + }) + } + envoyDeployment.Spec.Template = corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ @@ -333,16 +589,21 @@ func (r *APIGatewayReconciler) reconileEnvoyDeployment( Name: "envoy-proxy", Image: envoySpec.WorkloadTemplate.Image(supabase.Images.Envoy.String()), ImagePullPolicy: envoySpec.WorkloadTemplate.ImagePullPolicy(), - Args: []string{"-c /etc/envoy/config.yaml"}, + Args: []string{"-c /etc/envoy/config.yaml"}, // , "--component-log-level", "upstream:debug,connection:debug" Ports: []corev1.ContainerPort{ { - Name: "http", - ContainerPort: 8000, + Name: serviceCfg.Defaults.StudioPortName, + ContainerPort: serviceCfg.Defaults.StudioPort, + Protocol: corev1.ProtocolTCP, + }, + { + Name: serviceCfg.Defaults.ApiPortName, + ContainerPort: serviceCfg.Defaults.ApiPort, Protocol: corev1.ProtocolTCP, }, { Name: "admin", - ContainerPort: 19000, + ContainerPort: serviceCfg.Defaults.AdminPort, Protocol: corev1.ProtocolTCP, }, }, @@ -371,49 +632,11 @@ func (r *APIGatewayReconciler) reconileEnvoyDeployment( }, SecurityContext: envoySpec.WorkloadTemplate.ContainerSecurityContext(serviceCfg.Defaults.UID, serviceCfg.Defaults.GID), Resources: envoySpec.WorkloadTemplate.Resources(), - VolumeMounts: envoySpec.WorkloadTemplate.AdditionalVolumeMounts( - corev1.VolumeMount{ - Name: "config", - ReadOnly: true, - MountPath: "/etc/envoy", - }, - ), + VolumeMounts: envoySpec.WorkloadTemplate.AdditionalVolumeMounts(volumeMounts...), }, }, SecurityContext: envoySpec.WorkloadTemplate.PodSecurityContext(), - Volumes: []corev1.Volume{ - { - Name: "config", - VolumeSource: corev1.VolumeSource{ - Projected: &corev1.ProjectedVolumeSource{ - Sources: []corev1.VolumeProjection{ - { - ConfigMap: &corev1.ConfigMapProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: supabase.ServiceConfig.Envoy.ObjectName(gateway), - }, - Items: []corev1.KeyToPath{{ - Key: "config.yaml", - Path: "config.yaml", - }}, - }, - }, - { - Secret: &corev1.SecretProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: gateway.Spec.ApiEndpoint.JWKSSelector.Name, - }, - Items: []corev1.KeyToPath{{ - Key: gateway.Spec.ApiEndpoint.JWKSSelector.Key, - Path: "jwks.json", - }}, - }, - }, - }, - }, - }, - }, - }, + Volumes: volumes, }, } @@ -431,12 +654,15 @@ func (r *APIGatewayReconciler) reconcileEnvoyService( ctx context.Context, gateway *supabasev1alpha1.APIGateway, ) error { - envoyService := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: supabase.ServiceConfig.Envoy.ObjectName(gateway), - Namespace: gateway.Namespace, - }, - } + var ( + serviceCfg = supabase.ServiceConfig.Envoy + envoyService = &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: supabase.ServiceConfig.Envoy.ObjectName(gateway), + Namespace: gateway.Namespace, + }, + } + ) _, err := controllerutil.CreateOrUpdate(ctx, r.Client, envoyService, func() error { envoyService.Labels = MergeLabels(objectLabels(gateway, "envoy", "api-gateway", supabase.Images.Envoy.Tag), gateway.Labels) @@ -445,11 +671,18 @@ func (r *APIGatewayReconciler) reconcileEnvoyService( Selector: selectorLabels(gateway, "envoy"), Ports: []corev1.ServicePort{ { - Name: "rest", + Name: serviceCfg.Defaults.StudioPortName, Protocol: corev1.ProtocolTCP, AppProtocol: ptrOf("http"), - Port: 8000, - TargetPort: intstr.IntOrString{IntVal: 8000}, + Port: serviceCfg.Defaults.StudioPort, + TargetPort: intstr.IntOrString{IntVal: serviceCfg.Defaults.StudioPort}, + }, + { + Name: serviceCfg.Defaults.ApiPortName, + Protocol: corev1.ProtocolTCP, + AppProtocol: ptrOf("http"), + Port: serviceCfg.Defaults.ApiPort, + TargetPort: intstr.IntOrString{IntVal: serviceCfg.Defaults.ApiPort}, }, }, } diff --git a/internal/controller/core_gotrue_controller.go b/internal/controller/core_gotrue_controller.go index fdfa82f..95b358b 100644 --- a/internal/controller/core_gotrue_controller.go +++ b/internal/controller/core_gotrue_controller.go @@ -192,7 +192,7 @@ func (r *CoreAuthReconciler) reconcileAuthDeployment( SuccessThreshold: 2, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: "/health", + Path: svcCfg.LivenessProbePath, Port: intstr.IntOrString{IntVal: svcCfg.Defaults.APIPort}, }, }, @@ -203,7 +203,7 @@ func (r *CoreAuthReconciler) reconcileAuthDeployment( TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: "/health", + Path: svcCfg.LivenessProbePath, Port: intstr.IntOrString{IntVal: svcCfg.Defaults.APIPort}, }, }, diff --git a/internal/controller/core_postgrest_controller.go b/internal/controller/core_postgrest_controller.go index bee341c..e9e3811 100644 --- a/internal/controller/core_postgrest_controller.go +++ b/internal/controller/core_postgrest_controller.go @@ -195,7 +195,7 @@ func (r *CorePostgrestReconiler) reconilePostgrestDeployment( SuccessThreshold: 2, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: "/ready", + Path: serviceCfg.LivenessProbePath, Port: intstr.IntOrString{IntVal: serviceCfg.Defaults.AdminPort}, }, }, diff --git a/internal/controller/dashboard_pg-meta_controller.go b/internal/controller/dashboard_pg-meta_controller.go index 9378bb1..6ca7540 100644 --- a/internal/controller/dashboard_pg-meta_controller.go +++ b/internal/controller/dashboard_pg-meta_controller.go @@ -147,7 +147,7 @@ func (r *DashboardPGMetaReconciler) reconcilePGMetaDeployment( SuccessThreshold: 2, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: "/health", + Path: serviceCfg.LivenessProbePath, Port: intstr.IntOrString{IntVal: serviceCfg.Defaults.APIPort}, }, }, @@ -158,7 +158,7 @@ func (r *DashboardPGMetaReconciler) reconcilePGMetaDeployment( TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: "/health", + Path: serviceCfg.LivenessProbePath, Port: intstr.IntOrString{IntVal: serviceCfg.Defaults.APIPort}, }, }, diff --git a/internal/controller/dashboard_studio_controller.go b/internal/controller/dashboard_studio_controller.go index 44df127..453c1f2 100644 --- a/internal/controller/dashboard_studio_controller.go +++ b/internal/controller/dashboard_studio_controller.go @@ -161,7 +161,7 @@ func (r *DashboardStudioReconciler) reconcileStudioDeployment( SuccessThreshold: 2, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: "/api/profile", + Path: serviceCfg.LivenessProbePath, Port: intstr.IntOrString{IntVal: serviceCfg.Defaults.APIPort}, }, }, @@ -172,7 +172,7 @@ func (r *DashboardStudioReconciler) reconcileStudioDeployment( TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: "/api/profile", + Path: serviceCfg.LivenessProbePath, Port: intstr.IntOrString{IntVal: serviceCfg.Defaults.APIPort}, }, }, diff --git a/internal/controller/storage_api_controller.go b/internal/controller/storage_api_controller.go index 587b00d..c46155b 100644 --- a/internal/controller/storage_api_controller.go +++ b/internal/controller/storage_api_controller.go @@ -225,7 +225,7 @@ func (r *StorageApiReconciler) reconcileStorageApiDeployment( SuccessThreshold: 2, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: "/status", + Path: serviceCfg.LivenessProbePath, Port: intstr.IntOrString{IntVal: serviceCfg.Defaults.ApiPort}, }, }, @@ -236,7 +236,7 @@ func (r *StorageApiReconciler) reconcileStorageApiDeployment( TimeoutSeconds: 3, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: "/status", + Path: serviceCfg.LivenessProbePath, Port: intstr.IntOrString{IntVal: serviceCfg.Defaults.ApiPort}, }, }, diff --git a/internal/controller/storage_imgproxy_controller.go b/internal/controller/storage_imgproxy_controller.go index fca0b30..e197574 100644 --- a/internal/controller/storage_imgproxy_controller.go +++ b/internal/controller/storage_imgproxy_controller.go @@ -184,7 +184,7 @@ func (r *StorageImgProxyReconciler) reconcileImgProxyService( var ( serviceCfg = supabase.ServiceConfig.ImgProxy imgProxyService = &corev1.Service{ - ObjectMeta: supabase.ServiceConfig.Storage.ObjectMeta(storage), + ObjectMeta: supabase.ServiceConfig.ImgProxy.ObjectMeta(storage), } ) diff --git a/internal/controller/templates/envoy_control_plane_config.yaml.tmpl b/internal/controller/templates/envoy_control_plane_config.yaml.tmpl index 596dc8d..b1d2581 100644 --- a/internal/controller/templates/envoy_control_plane_config.yaml.tmpl +++ b/internal/controller/templates/envoy_control_plane_config.yaml.tmpl @@ -15,13 +15,9 @@ dynamic_resources: static_resources: clusters: - - type: STRICT_DNS - typed_extension_protocol_options: - envoy.extensions.upstreams.http.v3.HttpProtocolOptions: - "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions - explicit_http_config: - http2_protocol_options: {} - name: {{ .ControlPlane.Name }} + - name: {{ .ControlPlane.Name }} + type: STRICT_DNS + connect_timeout: 1s load_assignment: cluster_name: {{ .ControlPlane.Name }} endpoints: @@ -31,9 +27,38 @@ static_resources: socket_address: address: {{ .ControlPlane.Host }} port_value: {{ .ControlPlane.Port }} + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: "envoy.transport_sockets.tls" + typed_config: + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext" + sni: {{ .ControlPlane.Host }} + common_tls_context: + tls_certificates: + - certificate_chain: + filename: /etc/envoy/certs/cp/tls.crt + private_key: + filename: /etc/envoy/certs/cp/tls.key + validation_context: + trusted_ca: + filename: /etc/envoy/certs/cp/ca.crt + admin: address: socket_address: address: 0.0.0.0 port_value: 19000 + +application_log_config: + log_format: + json_format: + type: "app" + name: "%n" + timestamp: "%Y-%m-%dT%T.%F" + level: "%l" + message: "%j" diff --git a/internal/controlplane/apigateway_controller.go b/internal/controlplane/apigateway_controller.go index bbf779c..5021f3c 100644 --- a/internal/controlplane/apigateway_controller.go +++ b/internal/controlplane/apigateway_controller.go @@ -19,14 +19,14 @@ package controlplane import ( "bytes" "context" - "encoding/json" "fmt" - "hash/fnv" "strconv" "strings" + "sync/atomic" "time" cachev3 "github.com/envoyproxy/go-control-plane/pkg/cache/v3" + corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -46,6 +46,7 @@ import ( // APIGatewayReconciler reconciles a APIGateway object type APIGatewayReconciler struct { + initialReconciliation atomic.Bool client.Client Scheme *runtime.Scheme Cache cachev3.SnapshotCache @@ -63,7 +64,7 @@ func (r *APIGatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) endpointSliceList discoveryv1.EndpointSliceList ) - logger.Info("Reconciling APIGateway") + logger.Info("Reconciling Envoy control-plane config") if err := r.Get(ctx, req.NamespacedName, &gateway); client.IgnoreNotFound(err) != nil { logger.Error(err, "unable to fetch Gateway") @@ -79,25 +80,31 @@ func (r *APIGatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - services := EnvoyServices{ServiceLabelKey: gateway.Spec.ComponentTypeLabel} + services := EnvoyServices{ + ServiceLabelKey: gateway.Spec.ComponentTypeLabel, + Gateway: &gateway, + Client: r.Client, + } services.UpsertEndpointSlices(endpointSliceList.Items...) - rawServices, err := json.Marshal(services) + instance := fmt.Sprintf("%s:%s", gateway.Spec.Envoy.NodeName, gateway.Namespace) + + logger.Info("Computing Envoy snapshot for current service targets", "version", gateway.Status.Envoy.ConfigVersion) + snapshot, snapshotHash, err := services.snapshot(ctx, instance, gateway.Status.Envoy.ConfigVersion) if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to prepare config hash: %w", err) + return ctrl.Result{}, fmt.Errorf("failed to prepare snapshot: %w", err) } - serviceHash := fnv.New64a().Sum(rawServices) - if bytes.Equal(serviceHash, gateway.Status.Envoy.ResourceHash) { - logger.Info("Resource hash did not change - skipping reconciliation") + if !r.initialReconciliation.CompareAndSwap(false, true) && bytes.Equal(gateway.Status.Envoy.ResourceHash, snapshotHash) { + logger.Info("No changes detected, skipping update") return ctrl.Result{}, nil } logger.Info("Updating service targets") - _, err = controllerutil.CreateOrPatch(ctx, r.Client, &gateway, func() error { + _, err = controllerutil.CreateOrUpdate(ctx, r.Client, &gateway, func() error { gateway.Status.ServiceTargets = services.Targets() gateway.Status.Envoy.ConfigVersion = strconv.FormatInt(time.Now().UTC().UnixMilli(), 10) - gateway.Status.Envoy.ResourceHash = serviceHash + gateway.Status.Envoy.ResourceHash = snapshotHash return nil }) @@ -105,14 +112,6 @@ func (r *APIGatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - instance := fmt.Sprintf("%s:%s", gateway.Spec.Envoy.NodeName, gateway.Namespace) - - logger.Info("Computing Envoy snapshot for current service targets", "version", gateway.Status.Envoy.ConfigVersion) - snapshot, err := services.snapshot(ctx, instance, gateway.Status.Envoy.ConfigVersion) - if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to prepare snapshot: %w", err) - } - logger.Info("Propagating Envoy snapshot", "version", gateway.Status.Envoy.ConfigVersion) if err := r.Cache.SetSnapshot(ctx, instance, snapshot); err != nil { return ctrl.Result{}, fmt.Errorf("failed to propagate snapshot: %w", err) @@ -133,7 +132,8 @@ func (r *APIGatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { } return ctrl.NewControllerManagedBy(mgr). - For(new(supabasev1alpha1.APIGateway)). + For(new(supabasev1alpha1.APIGateway), builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Owns(new(corev1.Secret), builder.WithPredicates(predicate.GenerationChangedPredicate{})). Watches( new(discoveryv1.EndpointSlice), r.endpointSliceEventHandler(), @@ -156,6 +156,7 @@ func (r *APIGatewayReconciler) endpointSliceEventHandler() handler.TypedEventHan return nil } + logger.Info("Triggering APIGateway reconciliation", "obj_name", obj.GetName(), "obj_namespace", obj.GetNamespace()) if err := r.Client.List(ctx, &apiGatewayList, client.InNamespace(endpointSlice.Namespace)); err != nil { logger.Error(err, "failed to list APIGateways to determine reconcile targets") return nil diff --git a/internal/controlplane/endpoints.go b/internal/controlplane/endpoints.go index 355cfb0..e9d3699 100644 --- a/internal/controlplane/endpoints.go +++ b/internal/controlplane/endpoints.go @@ -17,9 +17,7 @@ limitations under the License. package controlplane import ( - "encoding/json" "fmt" - "slices" "strings" "time" @@ -30,27 +28,10 @@ import ( discoveryv1 "k8s.io/api/discovery/v1" ) -var _ json.Marshaler = (*ServiceCluster)(nil) - type ServiceCluster struct { ServiceEndpoints map[string]Endpoints } -// MarshalJSON implements json.Marshaler. -func (c *ServiceCluster) MarshalJSON() ([]byte, error) { - tmp := struct { - Endpoints []string `json:"endpoints"` - }{} - - for _, endpoints := range c.ServiceEndpoints { - tmp.Endpoints = append(tmp.Endpoints, endpoints.Targets...) - } - - slices.Sort(tmp.Endpoints) - - return json.Marshal(tmp) -} - func (c *ServiceCluster) AddOrUpdateEndpoints(eps discoveryv1.EndpointSlice) { if c.ServiceEndpoints == nil { c.ServiceEndpoints = make(map[string]Endpoints) diff --git a/internal/controlplane/filters.go b/internal/controlplane/filters.go index 1c2b1ed..a636275 100644 --- a/internal/controlplane/filters.go +++ b/internal/controlplane/filters.go @@ -22,4 +22,6 @@ const ( FilterNameCORS = "envoy.filters.http.cors" FilterNameHttpRouter = "envoy.filters.http.router" FilterNameHttpConnectionManager = "envoy.filters.network.http_connection_manager" + FilterNameBasicAuth = "envoy.filters.http.basic_auth" + FilterNameOAuth2 = "envoy.filters.http.oauth2" ) diff --git a/internal/controlplane/logging.go b/internal/controlplane/logging.go new file mode 100644 index 0000000..190a1e1 --- /dev/null +++ b/internal/controlplane/logging.go @@ -0,0 +1,62 @@ +/* +Copyright 2025 Peter Kurfer. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controlplane + +import ( + accesslogv3 "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3" + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + streamv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/stream/v3" + "google.golang.org/protobuf/types/known/structpb" +) + +func AccessLog(name string) *accesslogv3.AccessLog { + return &accesslogv3.AccessLog{ + Name: name, + Filter: &accesslogv3.AccessLogFilter{ + FilterSpecifier: &accesslogv3.AccessLogFilter_NotHealthCheckFilter{ + NotHealthCheckFilter: new(accesslogv3.NotHealthCheckFilter), + }, + }, + ConfigType: &accesslogv3.AccessLog_TypedConfig{ + TypedConfig: MustAny(&streamv3.StderrAccessLog{ + AccessLogFormat: &streamv3.StderrAccessLog_LogFormat{ + LogFormat: &corev3.SubstitutionFormatString{ + Format: &corev3.SubstitutionFormatString_JsonFormat{ + JsonFormat: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "type": {Kind: &structpb.Value_StringValue{StringValue: "access_log"}}, + "timestamp": {Kind: &structpb.Value_StringValue{StringValue: "%START_TIME%"}}, + "protocol": {Kind: &structpb.Value_StringValue{StringValue: "%PROTOCOL%"}}, + "http_method": {Kind: &structpb.Value_StringValue{StringValue: "%REQ(:METHOD)%"}}, + "original_path": {Kind: &structpb.Value_StringValue{StringValue: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%"}}, + "forwarded_for": {Kind: &structpb.Value_StringValue{StringValue: "%REQ(X-FORWARDED-FOR)%"}}, + "user_agent": {Kind: &structpb.Value_StringValue{StringValue: "%REQ(USER-AGENT)%"}}, + "request_id": {Kind: &structpb.Value_StringValue{StringValue: "%REQ(X-REQUEST-ID)%"}}, + "authority": {Kind: &structpb.Value_StringValue{StringValue: "%REQ(:AUTHORITY)%"}}, + "upstream_host": {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_HOST%"}}, + "response_code": {Kind: &structpb.Value_StringValue{StringValue: "%RESPONSE_CODE%"}}, + "response_code_details": {Kind: &structpb.Value_StringValue{StringValue: "%RESPONSE_CODE_DETAILS%"}}, + "trace_id": {Kind: &structpb.Value_StringValue{StringValue: "%TRACE_ID%"}}, + }, + }, + }, + }, + }, + }), + }, + } +} diff --git a/internal/controlplane/snapshot.go b/internal/controlplane/snapshot.go index 8d24e3a..93b0da7 100644 --- a/internal/controlplane/snapshot.go +++ b/internal/controlplane/snapshot.go @@ -18,30 +18,56 @@ package controlplane import ( "context" + "crypto/sha256" + "fmt" + "net/url" "slices" + "strconv" + "time" + accesslogv3 "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3" + clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + endpointv3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" listenerv3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" - route "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" - router "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + oauth2v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/oauth2/v3" + routerv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" hcm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + matcherv3 "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" "github.com/envoyproxy/go-control-plane/pkg/cache/types" "github.com/envoyproxy/go-control-plane/pkg/cache/v3" "github.com/envoyproxy/go-control-plane/pkg/resource/v3" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/durationpb" + corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + supabasev1alpha1 "code.icb4dc0.de/prskr/supabase-operator/api/v1alpha1" "code.icb4dc0.de/prskr/supabase-operator/internal/supabase" ) +const ( + studioRouteName = "supabase-studio" + dashboardOAuth2ClusterName = "dashboard-oauth" + apiRouteName = "supabase-api" + apilistenerName = "supabase-api" +) + type EnvoyServices struct { - ServiceLabelKey string `json:"-"` - Postgrest *PostgrestCluster `json:"postgrest,omitempty"` - GoTrue *GoTrueCluster `json:"auth,omitempty"` - StorageApi *StorageApiCluster `json:"storageApi,omitempty"` - PGMeta *PGMetaCluster `json:"pgmeta,omitempty"` - Studio *StudioCluster `json:"studio,omitempty"` + client.Client + ServiceLabelKey string + Gateway *supabasev1alpha1.APIGateway + Postgrest *PostgrestCluster + GoTrue *GoTrueCluster + StorageApi *StorageApiCluster + PGMeta *PGMetaCluster + Studio *StudioCluster } func (s *EnvoyServices) UpsertEndpointSlices(endpointSlices ...discoveryv1.EndpointSlice) { @@ -67,6 +93,12 @@ func (s *EnvoyServices) UpsertEndpointSlices(endpointSlices ...discoveryv1.Endpo s.PGMeta = new(PGMetaCluster) } s.PGMeta.AddOrUpdateEndpoints(eps) + case supabase.ServiceConfig.Studio.Name: + if s.Studio == nil { + s.Studio = new(StudioCluster) + } + + s.Studio.AddOrUpdateEndpoints(eps) } } } @@ -97,19 +129,173 @@ func (s EnvoyServices) Targets() map[string][]string { return targets } -func (s *EnvoyServices) snapshot(ctx context.Context, instance, version string) (*cache.Snapshot, error) { - const ( - apiRouteName = "supabase" - studioRouteName = "supabas-studio" - vHostName = "supabase" - listenerName = "supabase" - ) - +func (s *EnvoyServices) snapshot(ctx context.Context, instance, version string) (snapshot *cache.Snapshot, snapshotHash []byte, err error) { logger := log.FromContext(ctx) - apiConnectionManager := &hcm.HttpConnectionManager{ + listeners := []*listenerv3.Listener{{ + Name: apilistenerName, + Address: &corev3.Address{ + Address: &corev3.Address_SocketAddress{ + SocketAddress: &corev3.SocketAddress{ + Protocol: corev3.SocketAddress_TCP, + Address: "0.0.0.0", + PortSpecifier: &corev3.SocketAddress_PortValue{ + PortValue: 8000, + }, + }, + }, + }, + FilterChains: []*listenerv3.FilterChain{ + { + Filters: []*listenerv3.Filter{ + { + Name: FilterNameHttpConnectionManager, + ConfigType: &listenerv3.Filter_TypedConfig{ + TypedConfig: MustAny(s.apiConnectionManager()), + }, + }, + }, + }, + }, + }} + + if studioListener := s.studioListener(); studioListener != nil { + logger.Info("Adding studio listener") + listeners = append(listeners, studioListener) + } + + routes := []types.Resource{s.apiRouteConfiguration(instance)} + + if studioRouteCfg := s.studioRoute(instance); studioRouteCfg != nil { + logger.Info("Adding studio route") + routes = append(routes, studioRouteCfg) + } + + clusters := castResources( + slices.Concat( + s.Postgrest.Cluster(instance), + s.GoTrue.Cluster(instance), + s.StorageApi.Cluster(instance), + s.PGMeta.Cluster(instance), + s.Studio.Cluster(instance), + )...) + + if oauth2Spec := s.Gateway.Spec.DashboardEndpoint.OAuth2(); oauth2Spec != nil { + if oauth2TokenEndpointCluster, err := s.oauth2TokenEndpointCluster(); err != nil { + return nil, nil, err + } else { + logger.Info("Adding OAuth2 token endpoint cluster") + clusters = append(clusters, oauth2TokenEndpointCluster) + } + } + + sdsSecrets, err := s.secrets(ctx) + if err != nil { + return nil, nil, fmt.Errorf("failed to collect dynamic secrets: %w", err) + } + + rawSnapshot := map[resource.Type][]types.Resource{ + resource.ClusterType: clusters, + resource.RouteType: routes, + resource.ListenerType: castResources(listeners...), + resource.SecretType: sdsSecrets, + } + + hash := sha256.New() + + for _, resType := range []resource.Type{resource.ClusterType, resource.RouteType, resource.ListenerType, resource.SecretType} { + for _, resource := range rawSnapshot[resType] { + if raw, err := proto.Marshal(resource); err != nil { + return nil, nil, err + } else { + _, _ = hash.Write(raw) + } + } + } + + snapshot, err = cache.NewSnapshot( + version, + rawSnapshot, + ) + if err != nil { + return nil, nil, err + } + + if err := snapshot.Consistent(); err != nil { + return nil, nil, err + } + + return snapshot, hash.Sum(nil), nil +} + +func (s *EnvoyServices) secrets(ctx context.Context) ([]types.Resource, error) { + var ( + serviceConfig = supabase.ServiceConfig.Envoy + resources = make([]types.Resource, 0, 2) + ) + + hmacSecret, err := s.k8sSecretKeyToSecret( + ctx, + serviceConfig.HmacSecretName(s.Gateway), + serviceConfig.Defaults.HmacSecretKey, + serviceConfig.Defaults.HmacSecretKey, + ) + + if err == nil { + resources = append(resources, hmacSecret) + } else if client.IgnoreNotFound(err) != nil { + return nil, err + } + + if oauth2Spec := s.Gateway.Spec.DashboardEndpoint.OAuth2(); oauth2Spec != nil { + oauth2ClientSecret, err := s.k8sSecretKeyToSecret( + ctx, + oauth2Spec.ClientSecretRef.Name, + oauth2Spec.ClientSecretRef.Key, + serviceConfig.Defaults.OAuth2ClientSecretKey, + ) + + if err == nil { + resources = append(resources, oauth2ClientSecret) + } else if client.IgnoreNotFound(err) != nil { + return nil, err + } + } + + return resources, nil +} + +func (s *EnvoyServices) k8sSecretKeyToSecret(ctx context.Context, secretName, secretKey, envoySecretName string) (*tlsv3.Secret, error) { + k8sSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: s.Gateway.Namespace, + }, + } + + if err := s.Get(ctx, client.ObjectKeyFromObject(k8sSecret), k8sSecret); err != nil { + return nil, err + } + + return &tlsv3.Secret{ + Name: envoySecretName, + Type: &tlsv3.Secret_GenericSecret{ + GenericSecret: &tlsv3.GenericSecret{ + Secret: &corev3.DataSource{ + Specifier: &corev3.DataSource_InlineBytes{ + InlineBytes: k8sSecret.Data[secretKey], + }, + }, + }, + }, + }, nil +} + +func (s *EnvoyServices) apiConnectionManager() *hcm.HttpConnectionManager { + return &hcm.HttpConnectionManager{ CodecType: hcm.HttpConnectionManager_AUTO, - StatPrefix: "http", + StatPrefix: "supabase_rest", + AccessLog: []*accesslogv3.AccessLog{AccessLog("supabase-rest-access-log")}, RouteSpecifier: &hcm.HttpConnectionManager_Rds{ Rds: &hcm.Rds{ ConfigSource: &corev3.ConfigSource{ @@ -141,14 +327,109 @@ func (s *EnvoyServices) snapshot(ctx context.Context, instance, version string) }, { Name: FilterNameHttpRouter, - ConfigType: &hcm.HttpFilter_TypedConfig{TypedConfig: MustAny(new(router.Router))}, + ConfigType: &hcm.HttpFilter_TypedConfig{TypedConfig: MustAny(new(routerv3.Router))}, }, }, } +} + +func (s *EnvoyServices) apiRouteConfiguration(instance string) *routev3.RouteConfiguration { + return &routev3.RouteConfiguration{ + Name: apiRouteName, + VirtualHosts: []*routev3.VirtualHost{{ + Name: "supabase", + Domains: []string{"*"}, + TypedPerFilterConfig: map[string]*anypb.Any{ + FilterNameJwtAuthn: MustAny(JWTPerRouteConfig()), + FilterNameRBAC: MustAny(RBACPerRoute(RBACRequireAuthConfig())), + }, + Routes: slices.Concat( + s.Postgrest.Routes(instance), + s.GoTrue.Routes(instance), + s.StorageApi.Routes(instance), + s.PGMeta.Routes(instance), + ), + }}, + TypedPerFilterConfig: map[string]*anypb.Any{ + FilterNameCORS: MustAny(CorsPolicy()), + }, + } +} + +func (s *EnvoyServices) studioListener() *listenerv3.Listener { + if s.Studio == nil { + return nil + } + + var ( + httpFilters []*hcm.HttpFilter + serviceCfg = supabase.ServiceConfig.Envoy + ) + + if oauth2Spec := s.Gateway.Spec.DashboardEndpoint.OAuth2(); oauth2Spec != nil { + httpFilters = append(httpFilters, &hcm.HttpFilter{ + Name: FilterNameOAuth2, + ConfigType: &hcm.HttpFilter_TypedConfig{TypedConfig: MustAny(&oauth2v3.OAuth2{ + Config: &oauth2v3.OAuth2Config{ + TokenEndpoint: &corev3.HttpUri{ + HttpUpstreamType: &corev3.HttpUri_Cluster{ + Cluster: dashboardOAuth2ClusterName, + }, + Uri: s.Gateway.Spec.DashboardEndpoint.Auth.OAuth2.TokenEndpoint, + Timeout: durationpb.New(3 * time.Second), + }, + AuthorizationEndpoint: s.Gateway.Spec.DashboardEndpoint.Auth.OAuth2.AuthorizationEndpoint, + RedirectUri: "%REQ(x-forwarded-proto)%://%REQ(:authority)%/callback", + RedirectPathMatcher: &matcherv3.PathMatcher{ + Rule: &matcherv3.PathMatcher_Path{ + Path: &matcherv3.StringMatcher{ + MatchPattern: &matcherv3.StringMatcher_Exact{ + Exact: "/callback", + }, + }, + }, + }, + SignoutPath: &matcherv3.PathMatcher{ + Rule: &matcherv3.PathMatcher_Path{ + Path: &matcherv3.StringMatcher{ + MatchPattern: &matcherv3.StringMatcher_Exact{ + Exact: "/signout", + }, + }, + }, + }, + Credentials: &oauth2v3.OAuth2Credentials{ + ClientId: oauth2Spec.ClientID, + TokenSecret: &tlsv3.SdsSecretConfig{ + Name: serviceCfg.Defaults.OAuth2ClientSecretKey, + SdsConfig: &corev3.ConfigSource{ + ConfigSourceSpecifier: &corev3.ConfigSource_Ads{ + Ads: new(corev3.AggregatedConfigSource), + }, + }, + }, + TokenFormation: &oauth2v3.OAuth2Credentials_HmacSecret{ + HmacSecret: &tlsv3.SdsSecretConfig{ + Name: serviceCfg.Defaults.HmacSecretKey, + SdsConfig: &corev3.ConfigSource{ + ConfigSourceSpecifier: &corev3.ConfigSource_Ads{ + Ads: new(corev3.AggregatedConfigSource), + }, + }, + }, + }, + }, + AuthScopes: oauth2Spec.Scopes, + Resources: oauth2Spec.Resources, + }, + })}, + }) + } studioConnetionManager := &hcm.HttpConnectionManager{ CodecType: hcm.HttpConnectionManager_AUTO, - StatPrefix: "http", + StatPrefix: "supbase_studio", + AccessLog: []*accesslogv3.AccessLog{AccessLog("supbase_studio_access_log")}, RouteSpecifier: &hcm.HttpConnectionManager_Rds{ Rds: &hcm.Rds{ ConfigSource: &corev3.ConfigSource{ @@ -169,46 +450,21 @@ func (s *EnvoyServices) snapshot(ctx context.Context, instance, version string) RouteConfigName: studioRouteName, }, }, - HttpFilters: []*hcm.HttpFilter{ - { - Name: FilterNameHttpRouter, - ConfigType: &hcm.HttpFilter_TypedConfig{TypedConfig: MustAny(new(router.Router))}, - }, - }, + HttpFilters: append(httpFilters, &hcm.HttpFilter{ + Name: FilterNameHttpRouter, + ConfigType: &hcm.HttpFilter_TypedConfig{TypedConfig: MustAny(new(routerv3.Router))}, + }), } - apiRouteCfg := &route.RouteConfiguration{ - Name: apiRouteName, - VirtualHosts: []*route.VirtualHost{{ - Name: "supabase", - Domains: []string{"*"}, - TypedPerFilterConfig: map[string]*anypb.Any{ - FilterNameJwtAuthn: MustAny(JWTPerRouteConfig()), - FilterNameRBAC: MustAny(RBACPerRoute(RBACRequireAuthConfig())), - }, - Routes: slices.Concat( - s.Postgrest.Routes(instance), - s.GoTrue.Routes(instance), - s.StorageApi.Routes(instance), - s.PGMeta.Routes(instance), - ), - }}, - TypedPerFilterConfig: map[string]*anypb.Any{ - FilterNameCORS: MustAny(CorsPolicy()), - }, - } - - // TODO add studio route config - - listeners := []*listenerv3.Listener{{ - Name: listenerName, + return &listenerv3.Listener{ + Name: "studio", Address: &corev3.Address{ Address: &corev3.Address_SocketAddress{ SocketAddress: &corev3.SocketAddress{ Protocol: corev3.SocketAddress_TCP, Address: "0.0.0.0", PortSpecifier: &corev3.SocketAddress_PortValue{ - PortValue: 8000, + PortValue: 3000, }, }, }, @@ -219,70 +475,112 @@ func (s *EnvoyServices) snapshot(ctx context.Context, instance, version string) { Name: FilterNameHttpConnectionManager, ConfigType: &listenerv3.Filter_TypedConfig{ - TypedConfig: MustAny(apiConnectionManager), + TypedConfig: MustAny(studioConnetionManager), }, }, }, }, }, - }} + } +} - if s.Studio != nil { - logger.Info("Adding studio listener") +func (s *EnvoyServices) studioRoute(instance string) *routev3.RouteConfiguration { + if s.Studio == nil { + return nil + } - listeners = append(listeners, &listenerv3.Listener{ - Name: "studio", - Address: &corev3.Address{ - Address: &corev3.Address_SocketAddress{ - SocketAddress: &corev3.SocketAddress{ - Protocol: corev3.SocketAddress_TCP, - Address: "0.0.0.0", - PortSpecifier: &corev3.SocketAddress_PortValue{ - PortValue: 3000, - }, - }, - }, - }, - FilterChains: []*listenerv3.FilterChain{ - { - Filters: []*listenerv3.Filter{ - { - Name: FilterNameHttpConnectionManager, - ConfigType: &listenerv3.Filter_TypedConfig{ - TypedConfig: MustAny(studioConnetionManager), + return &routev3.RouteConfiguration{ + Name: studioRouteName, + VirtualHosts: []*routev3.VirtualHost{{ + Name: "supabase-studio", + Domains: []string{"*"}, + Routes: s.Studio.Routes(instance), + }}, + } +} + +func (s *EnvoyServices) oauth2TokenEndpointCluster() (*clusterv3.Cluster, error) { + oauth2Spec := s.Gateway.Spec.DashboardEndpoint.OAuth2() + parsedTokenEndpoint, err := url.Parse(oauth2Spec.TokenEndpoint) + if err != nil { + return nil, fmt.Errorf("failed to parse token endpoint: %w", err) + } + + var ( + endpointPort uint32 + tls bool + ) + switch parsedTokenEndpoint.Scheme { + case "http": + endpointPort = 80 + case "https": + endpointPort = 443 + tls = true + default: + return nil, fmt.Errorf("unsupported token endpoint scheme: %s", parsedTokenEndpoint.Scheme) + } + + if tokenEndpointPort := parsedTokenEndpoint.Port(); tokenEndpointPort != "" { + if parsedPort, err := strconv.ParseUint(tokenEndpointPort, 10, 32); err != nil { + return nil, fmt.Errorf("failed to parse token endpoint port: %w", err) + } else { + endpointPort = uint32(parsedPort) + } + } + + cluster := &clusterv3.Cluster{ + Name: dashboardOAuth2ClusterName, + ConnectTimeout: durationpb.New(3 * time.Second), + ClusterDiscoveryType: &clusterv3.Cluster_Type{ + Type: clusterv3.Cluster_LOGICAL_DNS, + }, + LbPolicy: clusterv3.Cluster_ROUND_ROBIN, + LoadAssignment: &endpointv3.ClusterLoadAssignment{ + ClusterName: dashboardOAuth2ClusterName, + Endpoints: []*endpointv3.LocalityLbEndpoints{{ + LbEndpoints: []*endpointv3.LbEndpoint{{ + HostIdentifier: &endpointv3.LbEndpoint_Endpoint{ + Endpoint: &endpointv3.Endpoint{ + Address: &corev3.Address{ + Address: &corev3.Address_SocketAddress{ + SocketAddress: &corev3.SocketAddress{ + Address: parsedTokenEndpoint.Hostname(), + PortSpecifier: &corev3.SocketAddress_PortValue{ + PortValue: endpointPort, + }, + Protocol: corev3.SocketAddress_TCP, + }, + }, }, }, }, - }, + }}, + }}, + }, + } + + if s.Gateway.Spec.Envoy != nil && s.Gateway.Spec.Envoy.DisableIPv6 { + cluster.DnsLookupFamily = clusterv3.Cluster_V4_ONLY + } + + if tls { + cluster.TransportSocket = &corev3.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &corev3.TransportSocket_TypedConfig{ + TypedConfig: MustAny(&tlsv3.UpstreamTlsContext{ + Sni: parsedTokenEndpoint.Hostname(), + AllowRenegotiation: true, + CommonTlsContext: &tlsv3.CommonTlsContext{ + TlsParams: &tlsv3.TlsParameters{ + TlsMinimumProtocolVersion: tlsv3.TlsParameters_TLSv1_2, + }, + }, + }), }, - }) + } } - rawSnapshot := map[resource.Type][]types.Resource{ - resource.ClusterType: castResources( - slices.Concat( - s.Postgrest.Cluster(instance), - s.GoTrue.Cluster(instance), - s.StorageApi.Cluster(instance), - s.PGMeta.Cluster(instance), - )...), - resource.RouteType: {apiRouteCfg}, - resource.ListenerType: castResources(listeners...), - } - - snapshot, err := cache.NewSnapshot( - version, - rawSnapshot, - ) - if err != nil { - return nil, err - } - - if err := snapshot.Consistent(); err != nil { - return nil, err - } - - return snapshot, nil + return cluster, nil } func castResources[T types.Resource](from ...T) []types.Resource { diff --git a/internal/controlplane/studio.go b/internal/controlplane/studio.go index 1aff3b3..fe79520 100644 --- a/internal/controlplane/studio.go +++ b/internal/controlplane/studio.go @@ -16,6 +16,50 @@ limitations under the License. package controlplane +import ( + "fmt" + + clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + + "code.icb4dc0.de/prskr/supabase-operator/internal/supabase" +) + type StudioCluster struct { ServiceCluster } + +func (c *StudioCluster) Cluster(instance string) []*clusterv3.Cluster { + if c == nil { + return nil + } + + serviceCfg := supabase.ServiceConfig.Studio + + return []*clusterv3.Cluster{ + c.ServiceCluster.Cluster(fmt.Sprintf("%s@%s", serviceCfg.Name, instance), uint32(serviceCfg.Defaults.APIPort)), + } +} + +func (c *StudioCluster) Routes(instance string) []*routev3.Route { + if c == nil { + return nil + } + + return []*routev3.Route{{ + Name: "Studio: /* -> http://studio:3000/*", + Match: &routev3.RouteMatch{ + PathSpecifier: &routev3.RouteMatch_Prefix{ + Prefix: "/", + }, + }, + Action: &routev3.Route_Route{ + Route: &routev3.RouteAction{ + ClusterSpecifier: &routev3.RouteAction_Cluster{ + Cluster: fmt.Sprintf("%s@%s", supabase.ServiceConfig.Studio.Name, instance), + }, + PrefixRewrite: "/", + }, + }, + }} +} diff --git a/internal/health/cert_valid.go b/internal/health/cert_valid.go new file mode 100644 index 0000000..b3fbbeb --- /dev/null +++ b/internal/health/cert_valid.go @@ -0,0 +1,42 @@ +/* +Copyright 2025 Peter Kurfer. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package health + +import ( + "crypto/tls" + "errors" + "fmt" + "net/http" + "time" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" +) + +var ErrCertExpired = errors.New("certificate expired") + +func CertValidCheck(cert tls.Certificate) healthz.Checker { + return func(req *http.Request) error { + if cert.Leaf.NotAfter.Before(time.Now()) { + err := fmt.Errorf("%w: %s", ErrCertExpired, cert.Leaf.Subject.CommonName) + ctrl.Log.Error(err, "certificate expired", "commonName", cert.Leaf.Subject.CommonName, "not_after", cert.Leaf.NotAfter) + return err + } + + return nil + } +} diff --git a/internal/supabase/auth.go b/internal/supabase/auth.go index 350c3ba..55f8101 100644 --- a/internal/supabase/auth.go +++ b/internal/supabase/auth.go @@ -50,7 +50,8 @@ type authConfigDefaults struct { func authServiceConfig() serviceConfig[authEnvKeys, authConfigDefaults] { return serviceConfig[authEnvKeys, authConfigDefaults]{ - Name: "auth", + Name: "auth", + LivenessProbePath: "/health", EnvKeys: authEnvKeys{ ApiHost: fixedEnvOf("GOTRUE_API_HOST", "0.0.0.0"), ApiPort: fixedEnvOf("GOTRUE_API_PORT", "9999"), diff --git a/internal/supabase/env.go b/internal/supabase/env.go index 78af7e5..7e3d870 100644 --- a/internal/supabase/env.go +++ b/internal/supabase/env.go @@ -23,9 +23,22 @@ import ( ) type serviceConfig[TEnvKeys, TDefaults any] struct { - Name string - EnvKeys TEnvKeys - Defaults TDefaults + Name string + LivenessProbePath string + ReadinessProbePath string + EnvKeys TEnvKeys + Defaults TDefaults +} + +func (cfg serviceConfig[TEnvKeys, TDefaults]) ReadinessPath() string { + return cfg.ReadinessProbePath +} + +func (cfg serviceConfig[TEnvKeys, TDefaults]) LivenessPath() string { + if cfg.LivenessProbePath == "" { + return cfg.ReadinessProbePath + } + return cfg.LivenessProbePath } func (cfg serviceConfig[TEnvKeys, TDefaults]) ObjectName(obj metav1.Object) string { diff --git a/internal/supabase/envoy.go b/internal/supabase/envoy.go index 27a465f..f2051f9 100644 --- a/internal/supabase/envoy.go +++ b/internal/supabase/envoy.go @@ -25,16 +25,27 @@ import ( func newEnvoyServiceConfig() envoyServiceConfig { return envoyServiceConfig{ Defaults: envoyDefaults{ - ConfigKey: "config.yaml", - UID: 65532, - GID: 65532, + ConfigKey: "config.yaml", + OAuth2ClientSecretKey: "oauth2_client_secret", + HmacSecretKey: "oauth2_hmac_secret", + UID: 65532, + GID: 65532, + StudioPortName: "studio", + ApiPortName: "api", + StudioPort: 3000, + ApiPort: 8000, + AdminPort: 19000, }, } } type envoyDefaults struct { - ConfigKey string - UID, GID int64 + ConfigKey string + HmacSecretKey string + OAuth2ClientSecretKey string + UID, GID int64 + StudioPortName, ApiPortName string + StudioPort, ApiPort, AdminPort int32 } type envoyServiceConfig struct { @@ -44,3 +55,11 @@ type envoyServiceConfig struct { func (envoyServiceConfig) ObjectName(obj metav1.Object) string { return fmt.Sprintf("%s-envoy", obj.GetName()) } + +func (envoyServiceConfig) ControlPlaneClientCertSecretName(obj metav1.Object) string { + return fmt.Sprintf("%s-cp-client-cert", obj.GetName()) +} + +func (envoyServiceConfig) HmacSecretName(obj metav1.Object) string { + return fmt.Sprintf("%s-hmac-secret", obj.GetName()) +} diff --git a/internal/supabase/pg_meta.go b/internal/supabase/pg_meta.go index 5029fc5..b0b6eeb 100644 --- a/internal/supabase/pg_meta.go +++ b/internal/supabase/pg_meta.go @@ -34,7 +34,8 @@ type pgMetaDefaults struct { func pgMetaServiceConfig() serviceConfig[pgMetaEnvKeys, pgMetaDefaults] { return serviceConfig[pgMetaEnvKeys, pgMetaDefaults]{ - Name: "pg-meta", + Name: "pg-meta", + LivenessProbePath: "/health", EnvKeys: pgMetaEnvKeys{ APIPort: "PG_META_PORT", DBHost: "PG_META_DB_HOST", diff --git a/internal/supabase/postgrest.go b/internal/supabase/postgrest.go index 842a8f5..57553d6 100644 --- a/internal/supabase/postgrest.go +++ b/internal/supabase/postgrest.go @@ -42,7 +42,8 @@ type postgrestConfigDefaults struct { func postgrestServiceConfig() serviceConfig[postgrestEnvKeys, postgrestConfigDefaults] { return serviceConfig[postgrestEnvKeys, postgrestConfigDefaults]{ - Name: "postgrest", + Name: "postgrest", + LivenessProbePath: "/ready", EnvKeys: postgrestEnvKeys{ Host: fixedEnvOf("PGRST_SERVER_HOST", "*"), DBUri: "PGRST_DB_URI", diff --git a/internal/supabase/storage.go b/internal/supabase/storage.go index ecf0979..d9318d7 100644 --- a/internal/supabase/storage.go +++ b/internal/supabase/storage.go @@ -53,7 +53,8 @@ type storageApiDefaults struct { func storageServiceConfig() serviceConfig[storageEnvApiKeys, storageApiDefaults] { return serviceConfig[storageEnvApiKeys, storageApiDefaults]{ - Name: "storage-api", + Name: "storage-api", + LivenessProbePath: "/status", EnvKeys: storageEnvApiKeys{ AnonKey: "ANON_KEY", ServiceKey: "SERVICE_KEY", diff --git a/internal/supabase/studio.go b/internal/supabase/studio.go index 26ceb59..6e1991e 100644 --- a/internal/supabase/studio.go +++ b/internal/supabase/studio.go @@ -36,7 +36,8 @@ type studioDefaults struct { func studioServiceConfig() serviceConfig[studioEnvKeys, studioDefaults] { return serviceConfig[studioEnvKeys, studioDefaults]{ - Name: "studio", + Name: "studio", + LivenessProbePath: "/api/profile", EnvKeys: studioEnvKeys{ PGMetaURL: "STUDIO_PG_META_URL", DBPassword: "POSTGRES_PASSWORD",