prskr/supabase-operator
1
0
Fork 0
Code Issues6 Pull requests Projects Releases7 Packages2 Wiki Activity Actions

fix(envoy): version not handled properly

Browse source
This commit is contained in:
Peter 2025-02-03 15:16:54 +01:00
parent 0fccef973f
commit 867daaa375
Signed by: prskr
GPG key ID: F56BED6903BC5E37
9 changed files with 198 additions and 182 deletions

198
internal/controller/apigateway_controller.go
View file

@ -419,8 +419,6 @@ func (r *APIGatewayReconciler) reconileEnvoyDeployment(
const (
configVolumeName = "config"
controlPlaneTlsVolumeName = "cp-tls"
dashboardTlsVolumeName = "dashboard-tls"
apiTlsVolumeName = "api-tls"
)
envoyDeployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
@ -448,131 +446,6 @@ func (r *APIGatewayReconciler) reconileEnvoyDeployment(
envoyDeployment.Spec.Replicas = envoySpec.WorkloadTemplate.ReplicaCount()
configVolumeProjectionSources := []corev1.VolumeProjection{
{
ConfigMap: &corev1.ConfigMapProjection{
LocalObjectReference: corev1.LocalObjectReference{
Name: supabase.ServiceConfig.Envoy.ObjectName(gateway),
},
Items: []corev1.KeyToPath{
{
Key: "config.yaml",
Path: "config.yaml",
},
},
},
},
{
Secret: &corev1.SecretProjection{
LocalObjectReference: corev1.LocalObjectReference{
Name: gateway.Spec.ApiEndpoint.JWKSSelector.Name,
},
Items: []corev1.KeyToPath{{
Key: gateway.Spec.ApiEndpoint.JWKSSelector.Key,
Path: "jwks.json",
}},
},
},
{
Secret: &corev1.SecretProjection{
LocalObjectReference: corev1.LocalObjectReference{
Name: serviceCfg.ControlPlaneClientCertSecretName(gateway),
},
Items: []corev1.KeyToPath{
{
Key: "ca.crt",
Path: "certs/cp/ca.crt",
},
{
Key: "tls.crt",
Path: "certs/cp/tls.crt",
},
{
Key: "tls.key",
Path: "certs/cp/tls.key",
},
},
},
},
}
if oauth2Spec := gateway.Spec.DashboardEndpoint.OAuth2(); oauth2Spec != nil {
configVolumeProjectionSources = append(configVolumeProjectionSources, corev1.VolumeProjection{
Secret: &corev1.SecretProjection{
LocalObjectReference: corev1.LocalObjectReference{
Name: oauth2Spec.ClientSecretRef.Name,
},
Items: []corev1.KeyToPath{{
Key: oauth2Spec.ClientSecretRef.Key,
Path: serviceCfg.Defaults.OAuth2ClientSecretKey,
}},
},
})
}
volumeMounts := []corev1.VolumeMount{
{
Name: configVolumeName,
ReadOnly: true,
MountPath: "/etc/envoy",
},
}
volumes := []corev1.Volume{
{
Name: configVolumeName,
VolumeSource: corev1.VolumeSource{
Projected: &corev1.ProjectedVolumeSource{
Sources: configVolumeProjectionSources,
},
},
},
{
Name: controlPlaneTlsVolumeName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: serviceCfg.ControlPlaneClientCertSecretName(gateway),
},
},
},
}
if tlsSpec := gateway.Spec.ApiEndpoint.TLSSpec(); tlsSpec != nil {
volumes = append(volumes, corev1.Volume{
Name: apiTlsVolumeName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: tlsSpec.Cert.SecretName,
},
},
})
volumeMounts = append(volumeMounts, corev1.VolumeMount{
Name: dashboardTlsVolumeName,
ReadOnly: true,
MountPath: "/etc/envoy/certs/api",
SubPath: "certs/api",
})
}
if tlsSpec := gateway.Spec.DashboardEndpoint.TLSSpec(); tlsSpec != nil {
volumes = append(volumes, corev1.Volume{
Name: dashboardTlsVolumeName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: tlsSpec.Cert.SecretName,
},
},
})
volumeMounts = append(volumeMounts, corev1.VolumeMount{
Name: dashboardTlsVolumeName,
ReadOnly: true,
MountPath: "/etc/envoy/certs/dashboard",
SubPath: "certs/dashboard",
})
}
envoyDeployment.Spec.Template = corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
@ -632,11 +505,78 @@ func (r *APIGatewayReconciler) reconileEnvoyDeployment(
},
SecurityContext: envoySpec.WorkloadTemplate.ContainerSecurityContext(serviceCfg.Defaults.UID, serviceCfg.Defaults.GID),
Resources: envoySpec.WorkloadTemplate.Resources(),
VolumeMounts: envoySpec.WorkloadTemplate.AdditionalVolumeMounts(volumeMounts...),
VolumeMounts: envoySpec.WorkloadTemplate.AdditionalVolumeMounts(corev1.VolumeMount{
Name: configVolumeName,
ReadOnly: true,
MountPath: "/etc/envoy",
}),
},
},
SecurityContext: envoySpec.WorkloadTemplate.PodSecurityContext(),
Volumes: volumes,
Volumes: []corev1.Volume{
{
Name: configVolumeName,
VolumeSource: corev1.VolumeSource{
Projected: &corev1.ProjectedVolumeSource{
Sources: []corev1.VolumeProjection{
{
ConfigMap: &corev1.ConfigMapProjection{
LocalObjectReference: corev1.LocalObjectReference{
Name: supabase.ServiceConfig.Envoy.ObjectName(gateway),
},
Items: []corev1.KeyToPath{
{
Key: "config.yaml",
Path: "config.yaml",
},
},
},
},
{
Secret: &corev1.SecretProjection{
LocalObjectReference: corev1.LocalObjectReference{
Name: gateway.Spec.ApiEndpoint.JWKSSelector.Name,
},
Items: []corev1.KeyToPath{{
Key: gateway.Spec.ApiEndpoint.JWKSSelector.Key,
Path: "jwks.json",
}},
},
},
{
Secret: &corev1.SecretProjection{
LocalObjectReference: corev1.LocalObjectReference{
Name: serviceCfg.ControlPlaneClientCertSecretName(gateway),
},
Items: []corev1.KeyToPath{
{
Key: "ca.crt",
Path: "certs/cp/ca.crt",
},
{
Key: "tls.crt",
Path: "certs/cp/tls.crt",
},
{
Key: "tls.key",
Path: "certs/cp/tls.key",
},
},
},
},
},
},
},
},
{
Name: controlPlaneTlsVolumeName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: serviceCfg.ControlPlaneClientCertSecretName(gateway),
},
},
},
},
},
}

2
internal/controller/templates/envoy_control_plane_config.yaml.tmpl
View file

@ -17,7 +17,7 @@ static_resources:
clusters:
- name: {{ .ControlPlane.Name }}
type: STRICT_DNS
connect_timeout: 1s
connect_timeout: 5s
load_assignment:
cluster_name: {{ .ControlPlane.Name }}
endpoints:

37
internal/controlplane/apigateway_controller.go
View file

@ -17,7 +17,6 @@ limitations under the License.
package controlplane
import (
"bytes"
"context"
"fmt"
"strconv"
@ -87,23 +86,19 @@ func (r *APIGatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request)
}
services.UpsertEndpointSlices(endpointSliceList.Items...)
instance := fmt.Sprintf("%s:%s", gateway.Spec.Envoy.NodeName, gateway.Namespace)
var (
instance = fmt.Sprintf("%s:%s", gateway.Spec.Envoy.NodeName, gateway.Namespace)
snapshotVersion = strconv.FormatInt(time.Now().UTC().UnixMilli(), 10)
)
logger.Info("Computing Envoy snapshot for current service targets", "version", gateway.Status.Envoy.ConfigVersion)
snapshot, snapshotHash, err := services.snapshot(ctx, instance, gateway.Status.Envoy.ConfigVersion)
logger.Info("Computing Envoy snapshot for current service targets", "version", snapshotVersion)
snapshot, snapshotHash, err := services.snapshot(ctx, instance, snapshotVersion)
if err != nil {
return ctrl.Result{}, fmt.Errorf("failed to prepare snapshot: %w", err)
}
if !r.initialReconciliation.CompareAndSwap(false, true) && bytes.Equal(gateway.Status.Envoy.ResourceHash, snapshotHash) {
logger.Info("No changes detected, skipping update")
return ctrl.Result{}, nil
}
logger.Info("Updating service targets")
_, err = controllerutil.CreateOrUpdate(ctx, r.Client, &gateway, func() error {
opResult, err := controllerutil.CreateOrUpdate(ctx, r.Client, &gateway, func() error {
gateway.Status.ServiceTargets = services.Targets()
gateway.Status.Envoy.ConfigVersion = strconv.FormatInt(time.Now().UTC().UnixMilli(), 10)
gateway.Status.Envoy.ResourceHash = snapshotHash
return nil
@ -112,11 +107,27 @@ func (r *APIGatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return ctrl.Result{}, err
}
logger.Info("Propagating Envoy snapshot", "version", gateway.Status.Envoy.ConfigVersion)
if opResult == controllerutil.OperationResultNone {
logger.Info("No changes detected in APIGateway object")
return ctrl.Result{}, nil
}
logger.Info("Propagating Envoy snapshot", "version", snapshotVersion)
if err := r.Cache.SetSnapshot(ctx, instance, snapshot); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to propagate snapshot: %w", err)
}
if info := r.Cache.GetStatusInfo(instance); info != nil {
node := info.GetNode()
logger = logger.WithValues(
"node.id", node.Id,
"node.cluster", node.Cluster,
"node.num_watches", info.GetNumWatches(),
)
}
logger.Info("Envoy snapshot propagated successfully")
return ctrl.Result{}, nil
}

6
internal/controlplane/auth_filters.go
View file

@ -148,6 +148,12 @@ func RBACAllowAllConfig() *rbacv3.RBAC {
}
func RBACRequireAuthConfig() *rbacv3.RBAC {
/*
Identifier: &rbacv3cfg.Principal_SourcedMetadata{
SourcedMetadata: &rbacv3cfg.SourcedMetadata{
MetadataSource: rbacv3cfg.MetadataSource_DYNAMIC,
MetadataMatcher: &matcherv3.MetadataMatcher{
*/
return &rbacv3.RBAC{
Rules: &rbacv3cfg.RBAC{
Action: rbacv3cfg.RBAC_ALLOW,

6
internal/controlplane/snapshot.go
View file

@ -46,7 +46,6 @@ import (
discoveryv1 "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
supabasev1alpha1 "code.icb4dc0.de/prskr/supabase-operator/api/v1alpha1"
"code.icb4dc0.de/prskr/supabase-operator/internal/supabase"
@ -130,8 +129,6 @@ func (s EnvoyServices) Targets() map[string][]string {
}
func (s *EnvoyServices) snapshot(ctx context.Context, instance, version string) (snapshot *cache.Snapshot, snapshotHash []byte, err error) {
logger := log.FromContext(ctx)
listeners := []*listenerv3.Listener{{
Name: apilistenerName,
Address: &corev3.Address{
@ -160,14 +157,12 @@ func (s *EnvoyServices) snapshot(ctx context.Context, instance, version string)
}}
if studioListener := s.studioListener(); studioListener != nil {
logger.Info("Adding studio listener")
listeners = append(listeners, studioListener)
}
routes := []types.Resource{s.apiRouteConfiguration(instance)}
if studioRouteCfg := s.studioRoute(instance); studioRouteCfg != nil {
logger.Info("Adding studio route")
routes = append(routes, studioRouteCfg)
}
@ -184,7 +179,6 @@ func (s *EnvoyServices) snapshot(ctx context.Context, instance, version string)
if oauth2TokenEndpointCluster, err := s.oauth2TokenEndpointCluster(); err != nil {
return nil, nil, err
} else {
logger.Info("Adding OAuth2 token endpoint cluster")
clusters = append(clusters, oauth2TokenEndpointCluster)
}
}