supabase-operator/internal/controlplane/snapshot.go

587 lines
17 KiB
Go

/*
Copyright 2025 Peter Kurfer.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controlplane
import (
"context"
"crypto/sha256"
"fmt"
"net/url"
"slices"
"strconv"
"time"
accesslogv3 "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3"
clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
endpointv3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
listenerv3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
oauth2v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/oauth2/v3"
routerv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3"
hcm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
matcherv3 "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
"github.com/envoyproxy/go-control-plane/pkg/cache/types"
"github.com/envoyproxy/go-control-plane/pkg/cache/v3"
"github.com/envoyproxy/go-control-plane/pkg/resource/v3"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"google.golang.org/protobuf/types/known/durationpb"
corev1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
supabasev1alpha1 "code.icb4dc0.de/prskr/supabase-operator/api/v1alpha1"
"code.icb4dc0.de/prskr/supabase-operator/internal/supabase"
)
const (
studioRouteName = "supabase-studio"
dashboardOAuth2ClusterName = "dashboard-oauth"
apiRouteName = "supabase-api"
apilistenerName = "supabase-api"
)
type EnvoyServices struct {
client.Client
ServiceLabelKey string
Gateway *supabasev1alpha1.APIGateway
Postgrest *PostgrestCluster
GoTrue *GoTrueCluster
StorageApi *StorageApiCluster
PGMeta *PGMetaCluster
Studio *StudioCluster
}
func (s *EnvoyServices) UpsertEndpointSlices(endpointSlices ...discoveryv1.EndpointSlice) {
for _, eps := range endpointSlices {
switch eps.Labels[s.ServiceLabelKey] {
case supabase.ServiceConfig.Postgrest.Name:
if s.Postgrest == nil {
s.Postgrest = new(PostgrestCluster)
}
s.Postgrest.AddOrUpdateEndpoints(eps)
case supabase.ServiceConfig.Auth.Name:
if s.GoTrue == nil {
s.GoTrue = new(GoTrueCluster)
}
s.GoTrue.AddOrUpdateEndpoints(eps)
case supabase.ServiceConfig.Storage.Name:
if s.StorageApi == nil {
s.StorageApi = new(StorageApiCluster)
}
s.StorageApi.AddOrUpdateEndpoints(eps)
case supabase.ServiceConfig.PGMeta.Name:
if s.PGMeta == nil {
s.PGMeta = new(PGMetaCluster)
}
s.PGMeta.AddOrUpdateEndpoints(eps)
case supabase.ServiceConfig.Studio.Name:
if s.Studio == nil {
s.Studio = new(StudioCluster)
}
s.Studio.AddOrUpdateEndpoints(eps)
}
}
}
func (s EnvoyServices) Targets() map[string][]string {
targets := make(map[string][]string)
if s.Postgrest != nil {
targets[supabase.ServiceConfig.Postgrest.Name] = s.Postgrest.Targets()
}
if s.GoTrue != nil {
targets[supabase.ServiceConfig.Auth.Name] = s.GoTrue.Targets()
}
if s.StorageApi != nil {
targets[supabase.ServiceConfig.Storage.Name] = s.StorageApi.Targets()
}
if s.PGMeta != nil {
targets[supabase.ServiceConfig.PGMeta.Name] = s.PGMeta.Targets()
}
if s.Studio != nil {
targets[supabase.ServiceConfig.Studio.Name] = s.Studio.Targets()
}
return targets
}
func (s *EnvoyServices) snapshot(ctx context.Context, instance, version string) (snapshot *cache.Snapshot, snapshotHash []byte, err error) {
listeners := []*listenerv3.Listener{{
Name: apilistenerName,
Address: &corev3.Address{
Address: &corev3.Address_SocketAddress{
SocketAddress: &corev3.SocketAddress{
Protocol: corev3.SocketAddress_TCP,
Address: "0.0.0.0",
PortSpecifier: &corev3.SocketAddress_PortValue{
PortValue: 8000,
},
},
},
},
FilterChains: []*listenerv3.FilterChain{
{
Filters: []*listenerv3.Filter{
{
Name: FilterNameHttpConnectionManager,
ConfigType: &listenerv3.Filter_TypedConfig{
TypedConfig: MustAny(s.apiConnectionManager()),
},
},
},
},
},
}}
if studioListener := s.studioListener(instance); studioListener != nil {
listeners = append(listeners, studioListener)
}
routes := []types.Resource{s.apiRouteConfiguration(instance)}
if studioRouteCfg := s.studioRoute(instance); studioRouteCfg != nil {
routes = append(routes, studioRouteCfg)
}
clusters := castResources(
slices.Concat(
s.Postgrest.Cluster(instance),
s.GoTrue.Cluster(instance),
s.StorageApi.Cluster(instance),
s.PGMeta.Cluster(instance),
s.Studio.Cluster(instance),
)...)
if oauth2Spec := s.Gateway.Spec.DashboardEndpoint.OAuth2(); oauth2Spec != nil {
if oauth2TokenEndpointCluster, err := s.oauth2TokenEndpointCluster(instance); err != nil {
return nil, nil, err
} else {
clusters = append(clusters, oauth2TokenEndpointCluster)
}
}
sdsSecrets, err := s.secrets(ctx)
if err != nil {
return nil, nil, fmt.Errorf("failed to collect dynamic secrets: %w", err)
}
rawSnapshot := map[resource.Type][]types.Resource{
resource.ClusterType: clusters,
resource.RouteType: routes,
resource.ListenerType: castResources(listeners...),
resource.SecretType: sdsSecrets,
}
hash := sha256.New()
for _, resType := range []resource.Type{resource.ClusterType, resource.RouteType, resource.ListenerType, resource.SecretType} {
for _, resource := range rawSnapshot[resType] {
if raw, err := proto.Marshal(resource); err != nil {
return nil, nil, err
} else {
_, _ = hash.Write(raw)
}
}
}
snapshot, err = cache.NewSnapshot(
version,
rawSnapshot,
)
if err != nil {
return nil, nil, err
}
if err := snapshot.Consistent(); err != nil {
return nil, nil, err
}
return snapshot, hash.Sum(nil), nil
}
func (s *EnvoyServices) secrets(ctx context.Context) ([]types.Resource, error) {
var (
serviceConfig = supabase.ServiceConfig.Envoy
resources = make([]types.Resource, 0, 2)
)
hmacSecret, err := s.k8sSecretKeyToSecret(
ctx,
serviceConfig.HmacSecretName(s.Gateway),
serviceConfig.Defaults.HmacSecretKey,
serviceConfig.Defaults.HmacSecretKey,
)
if err == nil {
resources = append(resources, hmacSecret)
} else if client.IgnoreNotFound(err) != nil {
return nil, err
}
if oauth2Spec := s.Gateway.Spec.DashboardEndpoint.OAuth2(); oauth2Spec != nil {
oauth2ClientSecret, err := s.k8sSecretKeyToSecret(
ctx,
oauth2Spec.ClientSecretRef.Name,
oauth2Spec.ClientSecretRef.Key,
serviceConfig.Defaults.OAuth2ClientSecretKey,
)
if err == nil {
resources = append(resources, oauth2ClientSecret)
} else if client.IgnoreNotFound(err) != nil {
return nil, err
}
}
return resources, nil
}
func (s *EnvoyServices) k8sSecretKeyToSecret(ctx context.Context, secretName, secretKey, envoySecretName string) (*tlsv3.Secret, error) {
k8sSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: s.Gateway.Namespace,
},
}
if err := s.Get(ctx, client.ObjectKeyFromObject(k8sSecret), k8sSecret); err != nil {
return nil, err
}
return &tlsv3.Secret{
Name: envoySecretName,
Type: &tlsv3.Secret_GenericSecret{
GenericSecret: &tlsv3.GenericSecret{
Secret: &corev3.DataSource{
Specifier: &corev3.DataSource_InlineBytes{
InlineBytes: k8sSecret.Data[secretKey],
},
},
},
},
}, nil
}
func (s *EnvoyServices) apiConnectionManager() *hcm.HttpConnectionManager {
return &hcm.HttpConnectionManager{
CodecType: hcm.HttpConnectionManager_AUTO,
StatPrefix: "supabase_rest",
AccessLog: []*accesslogv3.AccessLog{AccessLog("supabase-rest-access-log")},
RouteSpecifier: &hcm.HttpConnectionManager_Rds{
Rds: &hcm.Rds{
ConfigSource: &corev3.ConfigSource{
ResourceApiVersion: resource.DefaultAPIVersion,
ConfigSourceSpecifier: &corev3.ConfigSource_ApiConfigSource{
ApiConfigSource: &corev3.ApiConfigSource{
TransportApiVersion: resource.DefaultAPIVersion,
ApiType: corev3.ApiConfigSource_GRPC,
SetNodeOnFirstMessageOnly: true,
GrpcServices: []*corev3.GrpcService{{
TargetSpecifier: &corev3.GrpcService_EnvoyGrpc_{
EnvoyGrpc: &corev3.GrpcService_EnvoyGrpc{ClusterName: "supabase-control-plane"},
},
}},
},
},
},
RouteConfigName: apiRouteName,
},
},
HttpFilters: []*hcm.HttpFilter{
{
Name: FilterNameJwtAuthn,
ConfigType: &hcm.HttpFilter_TypedConfig{TypedConfig: MustAny(JWTFilterConfig())},
},
{
Name: FilterNameCORS,
ConfigType: &hcm.HttpFilter_TypedConfig{TypedConfig: MustAny(Cors())},
},
{
Name: FilterNameHttpRouter,
ConfigType: &hcm.HttpFilter_TypedConfig{TypedConfig: MustAny(new(routerv3.Router))},
},
},
}
}
func (s *EnvoyServices) apiRouteConfiguration(instance string) *routev3.RouteConfiguration {
return &routev3.RouteConfiguration{
Name: apiRouteName,
VirtualHosts: []*routev3.VirtualHost{{
Name: "supabase",
Domains: []string{"*"},
TypedPerFilterConfig: map[string]*anypb.Any{
FilterNameJwtAuthn: MustAny(JWTPerRouteConfig()),
FilterNameRBAC: MustAny(RBACPerRoute(RBACRequireAuthConfig())),
},
Routes: slices.Concat(
s.Postgrest.Routes(instance),
s.GoTrue.Routes(instance),
s.StorageApi.Routes(instance),
s.PGMeta.Routes(instance),
),
}},
TypedPerFilterConfig: map[string]*anypb.Any{
FilterNameCORS: MustAny(CorsPolicy()),
},
}
}
func (s *EnvoyServices) studioListener(instance string) *listenerv3.Listener {
if s.Studio == nil {
return nil
}
var (
httpFilters []*hcm.HttpFilter
serviceCfg = supabase.ServiceConfig.Envoy
)
if oauth2Spec := s.Gateway.Spec.DashboardEndpoint.OAuth2(); oauth2Spec != nil {
httpFilters = append(httpFilters, &hcm.HttpFilter{
Name: FilterNameOAuth2,
ConfigType: &hcm.HttpFilter_TypedConfig{TypedConfig: MustAny(&oauth2v3.OAuth2{
Config: &oauth2v3.OAuth2Config{
TokenEndpoint: &corev3.HttpUri{
HttpUpstreamType: &corev3.HttpUri_Cluster{
Cluster: fmt.Sprintf("%s@%s", dashboardOAuth2ClusterName, instance),
},
Uri: s.Gateway.Spec.DashboardEndpoint.Auth.OAuth2.TokenEndpoint,
Timeout: durationpb.New(3 * time.Second),
},
AuthorizationEndpoint: s.Gateway.Spec.DashboardEndpoint.Auth.OAuth2.AuthorizationEndpoint,
RedirectUri: "%REQ(x-forwarded-proto)%://%REQ(:authority)%/callback",
RedirectPathMatcher: &matcherv3.PathMatcher{
Rule: &matcherv3.PathMatcher_Path{
Path: &matcherv3.StringMatcher{
MatchPattern: &matcherv3.StringMatcher_Exact{
Exact: "/callback",
},
},
},
},
SignoutPath: &matcherv3.PathMatcher{
Rule: &matcherv3.PathMatcher_Path{
Path: &matcherv3.StringMatcher{
MatchPattern: &matcherv3.StringMatcher_Exact{
Exact: "/signout",
},
},
},
},
Credentials: &oauth2v3.OAuth2Credentials{
ClientId: oauth2Spec.ClientID,
TokenSecret: &tlsv3.SdsSecretConfig{
Name: serviceCfg.Defaults.OAuth2ClientSecretKey,
SdsConfig: &corev3.ConfigSource{
ConfigSourceSpecifier: &corev3.ConfigSource_Ads{
Ads: new(corev3.AggregatedConfigSource),
},
},
},
TokenFormation: &oauth2v3.OAuth2Credentials_HmacSecret{
HmacSecret: &tlsv3.SdsSecretConfig{
Name: serviceCfg.Defaults.HmacSecretKey,
SdsConfig: &corev3.ConfigSource{
ConfigSourceSpecifier: &corev3.ConfigSource_Ads{
Ads: new(corev3.AggregatedConfigSource),
},
},
},
},
},
AuthScopes: oauth2Spec.Scopes,
Resources: oauth2Spec.Resources,
},
})},
})
}
studioConnetionManager := &hcm.HttpConnectionManager{
CodecType: hcm.HttpConnectionManager_AUTO,
StatPrefix: "supbase_studio",
AccessLog: []*accesslogv3.AccessLog{AccessLog("supbase_studio_access_log")},
RouteSpecifier: &hcm.HttpConnectionManager_Rds{
Rds: &hcm.Rds{
ConfigSource: &corev3.ConfigSource{
ResourceApiVersion: resource.DefaultAPIVersion,
ConfigSourceSpecifier: &corev3.ConfigSource_ApiConfigSource{
ApiConfigSource: &corev3.ApiConfigSource{
TransportApiVersion: resource.DefaultAPIVersion,
ApiType: corev3.ApiConfigSource_GRPC,
SetNodeOnFirstMessageOnly: true,
GrpcServices: []*corev3.GrpcService{{
TargetSpecifier: &corev3.GrpcService_EnvoyGrpc_{
EnvoyGrpc: &corev3.GrpcService_EnvoyGrpc{ClusterName: "supabase-control-plane"},
},
}},
},
},
},
RouteConfigName: studioRouteName,
},
},
HttpFilters: append(httpFilters, &hcm.HttpFilter{
Name: FilterNameHttpRouter,
ConfigType: &hcm.HttpFilter_TypedConfig{TypedConfig: MustAny(new(routerv3.Router))},
}),
}
return &listenerv3.Listener{
Name: "studio",
Address: &corev3.Address{
Address: &corev3.Address_SocketAddress{
SocketAddress: &corev3.SocketAddress{
Protocol: corev3.SocketAddress_TCP,
Address: "0.0.0.0",
PortSpecifier: &corev3.SocketAddress_PortValue{
PortValue: 3000,
},
},
},
},
FilterChains: []*listenerv3.FilterChain{
{
Filters: []*listenerv3.Filter{
{
Name: FilterNameHttpConnectionManager,
ConfigType: &listenerv3.Filter_TypedConfig{
TypedConfig: MustAny(studioConnetionManager),
},
},
},
},
},
}
}
func (s *EnvoyServices) studioRoute(instance string) *routev3.RouteConfiguration {
if s.Studio == nil {
return nil
}
return &routev3.RouteConfiguration{
Name: studioRouteName,
VirtualHosts: []*routev3.VirtualHost{{
Name: "supabase-studio",
Domains: []string{"*"},
Routes: s.Studio.Routes(instance),
}},
}
}
func (s *EnvoyServices) oauth2TokenEndpointCluster(instance string) (*clusterv3.Cluster, error) {
oauth2Spec := s.Gateway.Spec.DashboardEndpoint.OAuth2()
parsedTokenEndpoint, err := url.Parse(oauth2Spec.TokenEndpoint)
if err != nil {
return nil, fmt.Errorf("failed to parse token endpoint: %w", err)
}
var (
endpointPort uint32
tls bool
)
switch parsedTokenEndpoint.Scheme {
case "http":
endpointPort = 80
case "https":
endpointPort = 443
tls = true
default:
return nil, fmt.Errorf("unsupported token endpoint scheme: %s", parsedTokenEndpoint.Scheme)
}
if tokenEndpointPort := parsedTokenEndpoint.Port(); tokenEndpointPort != "" {
if parsedPort, err := strconv.ParseUint(tokenEndpointPort, 10, 32); err != nil {
return nil, fmt.Errorf("failed to parse token endpoint port: %w", err)
} else {
endpointPort = uint32(parsedPort)
}
}
cluster := &clusterv3.Cluster{
Name: fmt.Sprintf("%s@%s", dashboardOAuth2ClusterName, instance),
ConnectTimeout: durationpb.New(3 * time.Second),
ClusterDiscoveryType: &clusterv3.Cluster_Type{
Type: clusterv3.Cluster_LOGICAL_DNS,
},
LbPolicy: clusterv3.Cluster_ROUND_ROBIN,
LoadAssignment: &endpointv3.ClusterLoadAssignment{
ClusterName: dashboardOAuth2ClusterName,
Endpoints: []*endpointv3.LocalityLbEndpoints{{
LbEndpoints: []*endpointv3.LbEndpoint{{
HostIdentifier: &endpointv3.LbEndpoint_Endpoint{
Endpoint: &endpointv3.Endpoint{
Address: &corev3.Address{
Address: &corev3.Address_SocketAddress{
SocketAddress: &corev3.SocketAddress{
Address: parsedTokenEndpoint.Hostname(),
PortSpecifier: &corev3.SocketAddress_PortValue{
PortValue: endpointPort,
},
Protocol: corev3.SocketAddress_TCP,
},
},
},
},
},
}},
}},
},
}
if s.Gateway.Spec.Envoy != nil && s.Gateway.Spec.Envoy.DisableIPv6 {
cluster.DnsLookupFamily = clusterv3.Cluster_V4_ONLY
}
if tls {
cluster.TransportSocket = &corev3.TransportSocket{
Name: "envoy.transport_sockets.tls",
ConfigType: &corev3.TransportSocket_TypedConfig{
TypedConfig: MustAny(&tlsv3.UpstreamTlsContext{
Sni: parsedTokenEndpoint.Hostname(),
AllowRenegotiation: true,
CommonTlsContext: &tlsv3.CommonTlsContext{
TlsParams: &tlsv3.TlsParameters{
TlsMinimumProtocolVersion: tlsv3.TlsParameters_TLSv1_2,
},
},
}),
},
}
}
return cluster, nil
}
func castResources[T types.Resource](from ...T) []types.Resource {
result := make([]types.Resource, len(from))
for idx := range from {
result[idx] = from[idx]
}
return result
}