refactor: implement control plane as controller-runtime manager
This commit is contained in:
parent
a5c170a478
commit
3104f50c58
67 changed files with 3693 additions and 261 deletions
internal
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
Copyright 2024 Peter Kurfer.
|
||||
Copyright 2025 Peter Kurfer.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
Copyright 2024 Peter Kurfer.
|
||||
Copyright 2025 Peter Kurfer.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
|
|
@ -221,8 +221,8 @@ func (r *CoreAuthReconciler) reconcileAuthService(
|
|||
core.Labels,
|
||||
)
|
||||
|
||||
if _, ok := authService.Labels[meta.SupabaseLabel.EnvoyCluster]; !ok {
|
||||
authService.Labels[meta.SupabaseLabel.EnvoyCluster] = core.Name
|
||||
if _, ok := authService.Labels[meta.SupabaseLabel.ApiGatewayTarget]; !ok {
|
||||
authService.Labels[meta.SupabaseLabel.ApiGatewayTarget] = core.Name
|
||||
}
|
||||
|
||||
authService.Spec = corev1.ServiceSpec{
|
||||
|
|
|
@ -228,8 +228,8 @@ func (r *CorePostgrestReconiler) reconcilePostgrestService(
|
|||
core.Labels,
|
||||
)
|
||||
|
||||
if _, ok := postgrestService.Labels[meta.SupabaseLabel.EnvoyCluster]; !ok {
|
||||
postgrestService.Labels[meta.SupabaseLabel.EnvoyCluster] = core.Name
|
||||
if _, ok := postgrestService.Labels[meta.SupabaseLabel.ApiGatewayTarget]; !ok {
|
||||
postgrestService.Labels[meta.SupabaseLabel.ApiGatewayTarget] = ""
|
||||
}
|
||||
|
||||
postgrestService.Spec = corev1.ServiceSpec{
|
||||
|
|
|
@ -196,8 +196,8 @@ func (r *DashboardPGMetaReconciler) reconcilePGMetaService(
|
|||
dashboard.Labels,
|
||||
)
|
||||
|
||||
if _, ok := pgMetaService.Labels[meta.SupabaseLabel.EnvoyCluster]; !ok {
|
||||
pgMetaService.Labels[meta.SupabaseLabel.EnvoyCluster] = dashboard.Name
|
||||
if _, ok := pgMetaService.Labels[meta.SupabaseLabel.ApiGatewayTarget]; !ok {
|
||||
pgMetaService.Labels[meta.SupabaseLabel.ApiGatewayTarget] = ""
|
||||
}
|
||||
|
||||
pgMetaService.Spec = corev1.ServiceSpec{
|
||||
|
|
|
@ -218,8 +218,8 @@ func (r *DashboardStudioReconciler) reconcileStudioService(
|
|||
dashboard.Labels,
|
||||
)
|
||||
|
||||
if _, ok := studioService.Labels[meta.SupabaseLabel.EnvoyCluster]; !ok {
|
||||
studioService.Labels[meta.SupabaseLabel.EnvoyCluster] = dashboard.Name
|
||||
if _, ok := studioService.Labels[meta.SupabaseLabel.ApiGatewayTarget]; !ok {
|
||||
studioService.Labels[meta.SupabaseLabel.ApiGatewayTarget] = ""
|
||||
}
|
||||
|
||||
studioService.Spec = corev1.ServiceSpec{
|
||||
|
|
|
@ -1,3 +1,19 @@
|
|||
// /*
|
||||
// Copyright 2025 Peter Kurfer.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// */
|
||||
|
||||
package controller
|
||||
|
||||
import (
|
||||
|
|
63
internal/controller/storage_controller.go
Normal file
63
internal/controller/storage_controller.go
Normal file
|
@ -0,0 +1,63 @@
|
|||
/*
|
||||
Copyright 2025 Peter Kurfer.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
supabasev1alpha1 "code.icb4dc0.de/prskr/supabase-operator/api/v1alpha1"
|
||||
)
|
||||
|
||||
// StorageReconciler reconciles a Storage object
|
||||
type StorageReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
}
|
||||
|
||||
// +kubebuilder:rbac:groups=supabase.k8s.icb4dc0.de,resources=storages,verbs=get;list;watch;create;update;patch;delete
|
||||
// +kubebuilder:rbac:groups=supabase.k8s.icb4dc0.de,resources=storages/status,verbs=get;update;patch
|
||||
// +kubebuilder:rbac:groups=supabase.k8s.icb4dc0.de,resources=storages/finalizers,verbs=update
|
||||
|
||||
// Reconcile is part of the main kubernetes reconciliation loop which aims to
|
||||
// move the current state of the cluster closer to the desired state.
|
||||
// TODO(user): Modify the Reconcile function to compare the state specified by
|
||||
// the Storage object against the actual cluster state, and then
|
||||
// perform operations to make the cluster state reflect the state specified by
|
||||
// the user.
|
||||
//
|
||||
// For more details, check Reconcile and its Result here:
|
||||
// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.4/pkg/reconcile
|
||||
func (r *StorageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
_ = log.FromContext(ctx)
|
||||
|
||||
// TODO(user): your logic here
|
||||
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
// SetupWithManager sets up the controller with the Manager.
|
||||
func (r *StorageReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(&supabasev1alpha1.Storage{}).
|
||||
Named("storage").
|
||||
Complete(r)
|
||||
}
|
84
internal/controller/storage_controller_test.go
Normal file
84
internal/controller/storage_controller_test.go
Normal file
|
@ -0,0 +1,84 @@
|
|||
/*
|
||||
Copyright 2025 Peter Kurfer.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
supabasev1alpha1 "code.icb4dc0.de/prskr/supabase-operator/api/v1alpha1"
|
||||
)
|
||||
|
||||
var _ = Describe("Storage Controller", func() {
|
||||
Context("When reconciling a resource", func() {
|
||||
const resourceName = "test-resource"
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
typeNamespacedName := types.NamespacedName{
|
||||
Name: resourceName,
|
||||
Namespace: "default", // TODO(user):Modify as needed
|
||||
}
|
||||
storage := &supabasev1alpha1.Storage{}
|
||||
|
||||
BeforeEach(func() {
|
||||
By("creating the custom resource for the Kind Storage")
|
||||
err := k8sClient.Get(ctx, typeNamespacedName, storage)
|
||||
if err != nil && errors.IsNotFound(err) {
|
||||
resource := &supabasev1alpha1.Storage{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: resourceName,
|
||||
Namespace: "default",
|
||||
},
|
||||
// TODO(user): Specify other spec details if needed.
|
||||
}
|
||||
Expect(k8sClient.Create(ctx, resource)).To(Succeed())
|
||||
}
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
// TODO(user): Cleanup logic after each test, like removing the resource instance.
|
||||
resource := &supabasev1alpha1.Storage{}
|
||||
err := k8sClient.Get(ctx, typeNamespacedName, resource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
By("Cleanup the specific resource instance Storage")
|
||||
Expect(k8sClient.Delete(ctx, resource)).To(Succeed())
|
||||
})
|
||||
It("should successfully reconcile the resource", func() {
|
||||
By("Reconciling the created resource")
|
||||
controllerReconciler := &StorageReconciler{
|
||||
Client: k8sClient,
|
||||
Scheme: k8sClient.Scheme(),
|
||||
}
|
||||
|
||||
_, err := controllerReconciler.Reconcile(ctx, reconcile.Request{
|
||||
NamespacedName: typeNamespacedName,
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
// TODO(user): Add more specific assertions depending on your controller's reconciliation logic.
|
||||
// Example: If you expect a certain status condition after reconciliation, verify it here.
|
||||
})
|
||||
})
|
||||
})
|
197
internal/controlplane/apigateway_controller.go
Normal file
197
internal/controlplane/apigateway_controller.go
Normal file
|
@ -0,0 +1,197 @@
|
|||
/*
|
||||
Copyright 2025 Peter Kurfer.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package controlplane
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
cachev3 "github.com/envoyproxy/go-control-plane/pkg/cache/v3"
|
||||
discoveryv1 "k8s.io/api/discovery/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/builder"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
"sigs.k8s.io/controller-runtime/pkg/handler"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
"sigs.k8s.io/controller-runtime/pkg/predicate"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
supabasev1alpha1 "code.icb4dc0.de/prskr/supabase-operator/api/v1alpha1"
|
||||
"code.icb4dc0.de/prskr/supabase-operator/internal/meta"
|
||||
)
|
||||
|
||||
// APIGatewayReconciler reconciles a APIGateway object
|
||||
type APIGatewayReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
Cache cachev3.SnapshotCache
|
||||
}
|
||||
|
||||
// Reconcile is part of the main kubernetes reconciliation loop which aims to
|
||||
// move the current state of the cluster closer to the desired state.
|
||||
//
|
||||
// For more details, check Reconcile and its Result here:
|
||||
// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/reconcile
|
||||
func (r *APIGatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) {
|
||||
var (
|
||||
gateway supabasev1alpha1.APIGateway
|
||||
logger = log.FromContext(ctx)
|
||||
endpointSliceList discoveryv1.EndpointSliceList
|
||||
)
|
||||
|
||||
logger.Info("Reconciling APIGateway")
|
||||
|
||||
if err := r.Get(ctx, req.NamespacedName, &gateway); client.IgnoreNotFound(err) != nil {
|
||||
logger.Error(err, "unable to fetch Gateway")
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
selector, err := metav1.LabelSelectorAsSelector(gateway.Spec.ServiceSelector)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("failed to create selector for EndpointSlices: %w", err)
|
||||
}
|
||||
|
||||
if err := r.List(ctx, &endpointSliceList, client.MatchingLabelsSelector{Selector: selector}); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
services := EnvoyServices{ServiceLabelKey: gateway.Spec.ComponentTypeLabel}
|
||||
services.UpsertEndpointSlices(endpointSliceList.Items...)
|
||||
|
||||
rawServices, err := json.Marshal(services)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("failed to prepare config hash: %w", err)
|
||||
}
|
||||
|
||||
serviceHash := sha256.New().Sum(rawServices)
|
||||
if bytes.Equal(serviceHash, gateway.Status.Envoy.ResourceHash) {
|
||||
logger.Info("Resource hash did not change - skipping reconciliation")
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
logger.Info("Updating service targets")
|
||||
_, err = controllerutil.CreateOrPatch(ctx, r.Client, &gateway, func() error {
|
||||
gateway.Status.ServiceTargets = services.Targets()
|
||||
gateway.Status.Envoy.ConfigVersion = strconv.FormatInt(time.Now().UTC().UnixMilli(), 10)
|
||||
gateway.Status.Envoy.ResourceHash = serviceHash
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
instance := fmt.Sprintf("%s:%s", gateway.Spec.Envoy.NodeName, gateway.Namespace)
|
||||
|
||||
logger.Info("Computing Envoy snapshot for current service targets", "version", gateway.Status.Envoy.ConfigVersion)
|
||||
snapshot, err := services.snapshot(ctx, instance, gateway.Status.Envoy.ConfigVersion)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("failed to prepare snapshot: %w", err)
|
||||
}
|
||||
|
||||
logger.Info("Propagating Envoy snapshot", "version", gateway.Status.Envoy.ConfigVersion)
|
||||
if err := r.Cache.SetSnapshot(ctx, instance, snapshot); err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("failed to propagate snapshot: %w", err)
|
||||
}
|
||||
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
func (r *APIGatewayReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
gatewayTargetLabelSelector, err := predicate.LabelSelectorPredicate(metav1.LabelSelector{
|
||||
MatchExpressions: []metav1.LabelSelectorRequirement{{
|
||||
Key: meta.SupabaseLabel.ApiGatewayTarget,
|
||||
Operator: metav1.LabelSelectorOpExists,
|
||||
}},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to build gateway target predicate: %w", err)
|
||||
}
|
||||
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(new(supabasev1alpha1.APIGateway)).
|
||||
Watches(
|
||||
new(discoveryv1.EndpointSlice),
|
||||
r.endpointSliceEventHandler(),
|
||||
builder.WithPredicates(gatewayTargetLabelSelector)).
|
||||
Complete(r)
|
||||
}
|
||||
|
||||
// endpointSliceEventHandler - prepares an event handler that checks whether the EndpointSlice has a specific target
|
||||
// or if it is targeting the only APIGateway in its namespace (default behavior for the operator)
|
||||
func (r *APIGatewayReconciler) endpointSliceEventHandler() handler.TypedEventHandler[client.Object, reconcile.Request] {
|
||||
return handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request {
|
||||
var (
|
||||
logger = log.FromContext(ctx)
|
||||
apiGatewayList supabasev1alpha1.APIGatewayList
|
||||
)
|
||||
|
||||
endpointSlice, ok := obj.(*discoveryv1.EndpointSlice)
|
||||
if !ok {
|
||||
logger.Info("Cannot map event to reconcile request, because object has unexpected type", "type", fmt.Sprintf("%T", obj))
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := r.Client.List(ctx, &apiGatewayList, client.InNamespace(endpointSlice.Namespace)); err != nil {
|
||||
logger.Error(err, "failed to list APIGateways to determine reconcile targets")
|
||||
return nil
|
||||
}
|
||||
|
||||
target, ok := endpointSlice.Labels[meta.SupabaseLabel.ApiGatewayTarget]
|
||||
if !ok {
|
||||
// should not happen, just to be sure
|
||||
return nil
|
||||
}
|
||||
|
||||
var reconcileRequests []reconcile.Request
|
||||
|
||||
if target != "" {
|
||||
for _, gw := range apiGatewayList.Items {
|
||||
if strings.EqualFold(gw.Spec.Envoy.NodeName, target) {
|
||||
reconcileRequests = append(reconcileRequests, reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{
|
||||
Name: gw.Name,
|
||||
Namespace: gw.Namespace,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
} else {
|
||||
reconcileRequests = make([]reconcile.Request, 0, len(apiGatewayList.Items))
|
||||
for _, gw := range apiGatewayList.Items {
|
||||
reconcileRequests = append(reconcileRequests, reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{
|
||||
Name: gw.Name,
|
||||
Namespace: gw.Namespace,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return reconcileRequests
|
||||
})
|
||||
}
|
|
@ -1,6 +1,10 @@
|
|||
package controlplane
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
|
||||
|
@ -10,11 +14,28 @@ import (
|
|||
discoveryv1 "k8s.io/api/discovery/v1"
|
||||
)
|
||||
|
||||
var _ json.Marshaler = (*ServiceCluster)(nil)
|
||||
|
||||
type ServiceCluster struct {
|
||||
ServiceEndpoints map[string]Endpoints
|
||||
}
|
||||
|
||||
func (c *ServiceCluster) AddOrUpdateEndpoints(eps *discoveryv1.EndpointSlice) {
|
||||
// MarshalJSON implements json.Marshaler.
|
||||
func (c *ServiceCluster) MarshalJSON() ([]byte, error) {
|
||||
tmp := struct {
|
||||
Endpoints []string `json:"endpoints"`
|
||||
}{}
|
||||
|
||||
for _, endpoints := range c.ServiceEndpoints {
|
||||
tmp.Endpoints = append(tmp.Endpoints, endpoints.Targets...)
|
||||
}
|
||||
|
||||
slices.Sort(tmp.Endpoints)
|
||||
|
||||
return json.Marshal(tmp)
|
||||
}
|
||||
|
||||
func (c *ServiceCluster) AddOrUpdateEndpoints(eps discoveryv1.EndpointSlice) {
|
||||
if c.ServiceEndpoints == nil {
|
||||
c.ServiceEndpoints = make(map[string]Endpoints)
|
||||
}
|
||||
|
@ -22,6 +43,16 @@ func (c *ServiceCluster) AddOrUpdateEndpoints(eps *discoveryv1.EndpointSlice) {
|
|||
c.ServiceEndpoints[eps.Name] = newEndpointsFromSlice(eps)
|
||||
}
|
||||
|
||||
func (c ServiceCluster) Targets() []string {
|
||||
var targets []string
|
||||
|
||||
for _, ep := range c.ServiceEndpoints {
|
||||
targets = append(targets, ep.Targets...)
|
||||
}
|
||||
|
||||
return targets
|
||||
}
|
||||
|
||||
func (c ServiceCluster) Cluster(name string, port uint32) *clusterv3.Cluster {
|
||||
return &clusterv3.Cluster{
|
||||
Name: name,
|
||||
|
@ -47,12 +78,13 @@ func (c ServiceCluster) endpoints(port uint32) []*endpointv3.LocalityLbEndpoints
|
|||
return eps
|
||||
}
|
||||
|
||||
func newEndpointsFromSlice(eps *discoveryv1.EndpointSlice) Endpoints {
|
||||
func newEndpointsFromSlice(eps discoveryv1.EndpointSlice) Endpoints {
|
||||
var result Endpoints
|
||||
|
||||
for _, ep := range eps.Endpoints {
|
||||
if ep.Conditions.Ready != nil && *ep.Conditions.Ready {
|
||||
result.Addresses = append(result.Addresses, ep.Addresses...)
|
||||
result.Targets = append(result.Targets, strings.ToLower(fmt.Sprintf("%s/%s", ep.TargetRef.Kind, ep.TargetRef.Name)))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -61,6 +93,7 @@ func newEndpointsFromSlice(eps *discoveryv1.EndpointSlice) Endpoints {
|
|||
|
||||
type Endpoints struct {
|
||||
Addresses []string
|
||||
Targets []string
|
||||
}
|
||||
|
||||
func (e Endpoints) LBEndpoints(port uint32) []*endpointv3.LbEndpoint {
|
||||
|
|
|
@ -1,29 +1,8 @@
|
|||
/*
|
||||
Copyright 2024 Peter Kurfer.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package controlplane
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
|
||||
listenerv3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
|
||||
|
@ -35,171 +14,64 @@ import (
|
|||
"github.com/envoyproxy/go-control-plane/pkg/resource/v3"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
discoveryv1 "k8s.io/api/discovery/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/selection"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
|
||||
"code.icb4dc0.de/prskr/supabase-operator/internal/meta"
|
||||
"code.icb4dc0.de/prskr/supabase-operator/internal/supabase"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrUnexpectedObject = errors.New("unexpected object")
|
||||
ErrNoEnvoyClusterLabel = errors.New("no Envoy cluster label set")
|
||||
|
||||
supabaseServices = []string{
|
||||
supabase.ServiceConfig.Postgrest.Name,
|
||||
supabase.ServiceConfig.Auth.Name,
|
||||
supabase.ServiceConfig.PGMeta.Name,
|
||||
}
|
||||
)
|
||||
|
||||
type EndpointsController struct {
|
||||
lock sync.Mutex
|
||||
Client client.WithWatch
|
||||
Cache cache.SnapshotCache
|
||||
envoyClusters map[string]*envoyClusterServices
|
||||
type EnvoyServices struct {
|
||||
ServiceLabelKey string `json:"-"`
|
||||
Postgrest *PostgrestCluster `json:"postgrest,omitempty"`
|
||||
GoTrue *GoTrueCluster `json:"auth,omitempty"`
|
||||
PGMeta *PGMetaCluster `json:"pgmeta,omitempty"`
|
||||
Studio *StudioCluster `json:"studio,omitempty"`
|
||||
}
|
||||
|
||||
func (c *EndpointsController) Run(ctx context.Context) error {
|
||||
var (
|
||||
logger = ctrl.Log.WithName("endpoints-controller")
|
||||
endpointSlices discoveryv1.EndpointSliceList
|
||||
)
|
||||
|
||||
selector := labels.NewSelector()
|
||||
|
||||
partOfRequirement, err := labels.NewRequirement(meta.WellKnownLabel.PartOf, selection.Equals, []string{"supabase"})
|
||||
if err != nil {
|
||||
return fmt.Errorf("preparing watcher selectors: %w", err)
|
||||
}
|
||||
|
||||
nameRequirement, err := labels.NewRequirement(meta.WellKnownLabel.Name, selection.In, supabaseServices)
|
||||
if err != nil {
|
||||
return fmt.Errorf("preparing watcher selectors: %w", err)
|
||||
}
|
||||
|
||||
envoyClusterRequirement, err := labels.NewRequirement(meta.SupabaseLabel.EnvoyCluster, selection.Exists, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("preparing watcher selectors: %w", err)
|
||||
}
|
||||
|
||||
selector.Add(*partOfRequirement, *nameRequirement, *envoyClusterRequirement)
|
||||
|
||||
watcher, err := c.Client.Watch(
|
||||
ctx,
|
||||
&endpointSlices,
|
||||
client.MatchingLabelsSelector{
|
||||
Selector: selector.Add(*partOfRequirement, *nameRequirement, *envoyClusterRequirement),
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer watcher.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case ev, more := <-watcher.ResultChan():
|
||||
if !more {
|
||||
return nil
|
||||
func (s *EnvoyServices) UpsertEndpointSlices(endpointSlices ...discoveryv1.EndpointSlice) {
|
||||
for _, eps := range endpointSlices {
|
||||
switch eps.Labels[s.ServiceLabelKey] {
|
||||
case supabase.ServiceConfig.Postgrest.Name:
|
||||
if s.Postgrest == nil {
|
||||
s.Postgrest = new(PostgrestCluster)
|
||||
}
|
||||
eventLogger := logger.WithValues("event_type", ev.Type)
|
||||
switch ev.Type {
|
||||
case watch.Added, watch.Modified:
|
||||
eps, ok := ev.Object.(*discoveryv1.EndpointSlice)
|
||||
if !ok {
|
||||
logger.Error(fmt.Errorf("%w: %T", ErrUnexpectedObject, ev.Object), "expected EndpointSlice but got a different object type")
|
||||
continue
|
||||
}
|
||||
|
||||
if err := c.handleModificationEvent(log.IntoContext(ctx, eventLogger), eps); err != nil {
|
||||
logger.Error(err, "error occurred during event handling")
|
||||
}
|
||||
s.Postgrest.AddOrUpdateEndpoints(eps)
|
||||
case supabase.ServiceConfig.Auth.Name:
|
||||
if s.GoTrue == nil {
|
||||
s.GoTrue = new(GoTrueCluster)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
s.GoTrue.AddOrUpdateEndpoints(eps)
|
||||
case supabase.ServiceConfig.PGMeta.Name:
|
||||
if s.PGMeta == nil {
|
||||
s.PGMeta = new(PGMetaCluster)
|
||||
}
|
||||
s.PGMeta.AddOrUpdateEndpoints(eps)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *EndpointsController) handleModificationEvent(ctx context.Context, epSlice *discoveryv1.EndpointSlice) error {
|
||||
c.lock.Lock()
|
||||
defer c.lock.Unlock()
|
||||
func (s EnvoyServices) Targets() map[string][]string {
|
||||
targets := make(map[string][]string)
|
||||
|
||||
var (
|
||||
logger = log.FromContext(ctx)
|
||||
instanceKey string
|
||||
svc *envoyClusterServices
|
||||
)
|
||||
|
||||
logger.Info("Observed endpoint slice", "name", epSlice.Name)
|
||||
|
||||
if c.envoyClusters == nil {
|
||||
c.envoyClusters = make(map[string]*envoyClusterServices)
|
||||
if s.Postgrest != nil {
|
||||
targets[supabase.ServiceConfig.Postgrest.Name] = s.Postgrest.Targets()
|
||||
}
|
||||
|
||||
envoyNodeName, ok := epSlice.Labels[meta.SupabaseLabel.EnvoyCluster]
|
||||
if !ok {
|
||||
return fmt.Errorf("%w: at object %s", ErrNoEnvoyClusterLabel, epSlice.Name)
|
||||
if s.GoTrue != nil {
|
||||
targets[supabase.ServiceConfig.Auth.Name] = s.GoTrue.Targets()
|
||||
}
|
||||
|
||||
instanceKey = fmt.Sprintf("%s:%s", envoyNodeName, epSlice.Namespace)
|
||||
|
||||
if svc, ok = c.envoyClusters[instanceKey]; !ok {
|
||||
svc = new(envoyClusterServices)
|
||||
if s.PGMeta != nil {
|
||||
targets[supabase.ServiceConfig.PGMeta.Name] = s.PGMeta.Targets()
|
||||
}
|
||||
|
||||
svc.UpsertEndpoints(epSlice)
|
||||
if s.Studio != nil {
|
||||
targets[supabase.ServiceConfig.Studio.Name] = s.Studio.Targets()
|
||||
}
|
||||
|
||||
c.envoyClusters[instanceKey] = svc
|
||||
|
||||
return c.updateSnapshot(ctx, instanceKey)
|
||||
return targets
|
||||
}
|
||||
|
||||
func (c *EndpointsController) updateSnapshot(ctx context.Context, instance string) error {
|
||||
latestVersion := strconv.FormatInt(time.Now().UTC().UnixMilli(), 10)
|
||||
|
||||
snapshot, err := c.envoyClusters[instance].snapshot(instance, latestVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return c.Cache.SetSnapshot(ctx, instance, snapshot)
|
||||
}
|
||||
|
||||
type envoyClusterServices struct {
|
||||
Postgrest *PostgrestCluster
|
||||
GoTrue *GoTrueCluster
|
||||
PGMeta *PGMetaCluster
|
||||
Studio *StudioCluster
|
||||
}
|
||||
|
||||
func (s *envoyClusterServices) UpsertEndpoints(eps *discoveryv1.EndpointSlice) {
|
||||
switch eps.Labels[meta.WellKnownLabel.Name] {
|
||||
case supabase.ServiceConfig.Postgrest.Name:
|
||||
if s.Postgrest == nil {
|
||||
s.Postgrest = new(PostgrestCluster)
|
||||
}
|
||||
s.Postgrest.AddOrUpdateEndpoints(eps)
|
||||
case supabase.ServiceConfig.Auth.Name:
|
||||
if s.GoTrue == nil {
|
||||
s.GoTrue = new(GoTrueCluster)
|
||||
}
|
||||
s.GoTrue.AddOrUpdateEndpoints(eps)
|
||||
case supabase.ServiceConfig.PGMeta.Name:
|
||||
if s.PGMeta == nil {
|
||||
s.PGMeta = new(PGMetaCluster)
|
||||
}
|
||||
s.PGMeta.AddOrUpdateEndpoints(eps)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *envoyClusterServices) snapshot(instance, version string) (*cache.Snapshot, error) {
|
||||
func (s *EnvoyServices) snapshot(ctx context.Context, instance, version string) (*cache.Snapshot, error) {
|
||||
const (
|
||||
apiRouteName = "supabase"
|
||||
studioRouteName = "supabas-studio"
|
||||
|
@ -207,6 +79,8 @@ func (s *envoyClusterServices) snapshot(instance, version string) (*cache.Snapsh
|
|||
listenerName = "supabase"
|
||||
)
|
||||
|
||||
logger := log.FromContext(ctx)
|
||||
|
||||
apiConnectionManager := &hcm.HttpConnectionManager{
|
||||
CodecType: hcm.HttpConnectionManager_AUTO,
|
||||
StatPrefix: "http",
|
||||
|
@ -327,6 +201,8 @@ func (s *envoyClusterServices) snapshot(instance, version string) (*cache.Snapsh
|
|||
}}
|
||||
|
||||
if s.Studio != nil {
|
||||
logger.Info("Adding studio listener")
|
||||
|
||||
listeners = append(listeners, &listenerv3.Listener{
|
||||
Name: "studio",
|
||||
Address: &corev3.Address{
|
|
@ -39,9 +39,9 @@ var WellKnownLabel = struct {
|
|||
}
|
||||
|
||||
var SupabaseLabel = struct {
|
||||
Reload string
|
||||
EnvoyCluster string
|
||||
Reload string
|
||||
ApiGatewayTarget string
|
||||
}{
|
||||
Reload: supabasev1alpha1.GroupVersion.Group + "/reload",
|
||||
EnvoyCluster: supabasev1alpha1.GroupVersion.Group + "/envoy-cluster",
|
||||
Reload: supabasev1alpha1.GroupVersion.Group + "/reload",
|
||||
ApiGatewayTarget: supabasev1alpha1.GroupVersion.Group + "/api-gateway-target",
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue