diff --git a/.github/workflows/postgres.yml b/.github/workflows/postgres.yml index 5e1e994..70a03c1 100644 --- a/.github/workflows/postgres.yml +++ b/.github/workflows/postgres.yml @@ -13,7 +13,7 @@ on: - "v*" env: - MINOR_VERSIONS: '{"15": "12","17": "4"}' + MINOR_VERSIONS: '{"15":"12","17":"4"}' jobs: build: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 77cff7e..8639b77 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,9 +22,9 @@ jobs: - name: Login to container registry uses: docker/login-action@v3 with: - registry: code.icb4dc0.de - username: prskr - password: ${{ secrets.RELEASE_TOKEN }} + registry: registry.icb4dc0.de + username: ${{ secrets.HARBOR_USER }} + password: ${{ secrets.HARBOR_TOKEN }} - name: Setup Go uses: actions/setup-go@v5 diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 775fb4c..2bbb1b6 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -8,7 +8,7 @@ before: - go mod tidy - go run mage.go GenerateAll - mkdir -p out - - sh -c "cd config/release/default && kustomize edit set image supabase-operator=code.icb4dc0.de/prskr/supabase-operator:{{.Version}}" + - sh -c "cd config/release/default && kustomize edit set image supabase-operator=registry.icb4dc0.de/supabase-operator/controller:{{.Version}}" - sh -c "kustomize build config/release/default > out/operator_manifest.yaml" builds: @@ -28,7 +28,7 @@ kos: base_image: gcr.io/distroless/static:nonroot user: "65532:65532" repositories: - - code.icb4dc0.de/prskr/supabase-operator + - registry.icb4dc0.de/supabase-operator/controller platforms: - linux/amd64 - linux/arm64 diff --git a/.husky.toml b/.husky.toml index 0077416..e3069c5 100644 --- a/.husky.toml +++ b/.husky.toml @@ -3,7 +3,6 @@ # git hook pre commit pre-commit = [ "go mod tidy -go=1.24", - "go run mage.go GenerateAll", "husky lint-staged", # "golangci-lint run", ] diff --git a/.zed/tasks.json b/.zed/tasks.json new file mode 100644 index 0000000..8174f02 --- /dev/null +++ b/.zed/tasks.json @@ -0,0 +1,47 @@ +[ + { + "label": "Tilt Up", + "command": "tilt", + "args": ["up"], + "use_new_terminal": false, + "allow_concurrent_runs": false + }, + { + "label": "Generate CRDs", + "command": "go", + "args": ["run", "mage.go", "CRDs"], + "use_new_terminal": false, + "allow_concurrent_runs": false, + "tags": ["mage", "generate"] + }, + { + "label": "Generate CRD docs", + "command": "go", + "args": ["run", "mage.go", "CRDDocs"], + "use_new_terminal": false, + "allow_concurrent_runs": false, + "tags": ["mage", "generate"] + }, + { + "label": "Update image meta", + "command": "go", + "args": ["run", "mage.go", "FetchImageMeta"], + "use_new_terminal": false, + "allow_concurrent_runs": false, + "tags": ["mage"] + }, + { + "label": "Update DB migrations", + "command": "go", + "args": ["run", "mage.go", "FetchMigrations"], + "use_new_terminal": false, + "allow_concurrent_runs": false, + "tags": ["mage"] + }, + { + "label": "Run all Go tests", + "command": "go tool gotestsum ./...", + "use_new_terminal": false, + "allow_concurrent_runs": false + } +] diff --git a/Tiltfile b/Tiltfile index 6f29284..8001b4f 100644 --- a/Tiltfile +++ b/Tiltfile @@ -25,7 +25,7 @@ local_resource( k8s_kind('Cluster', api_version='postgresql.cnpg.io/v1') docker_build_with_restart( - 'supabase-operator', + 'controller', '.', entrypoint=['/app/bin/supabase-operator'], dockerfile='dev/Dockerfile', diff --git a/assets/migrations/init-scripts/00000000000000-initial-schema.sql b/assets/migrations/init-scripts/00000000000000-initial-schema.sql index ecce79a..3d71bb1 100644 --- a/assets/migrations/init-scripts/00000000000000-initial-schema.sql +++ b/assets/migrations/init-scripts/00000000000000-initial-schema.sql @@ -18,7 +18,15 @@ grant pg_read_all_data to supabase_read_only_user; create schema if not exists extensions; create extension if not exists "uuid-ossp" with schema extensions; create extension if not exists pgcrypto with schema extensions; -create extension if not exists pgjwt with schema extensions; +do $$ +begin + if exists (select 1 from pg_available_extensions where name = 'pgjwt') then + if not exists (select 1 from pg_extension where extname = 'pgjwt') then + create extension if not exists pgjwt with schema "extensions" cascade; + end if; + end if; +end $$; + -- Set up auth roles for the developer create role anon nologin noinherit; diff --git a/assets/migrations/migrations/20250205144616_move_orioledb_to_extensions_schema.sql b/assets/migrations/migrations/20250205144616_move_orioledb_to_extensions_schema.sql new file mode 100644 index 0000000..259a6b0 --- /dev/null +++ b/assets/migrations/migrations/20250205144616_move_orioledb_to_extensions_schema.sql @@ -0,0 +1,26 @@ +-- migrate:up +do $$ +declare + ext_schema text; + extensions_schema_exists boolean; +begin + -- check if the "extensions" schema exists + select exists ( + select 1 from pg_namespace where nspname = 'extensions' + ) into extensions_schema_exists; + + if extensions_schema_exists then + -- check if the "orioledb" extension is in the "public" schema + select nspname into ext_schema + from pg_extension e + join pg_namespace n on e.extnamespace = n.oid + where extname = 'orioledb'; + + if ext_schema = 'public' then + execute 'alter extension orioledb set schema extensions'; + end if; + end if; +end $$; + +-- migrate:down + diff --git a/assets/migrations/migrations/20250218031949_pgsodium_mask_role.sql b/assets/migrations/migrations/20250218031949_pgsodium_mask_role.sql new file mode 100644 index 0000000..f44fa98 --- /dev/null +++ b/assets/migrations/migrations/20250218031949_pgsodium_mask_role.sql @@ -0,0 +1,31 @@ +-- migrate:up + +DO $$ +BEGIN + IF EXISTS (SELECT FROM pg_extension WHERE extname = 'pgsodium') THEN + CREATE OR REPLACE FUNCTION pgsodium.mask_role(masked_role regrole, source_name text, view_name text) + RETURNS void + LANGUAGE plpgsql + SECURITY DEFINER + SET search_path TO '' + AS $function$ + BEGIN + EXECUTE format( + 'GRANT SELECT ON pgsodium.key TO %s', + masked_role); + + EXECUTE format( + 'GRANT pgsodium_keyiduser, pgsodium_keyholder TO %s', + masked_role); + + EXECUTE format( + 'GRANT ALL ON %I TO %s', + view_name, + masked_role); + RETURN; + END + $function$; + END IF; +END $$; + +-- migrate:down diff --git a/assets/migrations/migrations/20250220051611_pg_net_perms_fix.sql b/assets/migrations/migrations/20250220051611_pg_net_perms_fix.sql new file mode 100644 index 0000000..cc8ffc2 --- /dev/null +++ b/assets/migrations/migrations/20250220051611_pg_net_perms_fix.sql @@ -0,0 +1,64 @@ +-- migrate:up +CREATE OR REPLACE FUNCTION extensions.grant_pg_net_access() +RETURNS event_trigger +LANGUAGE plpgsql +AS $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_event_trigger_ddl_commands() AS ev + JOIN pg_extension AS ext + ON ev.objid = ext.oid + WHERE ext.extname = 'pg_net' + ) + THEN + IF NOT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = 'supabase_functions_admin' + ) + THEN + CREATE USER supabase_functions_admin NOINHERIT CREATEROLE LOGIN NOREPLICATION; + END IF; + + GRANT USAGE ON SCHEMA net TO supabase_functions_admin, postgres, anon, authenticated, service_role; + + IF EXISTS ( + SELECT FROM pg_extension + WHERE extname = 'pg_net' + -- all versions in use on existing projects as of 2025-02-20 + -- version 0.12.0 onwards don't need these applied + AND extversion IN ('0.2', '0.6', '0.7', '0.7.1', '0.8', '0.10.0', '0.11.0') + ) THEN + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; + + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; + + REVOKE ALL ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; + REVOKE ALL ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; + + GRANT EXECUTE ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; + GRANT EXECUTE ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; + END IF; + END IF; +END; +$$; + +DO $$ +BEGIN + IF EXISTS (SELECT FROM pg_extension WHERE extname = 'pg_net') + THEN + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY INVOKER; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY INVOKER; + + REVOKE EXECUTE ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) FROM supabase_functions_admin, postgres, anon, authenticated, service_role; + REVOKE EXECUTE ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) FROM supabase_functions_admin, postgres, anon, authenticated, service_role; + + GRANT ALL ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) TO PUBLIC; + GRANT ALL ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) TO PUBLIC; + END IF; +END $$; + +-- migrate:down diff --git a/config/control-plane/control-plane.yaml b/config/control-plane/control-plane.yaml index c55d2b7..912373a 100644 --- a/config/control-plane/control-plane.yaml +++ b/config/control-plane/control-plane.yaml @@ -25,7 +25,7 @@ spec: containers: - args: - control-plane - image: supabase-operator:latest + image: controller:latest name: control-plane env: - name: CONTROL_PLANE_NAMESPACE diff --git a/config/dev/cnpg-cluster.yaml b/config/dev/cnpg-cluster.yaml index e86bf9b..0738b5c 100644 --- a/config/dev/cnpg-cluster.yaml +++ b/config/dev/cnpg-cluster.yaml @@ -44,7 +44,7 @@ metadata: namespace: supabase-demo spec: instances: 1 - imageName: code.icb4dc0.de/prskr/supabase-operator/postgres:17.2 + imageName: registry.icb4dc0.de/supabase-operator/postgres:17.4 imagePullPolicy: Always postgresUID: 26 postgresGID: 102 diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 09b3ff4..144eebc 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -29,7 +29,7 @@ spec: - manager - --leader-elect - --health-probe-bind-address=:8081 - image: supabase-operator:latest + image: controller:latest name: manager env: - name: CONTROLLER_NAMESPACE diff --git a/config/release/default/kustomization.yaml b/config/release/default/kustomization.yaml index d5bad87..85e924a 100644 --- a/config/release/default/kustomization.yaml +++ b/config/release/default/kustomization.yaml @@ -2,9 +2,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: -- name: supabase-operator - newName: code.icb4dc0.de/prskr/supabase-operator - newTag: 0.1.0-SNAPSHOT-e6c7d68 + - name: controller + newName: registry.icb4dc0.de/supabase-operator/controller + newTag: 0.1.0-SNAPSHOT-e6c7d68 resources: -- ../../default + - ../../default diff --git a/hack/migrate.sh b/hack/migrate.sh deleted file mode 100755 index b6265b8..0000000 --- a/hack/migrate.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -export DATABASE_URL="postgres://supabase_admin:1n1t-R00t!@localhost:5432/app" - -go run mage.go Migrate diff --git a/internal/controller/common_test.go b/internal/controller/common_test.go new file mode 100644 index 0000000..7170611 --- /dev/null +++ b/internal/controller/common_test.go @@ -0,0 +1,16 @@ +package controller_test + +import ( + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + + supabasev1alpha1 "code.icb4dc0.de/prskr/supabase-operator/api/v1alpha1" +) + +var testingScheme = runtime.NewScheme() + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(testingScheme)) + utilruntime.Must(supabasev1alpha1.AddToScheme(testingScheme)) +} diff --git a/internal/controller/storage_s3_creds_controller_test.go b/internal/controller/storage_s3_creds_controller_test.go new file mode 100644 index 0000000..4e91b2f --- /dev/null +++ b/internal/controller/storage_s3_creds_controller_test.go @@ -0,0 +1,81 @@ +package controller_test + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + supabasev1alpha1 "code.icb4dc0.de/prskr/supabase-operator/api/v1alpha1" + "code.icb4dc0.de/prskr/supabase-operator/internal/controller" +) + +func TestStorageS3CredsController(t *testing.T) { + storage := new(supabasev1alpha1.Storage) + + MustReadObject(t, filepath.Join("testdata", "storage_basic.yaml"), storage) + + t.Log(string(MustMarshal(t, storage))) + + clientSet := fake.NewClientBuilder(). + WithObjects(storage). + WithScheme(testingScheme). + Build() + + log.SetLogger(zap.New(zap.UseDevMode(true))) + + controllerReconciler := &controller.StorageS3CredentialsReconciler{ + Client: clientSet, + Scheme: testingScheme, + } + + _, err := controllerReconciler.Reconcile(t.Context(), reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(storage), + }) + + assert.NoError(t, err) +} + +func MustMarshal(t testing.TB, obj runtime.Object) []byte { + t.Helper() + + raw, err := json.Marshal(obj) + if err != nil { + t.Errorf("failed to marshal object: %v", err) + t.Fail() + } + return raw +} + +func MustReadObject(t testing.TB, relativePath string, obj runtime.Object) { + t.Helper() + + gkvs, _, err := testingScheme.ObjectKinds(obj) + if err != nil { + t.Errorf("failed to determine group version kind: %v", err) + t.Fatal(err) + } + + decoder := scheme.Codecs.UniversalDecoder(testingScheme.PreferredVersionAllGroups()...) + + raw, err := os.ReadFile(relativePath) + if err != nil { + t.Errorf("failed to read file %s: %v", relativePath, err) + t.Fail() + } + + _, _, err = decoder.Decode(raw, &gkvs[0], obj) + if err != nil { + t.Errorf("failed to unmarshal file %s: %v", relativePath, err) + t.Fail() + } +} diff --git a/internal/controller/testdata/storage_basic.yaml b/internal/controller/testdata/storage_basic.yaml new file mode 100644 index 0000000..6b7ea3c --- /dev/null +++ b/internal/controller/testdata/storage_basic.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: supabase.k8s.icb4dc0.de/v1alpha1 +kind: Storage +metadata: + labels: + app.kubernetes.io/name: supabase-operator + app.kubernetes.io/managed-by: kustomize + name: storage-sample + namespace: supabase-demo +spec: + api: + s3: + credentialsSecretRef: + secretName: "storage-api-s3-credentials" + accessKeyIdKey: accessKeyId + accessSecretKeyKey: secretAccessKey + db: + host: cluster-example-rw.supabase-demo.svc + dbName: app + dbCredentialsRef: + # will be created by Core resource operator if not present + # just make sure the secret name is either based on the name of the core resource or explicitly set + # format <core-resource-name>-db-creds-supabase-storage-admin + secretName: core-sample-db-creds-supabase-storage-admin + jwtAuth: + # will be created by Core resource operator if not present + # just make sure the secret name is either based on the name of the core resource or explicitly set + secretName: core-sample-jwt diff --git a/internal/db/migrator.go b/internal/db/migrator.go index 62d8130..e5c4ea3 100644 --- a/internal/db/migrator.go +++ b/internal/db/migrator.go @@ -55,7 +55,8 @@ func (m Migrator) ApplyAll( logger.Info("Applying missing or outdated migration", "filename", s.FileName) err := status.Database.RecordMigrationCondition(s.FileName, s.Hash, m.Apply(ctx, s.Content)) if err != nil { - return false, err + logger.Error(err, "Failed to apply migrations", "filename", s.FileName) + continue } appliedSomething = true diff --git a/internal/supabase/studio.go b/internal/supabase/studio.go index 6e1991e..8f4123c 100644 --- a/internal/supabase/studio.go +++ b/internal/supabase/studio.go @@ -37,7 +37,7 @@ type studioDefaults struct { func studioServiceConfig() serviceConfig[studioEnvKeys, studioDefaults] { return serviceConfig[studioEnvKeys, studioDefaults]{ Name: "studio", - LivenessProbePath: "/api/profile", + LivenessProbePath: "/api/platform/profile", EnvKeys: studioEnvKeys{ PGMetaURL: "STUDIO_PG_META_URL", DBPassword: "POSTGRES_PASSWORD", @@ -47,7 +47,7 @@ func studioServiceConfig() serviceConfig[studioEnvKeys, studioDefaults] { AnonKey: "SUPABASE_ANON_KEY", ServiceKey: "SUPABASE_SERVICE_KEY", Host: fixedEnvOf("HOSTNAME", "0.0.0.0"), - LogsEnabled: fixedEnvOf("NEXT_PUBLIC_ENABLE_LOGS", "true"), + LogsEnabled: fixedEnvOf("NEXT_PUBLIC_ENABLE_LOGS", "false"), }, Defaults: studioDefaults{ NodeUID: 1000, diff --git a/magefiles/generate.go b/magefiles/generate.go index 9854c6f..b7bf0c8 100644 --- a/magefiles/generate.go +++ b/magefiles/generate.go @@ -44,6 +44,7 @@ const ( var ignoredMigrations = []string{ "10000000000000_demote-postgres.sql", + "20250312095419_pgbouncer_ownership.sql", } func GenerateAll(ctx context.Context) { @@ -147,7 +148,7 @@ func FetchImageMeta(ctx context.Context) (err error) { } } - latestEnvoyTag, err := latestReleaseVersion(ctx, "envoyproxy", "envoy") + latestEnvoyTag, err := latestReleaseVersion(ctx, "envoyproxy", "envoy", excludeDrafts, excludePreReleases) if err != nil { return err } @@ -169,11 +170,13 @@ func FetchImageMeta(ctx context.Context) (err error) { } func FetchMigrations(ctx context.Context) (err error) { - latestRelease, err := latestReleaseVersion(ctx, "supabase", "postgres") + latestRelease, err := latestReleaseVersion(ctx, "supabase", "postgres", excludeDrafts, excludePreReleases, matchesTagPattern(`15\..*`)) if err != nil { return err } + slog.InfoContext(ctx, "Extracting Postgres migrations for release", slog.String("release", latestRelease)) + releaseArtifactURL := fmt.Sprintf("https://github.com/supabase/postgres/archive/refs/tags/%s.tar.gz", latestRelease) req, err := http.NewRequestWithContext(ctx, http.MethodGet, releaseArtifactURL, nil) if err != nil { diff --git a/magefiles/github.go b/magefiles/github.go index d7bcc78..237ada7 100644 --- a/magefiles/github.go +++ b/magefiles/github.go @@ -21,12 +21,42 @@ import ( "encoding/json" "fmt" "net/http" + "regexp" "code.icb4dc0.de/prskr/supabase-operator/internal/errx" ) -func latestReleaseVersion(ctx context.Context, owner, repo string) (tagName string, err error) { - releaseURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/latest", owner, repo) +type releaseMatcher interface { + matchesRelease(r release) bool +} + +var ( + excludeDrafts releaseMatcher = releaseMatcherFunc(func(r release) bool { + return !r.Draft + }) + excludePreReleases releaseMatcher = releaseMatcherFunc(func(r release) bool { + return !r.PreRelease + }) +) + +func matchesTagPattern(pattern string) releaseMatcher { + compiled, err := regexp.Compile(pattern) + if err != nil { + panic(err) + } + return releaseMatcherFunc(func(r release) bool { + return compiled.MatchString(r.TagName) + }) +} + +type release struct { + TagName string `json:"tag_name"` + Draft bool `json:"draft"` + PreRelease bool `json:"prerelease"` +} + +func latestReleaseVersion(ctx context.Context, owner, repo string, matchers ...releaseMatcher) (tagName string, err error) { + releaseURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases", owner, repo) req, err := http.NewRequestWithContext(ctx, http.MethodGet, releaseURL, nil) if err != nil { return "", err @@ -43,13 +73,37 @@ func latestReleaseVersion(ctx context.Context, owner, repo string) (tagName stri return "", fmt.Errorf("failed to retrieve latest release: %s", resp.Status) } - var release struct { - TagName string `json:"tag_name"` - } + var ( + releases []release + matcher = multiMatcher(matchers) + ) - if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { + if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil { return "", err } - return release.TagName, nil + for _, release := range releases { + if matcher.matchesRelease(release) { + return release.TagName, nil + } + } + + return "", fmt.Errorf("no release found matching the criteria: %s/%s", owner, repo) +} + +type multiMatcher []releaseMatcher + +func (m multiMatcher) matchesRelease(r release) bool { + for _, matcher := range m { + if !matcher.matchesRelease(r) { + return false + } + } + return true +} + +type releaseMatcherFunc func(r release) bool + +func (f releaseMatcherFunc) matchesRelease(r release) bool { + return f(r) } diff --git a/testdata/dotnet-client/test/supabase-integration.api-test/ServiceKeyTest.cs b/testdata/dotnet-client/test/supabase-integration.api-test/ServiceKeyTest.cs index cb47f12..312b99a 100644 --- a/testdata/dotnet-client/test/supabase-integration.api-test/ServiceKeyTest.cs +++ b/testdata/dotnet-client/test/supabase-integration.api-test/ServiceKeyTest.cs @@ -21,7 +21,7 @@ public class TaskList : BaseModel [PrimaryKey("id")] public int Id { get; set; } [Column("user_id")] - public int UserId { get; set; } + public Guid UserId { get; set; } [Column("name")] public string Name { get; set; } -} \ No newline at end of file +}