Compare commits

..

No commits in common. "main" and "feature/wasi-plugins" have entirely different histories.

49 changed files with 4561 additions and 836 deletions

4
.buildr/.gitignore vendored
View file

@ -1,4 +0,0 @@
bin/
logs/
out/
state.sqlite

View file

@ -1,5 +0,0 @@
buildr {
bin_dir = ".buildr/bin"
out_dir = ".buildr/out"
logs_dir = ".buildr/logs"
}

View file

@ -1,54 +0,0 @@
task "script" "buf_generate" {
inline = [
"buf generate --debug"
]
out_dir = repo.root
input_mapping = {
"api" = "api",
"buf.gen.yaml" = "buf.gen.yaml"
"buf.work.yaml" = "buf.work.yaml"
}
container {
image = "docker.io/bufbuild/buf"
}
depends_on = [
tools.vtprotobuf.id
]
}
task "script" "generate_examples_hello_world" {
working_dir = "examples/hello_world_go"
inline = [
"go generate -x ./..."
]
depends_on = [
tasks.buf_generate.id,
tools.mockery.id
]
}
task "script" "generate_sdk" {
inline = [
"go generate -x ./..."
]
depends_on = [
tasks.buf_generate.id,
]
}
task "script" "go_generate" {
inline = [
"/bin/true"
]
depends_on = [
tasks.generate_sdk.id,
tasks.generate_examples_hello_world.id,
]
}

View file

@ -1,30 +0,0 @@
locals {
tool_versions {
mockery = gh_latest_release("vektra", "mockery")
vtprotobuf = gh_latest_release("planetscale", "vtprotobuf")
}
}
tool "go_tool" "mockery" {
binary_name = "mockery"
repository = "github.com/vektra/mockery/v2"
version = local.tool_versions.mockery
build_args = [
"-v",
"-trimpath",
"-a",
"-installsuffix=cgo"
]
}
tool "go_tool" vtprotobuf {
binary_name = "protoc-gen-go-vtproto"
repository = "github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto"
version = local.tool_versions.vtprotobuf
build_args = [
"-v",
"-trimpath",
"-a",
"-installsuffix=cgo"
]
}

View file

@ -1,27 +0,0 @@
root = true
[*]
charset = utf-8
end_of_line = lf
indent_size = 4
tab_width = 4
indent_style = space
insert_final_newline = false
max_line_length = 120
trim_trailing_whitespace = true
[*.go]
indent_style = tab
ij_smart_tabs = true
ij_go_GROUP_CURRENT_PROJECT_IMPORTS = true
ij_go_group_stdlib_imports = true
ij_go_import_sorting = goimports
ij_go_local_group_mode = project
ij_go_move_all_imports_in_one_declaration = true
ij_go_move_all_stdlib_imports_in_one_group = true
ij_go_remove_redundant_import_aliases = true
[*.{yml,yaml}]
indent_size = 2
tab_width = 2
insert_final_newline = true

3
.gitignore vendored
View file

@ -19,6 +19,5 @@
# vendor/ # vendor/
# Go workspace file # Go workspace file
go.work* go.work
.idea/

37
api.go
View file

@ -2,21 +2,28 @@ package sdk
import ( import (
"context" "context"
"fmt"
"github.com/mailru/easyjson"
"golang.org/x/exp/slog"
"io" "io"
"log/slog"
"time" "time"
commonv1 "code.icb4dc0.de/buildr/api/generated/common/v1"
) )
type Category = commonv1.Category type Category string
func (t Category) String() string {
return string(t)
}
func (t Category) GroupName() string {
return fmt.Sprintf("%ss", t)
}
const ( const (
CategoryTool = commonv1.Category_CategoryTool CategoryTool Category = "tool"
CategoryTask = commonv1.Category_CategoryTask CategoryTask Category = "task"
CategoryBuild = commonv1.Category_CategoryBuild CategoryBuild Category = "build"
CategoryPackage = commonv1.Category_CategoryPackage CategoryPackage Category = "package"
CategoryRelease = commonv1.Category_CategoryRelease
) )
type StateMetadata struct { type StateMetadata struct {
@ -36,23 +43,13 @@ type ExecutionContext interface {
SetState(ctx context.Context, key string, value []byte) error SetState(ctx context.Context, key string, value []byte) error
} }
type TaskSpec[T Module] struct {
Module T
ModuleName string
Container *commonv1.ContainerSpec
OutputDir string
}
type Module interface { type Module interface {
easyjson.Unmarshaler
Execute(ctx ExecutionContext) error Execute(ctx ExecutionContext) error
Category() Category Category() Category
Type() string Type() string
} }
type Helper interface {
Help() Help
}
type BinaryNamer interface { type BinaryNamer interface {
BinaryName() string BinaryName() string
} }

11
api/README.md Normal file
View file

@ -0,0 +1,11 @@
# WASI host - module data exchange
The module SDK re-uses a subset of protobuf messages also used in the remote protocol except for the `TaskOutput` message because WASI handles `STDOUT`/`STDERR` already.
The protobuf messages are not used with gRPC but only as binary encoded messages shared via pointers in the WASI modules memory.
The following 'RPC' calls are supported:
- `log_msg` accepting a pointer and an offset to a `TaskLog` message - won't return anything
- `get_state` accepting a pointer and an offset to a `GetStateRequest` message, returning a 64-bit integer (32-bit pointer, 32-bit size) to a `GetStateResponse` message
- `set_state` accepting a pointer and an offset to a `SetState` message, returning a 64-bit integer (32-bit pointer, 32-bit size) to a `Result` message optionally containing error details

11
api/buf.yaml Normal file
View file

@ -0,0 +1,11 @@
version: v1
name: buf.build/buildr/buildr
breaking:
use:
- FILE
lint:
use:
- DEFAULT
except:
- PACKAGE_DIRECTORY_MATCH
allow_comment_ignores: true

72
api/rpc/v1/executor.proto Normal file
View file

@ -0,0 +1,72 @@
syntax = "proto3";
package buildr.rpc.v1;
message Buildr {
message Repo {
string root = 1;
}
message GitHub {
string api_token = 1;
}
Repo repo = 1;
GitHub github = 2;
}
message ModuleReference {
string module_category = 1;
string module_type = 2;
}
message TaskReference {
string id = 1;
string name = 2;
ModuleReference module = 3;
}
message StartTaskRequest {
TaskReference reference = 1;
Buildr buildr = 2;
bytes raw_task = 3;
}
message TaskResult {
string error = 1;
string modified_files_archive_path = 2;
}
message TaskLog {
message LogAttribute {
string key = 1;
string value = 2;
}
int64 time = 1;
string message = 2;
int32 level = 3;
repeated LogAttribute attributes = 4;
}
message SetState {
bytes key = 1;
bytes data = 2;
}
message GetStateRequest {
bytes key = 1;
}
message GetStateResponse {
bytes key = 1;
bytes data = 2;
}
message Result {
bool success = 1;
string error = 2;
}
message PluginInventory {
repeated ModuleReference modules = 1;
}

17
buf.gen.yaml Normal file
View file

@ -0,0 +1,17 @@
version: v1
managed:
enabled: true
go_package_prefix:
default: code.icb4dc0.de/buildr/wasi-module-sdk-go/internal
except:
- buf.build/googleapis/googleapis
plugins:
- plugin: buf.build/protocolbuffers/go:v1.30.0
out: ./protocol/generated/
opt: paths=source_relative
- plugin: go-vtproto
out: ./protocol/generated/
opt:
- features=marshal+unmarshal+size+pool
- paths=source_relative
revision: 1

3
buf.work.yaml Normal file
View file

@ -0,0 +1,3 @@
version: v1
directories:
- api/

4
buf.yaml Normal file
View file

@ -0,0 +1,4 @@
version: v1
name: buf.build/buildr/module-sdk-go
deps:
- buf.build/buildr/buildr

View file

@ -2,8 +2,9 @@ package sdk
import ( import (
"context" "context"
"crypto/md5"
"golang.org/x/exp/slog"
"io" "io"
"log/slog"
"os" "os"
) )
@ -12,12 +13,14 @@ var _ ExecutionContext = (*wasiExecutionContext)(nil)
func newWasiExecutionContext( func newWasiExecutionContext(
ctx context.Context, ctx context.Context,
logger *slog.Logger, logger *slog.Logger,
modName string,
mod Module, mod Module,
repoRoot, binDir, outDir string, repoRoot, binDir, outDir string,
) *wasiExecutionContext { ) *wasiExecutionContext {
return &wasiExecutionContext{ return &wasiExecutionContext{
Context: ctx, Context: ctx,
logger: logger, logger: logger,
modName: modName,
mod: mod, mod: mod,
repoRoot: repoRoot, repoRoot: repoRoot,
outDir: outDir, outDir: outDir,
@ -30,6 +33,7 @@ type wasiExecutionContext struct {
stateProxy StateProxy stateProxy StateProxy
logger *slog.Logger logger *slog.Logger
mod Module mod Module
modName string
repoRoot string repoRoot string
outDir string outDir string
binDir string binDir string
@ -60,9 +64,22 @@ func (w wasiExecutionContext) Logger() *slog.Logger {
} }
func (w wasiExecutionContext) GetState(_ context.Context, key string) ([]byte, StateMetadata, error) { func (w wasiExecutionContext) GetState(_ context.Context, key string) ([]byte, StateMetadata, error) {
return w.stateProxy.Get([]byte(key)) return w.stateProxy.Get(w.keyBytes(w.mod.Category().String(), w.modName, key))
} }
func (w wasiExecutionContext) SetState(_ context.Context, key string, value []byte) error { func (w wasiExecutionContext) SetState(_ context.Context, key string, value []byte) error {
return w.stateProxy.Set([]byte(key), value) return w.stateProxy.Set(w.keyBytes(w.mod.Category().String(), w.modName, key), value)
}
func (w wasiExecutionContext) keyBytes(parts ...string) []byte {
if len(parts) == 0 {
return nil
}
h := md5.New()
for i := range parts {
_, _ = h.Write([]byte(parts[i]))
}
return h.Sum(nil)
} }

View file

@ -2,147 +2,54 @@ package sdk
import ( import (
"context" "context"
"errors"
"log/slog"
commonv1 "code.icb4dc0.de/buildr/api/generated/common/v1" "github.com/mailru/easyjson"
remotev1 "code.icb4dc0.de/buildr/api/generated/remote/v1" _ "github.com/tetratelabs/tinymem"
wasiv1 "code.icb4dc0.de/buildr/api/generated/wasi/v1"
"code.icb4dc0.de/buildr/common/protocol" "code.icb4dc0.de/buildr/wasi-module-sdk-go/mem"
rpcv1 "code.icb4dc0.de/buildr/wasi-module-sdk-go/protocol/generated/rpc/v1"
) )
var ( var defaultRegistry = NewTypeRegistry()
defaultRegistry = NewTypeRegistry()
startTaskWrapper = FuncExportWrapper[*remotev1.StartTaskRequest, *wasiv1.StartTaskResponse](StartTask)
inventoryWrapper = FuncExportWrapper[*wasiv1.PluginInventoryRequest, *wasiv1.PluginInventoryResponse](GetInventory)
helpForWrapper = FuncExportWrapper[*wasiv1.HelpRequest, *wasiv1.HelpResponse](HelpForModule)
binaryNameWrapper = FuncExportWrapper[*wasiv1.BinaryNameRequest, *wasiv1.BinaryNameResponse](BinaryNameForModule)
)
func Register(cat Category, moduleName string, factory Factory) { func Register(cat Category, moduleName string, factory Factory) {
defaultRegistry.Add(cat, moduleName, factory) defaultRegistry.Add(cat, moduleName, factory)
} }
func GetInventory(*wasiv1.PluginInventoryRequest) (*wasiv1.PluginInventoryResponse, error) { //export inventory
var inventory wasiv1.PluginInventoryResponse func Inventory() uint64 {
var inventory rpcv1.PluginInventory
for _, t := range defaultRegistry.List() { for _, t := range defaultRegistry.List() {
m := defaultRegistry.Get(t.Category, t.Type) inventory.Modules = append(inventory.Modules, &rpcv1.ModuleReference{
spec, err := protocol.Marshal(m) ModuleCategory: t.Category.String(),
if err != nil {
panic(err)
}
data, err := spec.MarshalVT()
if err != nil {
panic(err)
}
inventory.Specs = append(inventory.Specs, &wasiv1.PluginInventoryResponse_InventorySpec{
ModuleRef: &commonv1.ModuleReference{
ModuleCategory: t.Category,
ModuleType: t.Type, ModuleType: t.Type,
},
EmptySpec: data,
}) })
} }
return &inventory, nil data, err := inventory.MarshalVT()
}
func StartTask(req *remotev1.StartTaskRequest) (*wasiv1.StartTaskResponse, error) {
executor := NewExecutor(req.Buildr.Repo.Root, req.Buildr.OutDir, req.Buildr.BinDir)
reference := req.GetReference().GetModule()
module := defaultRegistry.Get(reference.GetModuleCategory(), reference.GetModuleType())
if err := protocol.Unmarshal(req.GetSpec(), module); err != nil {
executor.logger.Error("Failed to unmarshal spec", slog.String("error", err.Error()))
return nil, err
}
var startTaskResponse wasiv1.StartTaskResponse
if err := executor.Run(context.Background(), module); err != nil {
startTaskResponse.Error = err.Error()
}
return &startTaskResponse, nil
}
func HelpForModule(helpRequest *wasiv1.HelpRequest) (*wasiv1.HelpResponse, error) {
module := defaultRegistry.Get(helpRequest.ModuleReference.ModuleCategory, helpRequest.ModuleReference.ModuleType)
if module == nil {
return nil, errors.New("unknown module")
}
helper, ok := module.(Helper)
if !ok {
return new(wasiv1.HelpResponse), nil
}
modHelp := helper.Help()
helpResponse := &wasiv1.HelpResponse{
Name: modHelp.Name,
Description: modHelp.Description,
Examples: make([]*wasiv1.TaskExample, 0, len(modHelp.Examples)),
}
for _, e := range modHelp.Examples {
modSpec, err := protocol.Marshal(e.Spec.Module)
if err != nil { if err != nil {
panic(err) panic(err)
} }
helpResponse.Examples = append(helpResponse.Examples, &wasiv1.TaskExample{
Name: e.Name, return mem.UnifyPtrSize(mem.DataToUnmanagedPtr(data))
Description: e.Description,
TaskSpec: &commonv1.TaskSpec{
ModuleName: e.Spec.ModuleName,
Container: e.Spec.Container,
OutputDir: e.Spec.OutputDir,
ModuleSpec: modSpec,
},
})
} }
return helpResponse, nil //export run
func Run(specPtr, specSize uint32) {
var startTask rpcv1.StartTaskRequest
if err := startTask.UnmarshalVT(mem.DataFromPtr(specPtr, specSize)); err != nil {
panic(err)
} }
func BinaryNameForModule(req *wasiv1.BinaryNameRequest) (*wasiv1.BinaryNameResponse, error) { executor := NewExecutor(startTask.Buildr.Repo.Root, "", "")
module := defaultRegistry.Get(req.ModuleReference.ModuleCategory, req.ModuleReference.ModuleType) reference := startTask.GetReference().GetModule()
if module == nil { module := defaultRegistry.Get(Category(reference.GetModuleCategory()), reference.GetModuleType())
return nil, errors.New("unknown module")
if err := easyjson.Unmarshal(startTask.RawTask, module); err != nil {
panic(err)
} }
namer, ok := module.(BinaryNamer) executor.Run(context.Background(), startTask.GetReference().GetName(), module)
if !ok {
return new(wasiv1.BinaryNameResponse), nil
}
if err := protocol.Unmarshal(req.GetSpec(), module); err != nil {
return nil, err
}
return &wasiv1.BinaryNameResponse{
Name: namer.BinaryName(),
}, nil
}
//export /buildr.rpc.v1.WasiExecutorService/PluginInventory
func Inventory(ptr, size uint32) uint64 {
return inventoryWrapper.Call(ptr, size)
}
//export /buildr.rpc.v1.WasiExecutorService/StartTask
func Run(ptr, size uint32) uint64 {
return startTaskWrapper.Call(ptr, size)
}
//export /buildr.rpc.v1.WasiExecutorService/Help
func HelpFor(ptr, size uint32) uint64 {
return helpForWrapper.Call(ptr, size)
}
//export /buildr.rpc.v1.WasiExecutorService/BinaryName
func BinaryNameFor(ptr, size uint32) uint64 {
return binaryNameWrapper.Call(ptr, size)
} }

View file

@ -1,3 +1,2 @@
*.wasm *.wasm
*_easyjson.go *_easyjson.go
*.mock.go

View file

@ -1,12 +0,0 @@
inpackage: true
with-expecter: true
keeptree: false
testonly: true
packages:
code.icb4dc0.de/buildr/buildr/modules:
interfaces:
ExecutionContext:
config:
dir: mocks/modules
filename: execution_context.mock.go

View file

@ -3,19 +3,14 @@ module hello_world
go 1.20 go 1.20
require ( require (
code.icb4dc0.de/buildr/wasi-module-sdk-go v0.0.0-20230701111906-1f0c58b1c8a4 github.com/mailru/easyjson v0.7.7
github.com/stretchr/testify v1.8.4 golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53
golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819
) )
require ( require (
github.com/davecgh/go-spew v1.1.1 // indirect github.com/google/uuid v1.3.0 // indirect
github.com/kr/pretty v0.3.1 // indirect github.com/josharian/intern v1.0.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/tetratelabs/tinymem v0.1.0 // indirect github.com/tetratelabs/tinymem v0.1.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect github.com/tetratelabs/wazero v1.1.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
) )

View file

@ -1,33 +1,19 @@
code.icb4dc0.de/buildr/wasi-module-sdk-go v0.0.0-20230701111906-1f0c58b1c8a4 h1:sDSOMWGtf/c+GGG+K6QR7sa7U+7PFJA5jzaQ2MgpE+8=
code.icb4dc0.de/buildr/wasi-module-sdk-go v0.0.0-20230701111906-1f0c58b1c8a4/go.mod h1:4oTtECbg97YmFN2UHoHj4D59Cgq8/GACMjWeRpcyX4o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/tetratelabs/tinymem v0.1.0 h1:Qza1JAg9lquPPJ/CIei5qQYx7t18KLie83O2WR6CM58= github.com/tetratelabs/tinymem v0.1.0 h1:Qza1JAg9lquPPJ/CIei5qQYx7t18KLie83O2WR6CM58=
github.com/tetratelabs/tinymem v0.1.0/go.mod h1:WFFTZFhLod6lTL+UetFAopVbGaB+KFsVcIY+RUv7NeY= github.com/tetratelabs/tinymem v0.1.0/go.mod h1:WFFTZFhLod6lTL+UetFAopVbGaB+KFsVcIY+RUv7NeY=
golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 h1:EDuYyU/MkFXllv9QF9819VlI9a4tzGuCbhG0ExK9o1U= github.com/tetratelabs/wazero v1.1.0 h1:EByoAhC+QcYpwSZJSs/aV0uokxPwBgKxfiokSUwAknQ=
github.com/tetratelabs/wazero v1.1.0/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ=
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o=
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View file

@ -1,52 +0,0 @@
//go:generate tinygo build -o hello_world.wasm -scheduler=none -gc=leaking --no-debug -target=wasi ../../main.go
package integration_test
import (
"context"
_ "embed"
"log/slog"
"os"
"testing"
"github.com/stretchr/testify/mock"
"code.icb4dc0.de/buildr/buildr/modules"
mm "hello_world/mocks/modules"
)
//go:embed hello_world.wasm
var payload []byte
func TestModule(t *testing.T) {
mod := plugin.Module{
PluginCategory: modules.CategoryTask,
PluginType: "hello_world",
PluginPayload: plugin.MemoryPayload(payload),
}.WithSpec(map[string]any{
"Name": "Ted",
})
testCtx := context.Background()
mockCtx := mm.NewMockExecutionContext(t)
mockCtx.EXPECT().Value(mock.Anything).RunAndReturn(func(key any) any {
return testCtx.Value(key)
})
mockCtx.EXPECT().Done().RunAndReturn(func() <-chan struct{} {
return testCtx.Done()
})
mockCtx.EXPECT().BinariesDir().Return(t.TempDir())
mockCtx.EXPECT().WorkingDir().Return(t.TempDir())
mockCtx.EXPECT().OutDir().Return(t.TempDir())
mockCtx.EXPECT().Name().Return("integration_test")
mockCtx.EXPECT().StdOut().Return(os.Stdout)
mockCtx.EXPECT().StdErr().Return(os.Stderr)
mockCtx.EXPECT().Logger().Return(slog.Default()).Maybe()
if err := mod.Execute(mockCtx); err != nil {
t.Fatal(err)
}
}

View file

@ -1,6 +1,6 @@
package main package main
//go:generate mockery //go:generate tinygo build -o hello_world.wasm -scheduler=none -gc=leaking --no-debug -target=wasi main.go
import ( import (
sdk "code.icb4dc0.de/buildr/wasi-module-sdk-go" sdk "code.icb4dc0.de/buildr/wasi-module-sdk-go"

View file

@ -0,0 +1,31 @@
package main_test
import (
sdk "code.icb4dc0.de/buildr/wasi-module-sdk-go"
"code.icb4dc0.de/buildr/wasi-module-sdk-go/integration"
"context"
_ "embed"
"golang.org/x/exp/slog"
"testing"
)
//go:embed hello_world.wasm
var payload []byte
func TestModule(t *testing.T) {
h := integration.NewHost(
slog.New(slog.NewTextHandler(integration.NewTestWriter(t))),
integration.WithState(integration.StateKey(sdk.CategoryTask, "test", "hello"), []byte("world")),
)
s := integration.TestSpec{
ModuleCategory: sdk.CategoryTask,
ModuleType: "hello_world",
ModuleName: "test",
RawTaskSpec: []byte(`{"Name": "Ted"}`),
}
if err := h.Run(context.Background(), payload, s); err != nil {
t.Errorf("Failed to run module: %v", err)
}
}

View file

@ -1,20 +1,13 @@
package module package module
//go:generate go run -mod=mod github.com/mailru/easyjson/easyjson -all hello_world.go
import ( import (
"fmt"
"log/slog"
"os"
rpcv1 "code.icb4dc0.de/buildr/api/generated/rpc/v1"
"code.icb4dc0.de/buildr/wasi-module-sdk-go/exec"
sdk "code.icb4dc0.de/buildr/wasi-module-sdk-go" sdk "code.icb4dc0.de/buildr/wasi-module-sdk-go"
"golang.org/x/exp/slog"
) )
var ( var _ sdk.Module = (*HelloWorld)(nil)
_ sdk.Module = (*HelloWorld)(nil)
_ sdk.Helper = (*HelloWorld)(nil)
)
type HelloWorld struct { type HelloWorld struct {
Name string Name string
@ -23,28 +16,18 @@ type HelloWorld struct {
func (h HelloWorld) Execute(ctx sdk.ExecutionContext) error { func (h HelloWorld) Execute(ctx sdk.ExecutionContext) error {
logger := ctx.Logger() logger := ctx.Logger()
logger.Info("Executing hello world", slog.String("name", h.Name)) logger.Info("Executing hello world")
if f, err := os.CreateTemp(ctx.OutDir(), "hello_world.*.txt"); err != nil { val, _, err := ctx.GetState(ctx, "hello")
return err if err != nil {
} else if err = f.Close(); err != nil {
return err return err
} }
if foundPath, err := exec.LookPath("go"); err != nil { if err := ctx.SetState(ctx, "state", []byte(`{"hello":"world"}`)); err != nil {
return err
} else {
logger.Info("found path for go", slog.String("path", foundPath))
}
cmd := exec.NewCommand("/bin/bash", "-c", `set -ex; echo "Hello process execution!"`)
if err := cmd.Run(); err != nil {
return err return err
} }
fmt.Println("Hello world") logger.Info("Got value from state", slog.String("value", string(val)))
_, _ = fmt.Fprint(ctx.StdOut(), "Hello world via pipeline")
return nil return nil
} }
@ -56,35 +39,3 @@ func (HelloWorld) Category() sdk.Category {
func (HelloWorld) Type() string { func (HelloWorld) Type() string {
return "hello_world" return "hello_world"
} }
func (h HelloWorld) Help() sdk.Help {
return sdk.Help{
Name: "Hello World",
Description: `Example to illustrate how to use the Buildr plugin API.`,
Examples: []sdk.Example{
{
Name: "Simple example",
Description: `well, you know, hello world`,
Spec: sdk.TaskSpec[sdk.Module]{
ModuleName: "hello_world",
Module: HelloWorld{
Name: "Ted Tester",
},
},
},
{
Name: "Container example",
Description: `well, you know, hello world, but in a container!`,
Spec: sdk.TaskSpec[sdk.Module]{
ModuleName: "hello_world",
Module: HelloWorld{
Name: "Paul Player",
},
Container: &rpcv1.ContainerSpec{
Image: "busybox",
},
},
},
},
}
}

View file

@ -1,87 +0,0 @@
package exec
import (
"fmt"
"io"
wasiv1 "code.icb4dc0.de/buildr/api/generated/wasi/v1"
"code.icb4dc0.de/buildr/wasi-module-sdk-go/mem"
)
func NewCommand(name string, args ...string) *Command {
return &Command{
Name: name,
Args: args,
}
}
type Command struct {
Name string
Args []string
WorkingDir string
Env map[string]string
StdIn io.Reader
}
func (c *Command) AddEnv(envToAdd map[string]string) {
if envToAdd == nil {
return
}
if c.Env == nil {
c.Env = make(map[string]string)
}
for k, v := range envToAdd {
c.Env[k] = v
}
}
func (c *Command) Run() error {
execReq := &wasiv1.ProcessStartRequest{
Command: c.Name,
Args: c.Args,
WorkingDirectory: c.WorkingDir,
Environment: c.Env,
}
if c.StdIn != nil {
if stdInData, err := io.ReadAll(c.StdIn); err != nil {
return err
} else {
execReq.Stdin = stdInData
}
}
data, err := execReq.MarshalVT()
if err != nil {
return err
}
result := _exec(mem.DataToManagedPtr(data))
if result == 0 {
return fmt.Errorf("failed to execute command: %s", c.Name)
}
resultPtr := uint32(result >> 32)
resultSize := uint32(result)
if resultSize == 0 {
return nil
}
execResp := new(wasiv1.ProcessStartResponse)
if err := execResp.UnmarshalVT(mem.PtrToData(resultPtr, resultSize)); err != nil {
return err
}
if execResp.ExitCode != 0 {
return &Error{
ExitCode: int(execResp.ExitCode),
Message: execResp.Error,
StdErr: execResp.Stderr,
}
}
return nil
}

View file

@ -1,15 +0,0 @@
package exec
import "fmt"
var _ error = (*Error)(nil)
type Error struct {
ExitCode int
Message string
StdErr []byte
}
func (e Error) Error() string {
return fmt.Sprintf("command exited with code %d: %s - %s", e.ExitCode, e.Message, e.StdErr)
}

View file

@ -1,7 +0,0 @@
//go:build !wasi
package exec
func _exec(ptr, size uint32) (ptrSize uint64) { return 0 }
func _lookPath(ptr, size uint32) (ptrSize uint64) { return 0 }

View file

@ -1,11 +0,0 @@
//go:build wasi
package exec
//go:wasm-module buildr
//go:wasmimport buildr exec
func _exec(ptr, size uint32) (ptrSize uint64)
//go:wasm-module buildr
//go:wasmimport buildr lookPath
func _lookPath(ptr, size uint32) (ptrSize uint64)

View file

@ -1,43 +0,0 @@
package exec
import (
"errors"
"fmt"
wasiv1 "code.icb4dc0.de/buildr/api/generated/wasi/v1"
"code.icb4dc0.de/buildr/wasi-module-sdk-go/mem"
)
func LookPath(file string) (string, error) {
lookupPathReq := &wasiv1.LookupPathRequest{
Command: file,
}
data, err := lookupPathReq.MarshalVT()
if err != nil {
return "", err
}
result := _lookPath(mem.DataToManagedPtr(data))
if result == 0 {
return "", fmt.Errorf("failed to lookup in path: %s", file)
}
resultPtr := uint32(result >> 32)
resultSize := uint32(result)
if resultSize == 0 {
return "", fmt.Errorf("failed to lookup in path: %s", file)
}
lookupPathResp := new(wasiv1.LookupPathResponse)
if err := lookupPathResp.UnmarshalVT(mem.PtrToData(resultPtr, resultSize)); err != nil {
return "", err
}
if len(lookupPathResp.Error) > 0 {
return "", errors.New(lookupPathResp.Error)
}
return lookupPathResp.Path, nil
}

View file

@ -2,7 +2,7 @@ package sdk
import ( import (
"context" "context"
"log/slog" "golang.org/x/exp/slog"
) )
func NewExecutor(repoRoot, outDir, binDir string) Executor { func NewExecutor(repoRoot, outDir, binDir string) Executor {
@ -21,7 +21,9 @@ type Executor struct {
binDir string binDir string
} }
func (e Executor) Run(ctx context.Context, m Module) error { func (e Executor) Run(ctx context.Context, modName string, m Module) {
execCtx := newWasiExecutionContext(ctx, e.logger, m, e.repoRoot, e.binDir, e.outDir) execCtx := newWasiExecutionContext(ctx, e.logger, modName, m, e.repoRoot, e.binDir, e.outDir)
return m.Execute(execCtx) if err := m.Execute(execCtx); err != nil {
e.logger.Error("Failed to execute module", slog.String("err", err.Error()))
}
} }

View file

@ -1,31 +0,0 @@
package sdk
import (
"reflect"
bapi "code.icb4dc0.de/buildr/api"
"code.icb4dc0.de/buildr/wasi-module-sdk-go/mem"
)
type FuncExportWrapper[TRequest bapi.ProtoMessage, TResponse bapi.ProtoMessage] func(req TRequest) (TResponse, error)
func (w FuncExportWrapper[TRequest, TResponse]) Call(reqPtr, reqSize uint32) uint64 {
var req TRequest
if reflect.TypeOf(req).Kind() == reflect.Ptr {
req = reflect.New(reflect.TypeOf(req).Elem()).Interface().(TRequest)
}
if err := req.UnmarshalVT(mem.DataFromPtr(reqPtr, reqSize)); err != nil {
panic(err)
}
if resp, err := w(req); err != nil {
panic(err)
} else if data, err := resp.MarshalVT(); err != nil {
panic(err)
} else if len(data) == 0 {
return 0
} else {
return mem.UnifyPtrSize(mem.DataToUnmanagedPtr(data))
}
}

10
go.mod
View file

@ -1,13 +1,15 @@
module code.icb4dc0.de/buildr/wasi-module-sdk-go module code.icb4dc0.de/buildr/wasi-module-sdk-go
go 1.21 go 1.20
require ( require (
code.icb4dc0.de/buildr/api v0.0.0-20230912160519-4b705a6732bc github.com/mailru/easyjson v0.7.7
code.icb4dc0.de/buildr/common v0.0.0-20230912160755-da17cbfde028 github.com/tetratelabs/tinymem v0.1.0
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53
google.golang.org/protobuf v1.30.0
) )
require ( require (
github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-cmp v0.5.9 // indirect
google.golang.org/protobuf v1.31.0 // indirect github.com/josharian/intern v1.0.0 // indirect
) )

20
go.sum
View file

@ -1,16 +1,16 @@
code.icb4dc0.de/buildr/api v0.0.0-20230905195458-e4d230e5c7fd h1:N4WkVoqphjFXfFwfKpLcEtS5lBCfd9NVytUutiszQ90=
code.icb4dc0.de/buildr/api v0.0.0-20230905195458-e4d230e5c7fd/go.mod h1:eWSjeX25XbbNGKlxVoOf2a8V6xOCIaQh5W65T7nNcL8=
code.icb4dc0.de/buildr/api v0.0.0-20230912160519-4b705a6732bc h1:NpOz5K0Oo/WFWBRsBKf92MTX9iW+sqaImwMuU33z6/s=
code.icb4dc0.de/buildr/api v0.0.0-20230912160519-4b705a6732bc/go.mod h1:eWSjeX25XbbNGKlxVoOf2a8V6xOCIaQh5W65T7nNcL8=
code.icb4dc0.de/buildr/common v0.0.0-20230905195627-a3d5bb4f1ee7 h1:sqBO2IFI0Gs+FiM4kkQjj7QkC0p+AhJKqhCQqCEwlM4=
code.icb4dc0.de/buildr/common v0.0.0-20230905195627-a3d5bb4f1ee7/go.mod h1:biIRy/mBiVPAR/okWN5DFmB3Hnnxt2FP1Qswa4GBMeE=
code.icb4dc0.de/buildr/common v0.0.0-20230912160755-da17cbfde028 h1:12xr1rT8faxqY7QqI7Z4nlKw1UdQZNfAtv4FybTPtwk=
code.icb4dc0.de/buildr/common v0.0.0-20230912160755-da17cbfde028/go.mod h1:GMFttKFr14bCsjGHbq1tUCOuru7AqBkELvsk2Xhn0Bw=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/tetratelabs/tinymem v0.1.0 h1:Qza1JAg9lquPPJ/CIei5qQYx7t18KLie83O2WR6CM58=
github.com/tetratelabs/tinymem v0.1.0/go.mod h1:WFFTZFhLod6lTL+UetFAopVbGaB+KFsVcIY+RUv7NeY=
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o=
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=

13
help.go
View file

@ -1,13 +0,0 @@
package sdk
type Example struct {
Spec TaskSpec[Module]
Name string
Description string
}
type Help struct {
Name string
Description string
Examples []Example
}

View file

@ -2,7 +2,7 @@
package sdk package sdk
func _log_msg(ptr, size uint32) (ptrSize uint64) { return 0 } func _log_msg(ptr, size uint32) {}
func _get_state(ptr, size uint32) (ptrSize uint64) { return 0 } func _get_state(ptr, size uint32) (ptrSize uint64) { return 0 }

View file

@ -3,13 +3,13 @@
package sdk package sdk
//go:wasm-module buildr //go:wasm-module buildr
//go:wasmimport buildr log_msg //export log_msg
func _log_msg(ptr, size uint32) (ptrSize uint64) func _log_msg(ptr, size uint32)
//go:wasm-module buildr //go:wasm-module buildr
//go:wasmimport buildr get_state //export get_state
func _get_state(ptr, size uint32) (ptrSize uint64) func _get_state(ptr, size uint32) (ptrSize uint64)
//go:wasm-module buildr //go:wasm-module buildr
//go:wasmimport buildr set_state //export set_state
func _set_state(ptr, size uint32) (ptrSize uint64) func _set_state(ptr, size uint32) (ptrSize uint64)

13
integration/README.md Normal file
View file

@ -0,0 +1,13 @@
# Integration module
The integration module is meant as a helper to blackbox test a WASI module.
It's a shortcut to instantiate and execute a module in the context of buildr.
The workflow basically looks like this:
1. Compile module e.g. with tinygo to `*.wasm` file
1. Create a `integration.Host` (with `integration.NewHost(...)` function)
1. Specify the scenario (category, type, ...)
1. Execute the module
A very basic example can be found in the [`hello_world_go` example](../examples/hello_world_go/main_test.go)

16
integration/go.mod Normal file
View file

@ -0,0 +1,16 @@
module code.icb4dc0.de/buildr/wasi-module-sdk-go/integration
go 1.20
require (
github.com/google/uuid v1.3.0
github.com/tetratelabs/wazero v1.1.0
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53
)
require (
github.com/josharian/intern v1.0.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/tetratelabs/tinymem v0.1.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect
)

19
integration/go.sum Normal file
View file

@ -0,0 +1,19 @@
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/tetratelabs/tinymem v0.1.0 h1:Qza1JAg9lquPPJ/CIei5qQYx7t18KLie83O2WR6CM58=
github.com/tetratelabs/tinymem v0.1.0/go.mod h1:WFFTZFhLod6lTL+UetFAopVbGaB+KFsVcIY+RUv7NeY=
github.com/tetratelabs/wazero v1.1.0 h1:EByoAhC+QcYpwSZJSs/aV0uokxPwBgKxfiokSUwAknQ=
github.com/tetratelabs/wazero v1.1.0/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ=
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o=
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=

23
integration/logging.go Normal file
View file

@ -0,0 +1,23 @@
package integration
import (
"io"
"testing"
)
var _ io.Writer = (*TestWriter)(nil)
func NewTestWriter(tb testing.TB) TestWriter {
return TestWriter{
TB: tb,
}
}
type TestWriter struct {
TB testing.TB
}
func (t TestWriter) Write(p []byte) (n int, err error) {
t.TB.Log(string(p))
return len(p), nil
}

80
integration/mem_mgr.go Normal file
View file

@ -0,0 +1,80 @@
package integration
import (
"context"
"errors"
"github.com/tetratelabs/wazero/api"
)
type Message interface {
MarshalVT() (dAtA []byte, err error)
UnmarshalVT(dAtA []byte) error
}
func newMemoryManager(mod api.Module) *memoryManager {
return &memoryManager{
allocate: mod.ExportedFunction("malloc"),
deallocate: mod.ExportedFunction("free"),
}
}
type memoryManager struct {
allocate api.Function
deallocate api.Function
danglingAllocations []uint64
}
func (m *memoryManager) Deallocate(ctx context.Context, ptr uint64) error {
_, err := m.deallocate.Call(ctx, ptr)
return err
}
func (m *memoryManager) WriteMessage(ctx context.Context, mod api.Module, msg Message) (uint64, error) {
data, err := msg.MarshalVT()
if err != nil {
return 0, err
}
ptr, err := m.Allocate(ctx, uint64(len(data)))
if err != nil {
return 0, err
}
if !mod.Memory().Write(uint32(ptr), data) {
return 0, errors.New("failed to write message to memory")
}
return (ptr << uint64(32)) | uint64(len(data)), nil
}
func (m *memoryManager) Allocate(ctx context.Context, size uint64) (ptr uint64, err error) {
results, err := m.allocate.Call(ctx, size)
if err != nil {
return 0, err
}
m.danglingAllocations = append(m.danglingAllocations, results[0])
return results[0], nil
}
func (m *memoryManager) WithMem(ctx context.Context, size uint64, delegate func(ptr uint64) error) error {
results, err := m.allocate.Call(ctx, size)
if err != nil {
return err
}
defer m.deallocate.Call(ctx, results[0])
return delegate(results[0])
}
func (m *memoryManager) Close() (err error) {
ctx := context.Background()
for i := range m.danglingAllocations {
_, e := m.deallocate.Call(ctx, m.danglingAllocations[i])
err = errors.Join(err, e)
}
return err
}

276
integration/test_host.go Normal file
View file

@ -0,0 +1,276 @@
package integration
import (
"context"
"crypto/md5"
"errors"
"fmt"
"os"
"time"
"code.icb4dc0.de/buildr/wasi-module-sdk-go/mem"
"github.com/google/uuid"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
"golang.org/x/exp/slog"
sdk "code.icb4dc0.de/buildr/wasi-module-sdk-go"
rpcv1 "code.icb4dc0.de/buildr/wasi-module-sdk-go/protocol/generated/rpc/v1"
)
func StateKey(cat sdk.Category, modName, key string) string {
h := md5.New()
_, _ = h.Write([]byte(cat.String()))
_, _ = h.Write([]byte(modName))
_, _ = h.Write([]byte(key))
return string(h.Sum(nil))
}
type TestSpec struct {
ModuleCategory sdk.Category
ModuleType string
ModuleName string
RawTaskSpec []byte
}
type HostOption func(h *Host)
func WithState(key string, value []byte) HostOption {
return func(h *Host) {
h.state[key] = value
}
}
func NewHost(logger *slog.Logger, opts ...HostOption) *Host {
h := &Host{
Logger: logger,
state: make(map[string][]byte),
}
for i := range opts {
opts[i](h)
}
return h
}
type Host struct {
memMgr *memoryManager
state map[string][]byte
Logger *slog.Logger
}
func (h *Host) Run(ctx context.Context, wasiPayload []byte, spec TestSpec) (err error) {
runtimeConfig := wazero.NewRuntimeConfig().
WithCloseOnContextDone(true)
r := wazero.NewRuntimeWithConfig(ctx, runtimeConfig)
defer func() {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
err = errors.Join(err, r.Close(ctx))
}()
_, err = r.NewHostModuleBuilder("buildr").
NewFunctionBuilder().WithFunc(h.log).Export("log_msg").
NewFunctionBuilder().WithFunc(h.getState).Export("get_state").
NewFunctionBuilder().WithFunc(h.setState).Export("set_state").
Instantiate(ctx)
if err != nil {
return err
}
closer, err := wasi_snapshot_preview1.Instantiate(ctx, r)
if err != nil {
return err
}
defer func() {
err = errors.Join(err, closer.Close(context.Background()))
}()
config := wazero.NewFSConfig().
WithDirMount(".", "/work")
moduleConfig := wazero.NewModuleConfig().
WithStdout(os.Stdout).
WithStderr(os.Stderr).
WithFSConfig(config)
mod, err := r.InstantiateWithConfig(ctx, wasiPayload, moduleConfig)
if err != nil {
return err
}
h.memMgr = newMemoryManager(mod)
inv, err := h.getInventory(ctx, mod)
if err != nil {
return err
}
for _, ref := range inv {
fmt.Printf("%s/%s/n", ref.Category, ref.Type)
}
startTask := &rpcv1.StartTaskRequest{
Reference: &rpcv1.TaskReference{
Id: uuid.NewString(),
Module: &rpcv1.ModuleReference{
ModuleCategory: spec.ModuleCategory.String(),
ModuleType: spec.ModuleType,
},
Name: spec.ModuleName,
},
Buildr: &rpcv1.Buildr{
Repo: &rpcv1.Buildr_Repo{
Root: "/work",
},
},
RawTask: spec.RawTaskSpec,
}
data, err := startTask.MarshalVT()
if err != nil {
return err
}
run := mod.ExportedFunction("run")
defer func() {
err = errors.Join(err, h.memMgr.Close())
}()
err = h.memMgr.WithMem(ctx, uint64(len(data)), func(ptr uint64) error {
if !mod.Memory().Write(uint32(ptr), data) {
return errors.New("failed to write to memory")
}
_, err = run.Call(ctx, ptr, uint64(len(data)))
return err
})
return err
}
func (h *Host) getInventory(ctx context.Context, mod api.Module) (refs []sdk.Reference, err error) {
result, err := mod.ExportedFunction("inventory").Call(ctx)
if err != nil {
return nil, err
}
ptr, size := mem.UnifiedPtrToSizePtr(result[0])
if ptr == 0 {
return nil, errors.New("failed to get inventory - 0 pointer")
}
defer func() {
err = errors.Join(err, h.memMgr.Deallocate(ctx, uint64(ptr)))
}()
data, ok := mod.Memory().Read(ptr, size)
if !ok {
return nil, errors.New("failed to get inventory")
}
var inventory rpcv1.PluginInventory
if err = inventory.UnmarshalVT(data); err != nil {
return nil, err
}
refs = make([]sdk.Reference, 0, len(inventory.Modules))
for _, m := range inventory.Modules {
refs = append(refs, sdk.Reference{
Category: sdk.Category(m.GetModuleCategory()),
Type: m.GetModuleType(),
})
}
return refs, nil
}
func (h *Host) getState(ctx context.Context, m api.Module, offset, byteCount uint32) uint64 {
if h.state == nil {
h.state = make(map[string][]byte)
}
buf, ok := m.Memory().Read(offset, byteCount)
if !ok {
return 0
}
getStateReq := new(rpcv1.GetStateRequest)
if err := getStateReq.UnmarshalVT(buf); err != nil {
h.Logger.Error("failed to unmarshal getStateRequest", slog.String("err", err.Error()))
return 0
}
resp := new(rpcv1.GetStateResponse)
resp.Data, _ = h.state[string(getStateReq.Key)]
if ptr, err := h.memMgr.WriteMessage(ctx, m, resp); err != nil {
h.Logger.Error("Failed to write message", slog.String("err", err.Error()))
return 0
} else {
return ptr
}
}
func (h *Host) setState(ctx context.Context, m api.Module, offset, byteCount uint32) (result uint64) {
if h.state == nil {
h.state = make(map[string][]byte)
}
buf, ok := m.Memory().Read(offset, byteCount)
if !ok {
return 0
}
setState := new(rpcv1.SetState)
if err := setState.UnmarshalVT(buf); err != nil {
h.Logger.Error("failed to unmarshal SetState", slog.String("err", err.Error()))
return 0
}
var resp rpcv1.Result
if len(setState.Key) < 1 {
resp.Error = "key might not be empty"
} else {
h.state[string(setState.Key)] = setState.Data
resp.Success = true
}
if ptr, err := h.memMgr.WriteMessage(ctx, m, &resp); err != nil {
h.Logger.Error("Failed to write message", slog.String("err", err.Error()))
return 0
} else {
return ptr
}
}
func (h *Host) log(ctx context.Context, m api.Module, offset, byteCount uint32) {
buf, ok := m.Memory().Read(offset, byteCount)
if !ok {
return
}
taskLog := new(rpcv1.TaskLog)
if err := taskLog.UnmarshalVT(buf); err != nil {
h.Logger.Warn("failed to unmarshal task log", slog.String("err", err.Error()))
return
}
rec := slog.NewRecord(time.UnixMicro(taskLog.Time), slog.Level(taskLog.Level), taskLog.Message, 0)
for i := range taskLog.Attributes {
attr := taskLog.Attributes[i]
rec.AddAttrs(slog.String(attr.Key, attr.Value))
}
_ = h.Logger.Handler().Handle(ctx, rec)
}

View file

@ -1,51 +0,0 @@
package sdk
import (
"context"
"encoding/json"
)
type Encoder[T any] interface {
Get(ctx context.Context, key string) (val T, ok bool, meta StateMetadata, err error)
Set(ctx context.Context, key string, val T) error
}
var _ Encoder[struct{}] = (*JSONStateEncoder[struct{}])(nil)
func NewJSONStateEncoder[T any](ctx ExecutionContext) *JSONStateEncoder[T] {
return &JSONStateEncoder[T]{
Context: ctx,
}
}
type JSONStateEncoder[T any] struct {
Context ExecutionContext
}
func (j JSONStateEncoder[T]) Get(ctx context.Context, key string) (val T, ok bool, meta StateMetadata, err error) {
var data []byte
data, meta, err = j.Context.GetState(ctx, key)
if err != nil {
return val, false, StateMetadata{}, err
}
if len(data) == 0 {
return val, false, meta, nil
}
if err := json.Unmarshal(data, &val); err != nil {
return val, false, StateMetadata{}, err
}
return val, true, meta, nil
}
func (j JSONStateEncoder[T]) Set(ctx context.Context, key string, val T) error {
data, err := json.Marshal(val)
if err != nil {
return err
}
return j.Context.SetState(ctx, key, data)
}

View file

@ -5,10 +5,11 @@ import "C"
import ( import (
"context" "context"
"log/slog"
remotev1 "code.icb4dc0.de/buildr/api/generated/remote/v1" "golang.org/x/exp/slog"
"code.icb4dc0.de/buildr/wasi-module-sdk-go/mem" "code.icb4dc0.de/buildr/wasi-module-sdk-go/mem"
rpcv1 "code.icb4dc0.de/buildr/wasi-module-sdk-go/protocol/generated/rpc/v1"
) )
var _ slog.Handler = (*WASIHandler)(nil) var _ slog.Handler = (*WASIHandler)(nil)
@ -27,16 +28,16 @@ func (h WASIHandler) Enabled(_ context.Context, level slog.Level) bool {
return h.Level <= level return h.Level <= level
} }
func (h WASIHandler) Handle(_ context.Context, record slog.Record) error { func (h WASIHandler) Handle(ctx context.Context, record slog.Record) error {
taskLog := remotev1.TaskLog{ taskLog := rpcv1.TaskLog{
Time: record.Time.UnixMicro(), Time: record.Time.UnixMicro(),
Message: record.Message, Message: record.Message,
Level: int32(record.Level), Level: int32(record.Level),
Attributes: make([]*remotev1.TaskLog_LogAttribute, 0, record.NumAttrs()), Attributes: make([]*rpcv1.TaskLog_LogAttribute, 0, record.NumAttrs()),
} }
record.Attrs(func(attr slog.Attr) bool { record.Attrs(func(attr slog.Attr) bool {
taskLog.Attributes = append(taskLog.Attributes, &remotev1.TaskLog_LogAttribute{ taskLog.Attributes = append(taskLog.Attributes, &rpcv1.TaskLog_LogAttribute{
Key: attr.Key, Key: attr.Key,
Value: attr.Value.String(), Value: attr.Value.String(),
}) })
@ -44,10 +45,13 @@ func (h WASIHandler) Handle(_ context.Context, record slog.Record) error {
return true return true
}) })
if _, err := LogMsg(&taskLog); err != nil { data, err := taskLog.MarshalVT()
if err != nil {
return err return err
} }
_log_msg(mem.DataToManagedPtr(data))
return nil return nil
} }
@ -74,20 +78,3 @@ func (h WASIHandler) WithGroup(name string) slog.Handler {
return newHandler return newHandler
} }
func LogMsg(taskLog *remotev1.TaskLog) (*remotev1.Result, error) {
data, err := taskLog.MarshalVT()
if err != nil {
return nil, err
}
resultPtr := _log_msg(mem.DataToManagedPtr(data))
data = mem.PtrToData(mem.UnifiedPtrToSizePtr(resultPtr))
resp := new(remotev1.Result)
if err := resp.UnmarshalVT(data); err != nil {
return nil, err
}
return resp, nil
}

View file

@ -1,14 +1,9 @@
package mem package mem
import ( // #include <stdlib.h>
"sync" import "C"
"unsafe"
)
var ( import "unsafe"
alivePointers = map[uintptr][]byte{}
pointersLock sync.Mutex
)
func DataToManagedPtr(data []byte) (ptr uint32, size uint32) { func DataToManagedPtr(data []byte) (ptr uint32, size uint32) {
uptr := unsafe.Pointer(unsafe.SliceData(data)) uptr := unsafe.Pointer(unsafe.SliceData(data))
@ -16,16 +11,10 @@ func DataToManagedPtr(data []byte) (ptr uint32, size uint32) {
} }
func DataToUnmanagedPtr(data []byte) (uint32, uint32) { func DataToUnmanagedPtr(data []byte) (uint32, uint32) {
pointersLock.Lock() size := C.ulong(len(data))
defer pointersLock.Unlock() ptr := unsafe.Pointer(C.malloc(size))
copy(unsafe.Slice((*byte)(ptr), len(data)), data)
buf := make([]byte, len(data)) return uint32(uintptr(ptr)), uint32(size)
copy(buf, data)
ptr := &buf[0]
unsafePtr := uintptr(unsafe.Pointer(ptr))
alivePointers[unsafePtr] = buf
return uint32(unsafePtr), uint32(len(data))
} }
func DataFromPtr(ptr, size uint32) []byte { func DataFromPtr(ptr, size uint32) []byte {
@ -44,15 +33,3 @@ func UnifyPtrSize(ptr, size uint32) uint64 {
func UnifiedPtrToSizePtr(uint64ptr uint64) (ptr uint32, size uint32) { func UnifiedPtrToSizePtr(uint64ptr uint64) (ptr uint32, size uint32) {
return uint32(uint64ptr >> 32), uint32(uint64ptr) return uint32(uint64ptr >> 32), uint32(uint64ptr)
} }
//export allocate
func Allocate(size uint32) uintptr {
pointersLock.Lock()
defer pointersLock.Unlock()
buf := make([]byte, size)
unsafePtr := uintptr(unsafe.Pointer(unsafe.SliceData(buf)))
alivePointers[unsafePtr] = buf
return unsafePtr
}

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -1,6 +1,8 @@
package sdk package sdk
import ( import (
"fmt"
"strings"
"sync" "sync"
) )
@ -21,20 +23,28 @@ func (f RegistrationFunc) RegisterAt(registry *TypeRegistry) {
func NewTypeRegistry() *TypeRegistry { func NewTypeRegistry() *TypeRegistry {
return &TypeRegistry{ return &TypeRegistry{
registrations: make(map[Reference]Factory), registrations: make(map[string]Factory),
} }
} }
type TypeRegistry struct { type TypeRegistry struct {
lock sync.Mutex lock sync.Mutex
registrations map[Reference]Factory registrations map[string]Factory
} }
func (r *TypeRegistry) List() (refs []Reference) { func (r *TypeRegistry) List() (refs []Reference) {
refs = make([]Reference, 0, len(r.registrations)) refs = make([]Reference, 0, len(r.registrations))
for k := range r.registrations { for k := range r.registrations {
refs = append(refs, k) split := strings.SplitN(k, "/", 2)
if len(split) < 2 {
continue
}
refs = append(refs, Reference{
Category: Category(split[0]),
Type: split[1],
})
} }
return refs return refs
@ -56,9 +66,6 @@ func (r *TypeRegistry) Get(cat Category, moduleName string) Module {
return f.Create() return f.Create()
} }
func specOf(cat Category, moduleName string) Reference { func specOf(cat Category, moduleName string) string {
return Reference{ return fmt.Sprintf("%s/%s", cat.String(), moduleName)
Category: cat,
Type: moduleName,
}
} }

View file

@ -1,18 +0,0 @@
{
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"extends": [
":dependencyDashboard",
":ignoreModulesAndTests",
":semanticPrefixFixDepsChoreOthers",
":autodetectPinVersions",
":prHourlyLimit2",
":prConcurrentLimit10",
"group:monorepos",
"group:recommended",
"workarounds:all"
],
"postUpdateOptions": [
"gomodTidy1.17",
"gomodUpdateImportPaths"
]
}

View file

@ -3,68 +3,66 @@ package sdk
import ( import (
"errors" "errors"
remotev1 "code.icb4dc0.de/buildr/api/generated/remote/v1"
"code.icb4dc0.de/buildr/wasi-module-sdk-go/mem" "code.icb4dc0.de/buildr/wasi-module-sdk-go/mem"
rpcv1 "code.icb4dc0.de/buildr/wasi-module-sdk-go/protocol/generated/rpc/v1"
) )
type StateProxy struct{} type StateProxy struct {
}
func (s StateProxy) Set(key, state []byte) error { func (s StateProxy) Set(key, state []byte) error {
setCmd := &remotev1.SetState{ setCmd := &rpcv1.SetState{
Key: key, Key: key,
Data: state, Data: state,
} }
data, err := setCmd.MarshalVT()
if resp, err := SetState(setCmd); err != nil { if err != nil {
return err return err
} else if resp.Error != "" {
return errors.New(resp.Error)
} else {
return nil
} }
result := _set_state(mem.DataToManagedPtr(data))
if result == 0 {
return errors.New("unknown error occurred")
}
resultPtr := uint32(result >> 32)
resultSize := uint32(result)
resultMsg := new(rpcv1.Result)
if err := resultMsg.UnmarshalVT(mem.PtrToData(resultPtr, resultSize)); err != nil {
return err
}
if !resultMsg.Success {
if resultMsg.Error != "" {
return errors.New(resultMsg.Error)
}
return errors.New("unknown error occurred")
}
return nil
} }
func (s StateProxy) Get(key []byte) (state []byte, meta StateMetadata, err error) { func (s StateProxy) Get(key []byte) (state []byte, meta StateMetadata, err error) {
getCmd := &remotev1.GetStateRequest{ getCmd := &rpcv1.GetStateRequest{
Key: key, Key: key,
} }
if resp, err := GetState(getCmd); err != nil { data, err := getCmd.MarshalVT()
if err != nil {
return nil, StateMetadata{}, err return nil, StateMetadata{}, err
} else {
return resp.Data, StateMetadata{}, nil
}
} }
func GetState(req *remotev1.GetStateRequest) (*remotev1.GetStateResponse, error) { result := _get_state(mem.DataToManagedPtr(data))
data, err := req.MarshalVT() if result == 0 {
if err != nil { return nil, StateMetadata{}, errors.New("error occurred while processing request")
return nil, err }
resultPtr := uint32(result >> 32)
resultSize := uint32(result)
getStateResult := new(rpcv1.GetStateResponse)
if err := getStateResult.UnmarshalVT(mem.PtrToData(resultPtr, resultSize)); err != nil {
return nil, StateMetadata{}, err
} }
resultPtr := _get_state(mem.DataToManagedPtr(data)) return getStateResult.Data, StateMetadata{}, nil
data = mem.PtrToData(mem.UnifiedPtrToSizePtr(resultPtr))
resp := new(remotev1.GetStateResponse)
if err := resp.UnmarshalVT(data); err != nil {
return nil, err
}
return resp, nil
}
func SetState(req *remotev1.SetState) (*remotev1.Result, error) {
data, err := req.MarshalVT()
if err != nil {
return nil, err
}
resultPtr := _set_state(mem.DataToManagedPtr(data))
data = mem.PtrToData(mem.UnifiedPtrToSizePtr(resultPtr))
resp := new(remotev1.Result)
if err := resp.UnmarshalVT(data); err != nil {
return nil, err
}
return resp, nil
} }