diff --git a/examples/nodenumber/main.go b/examples/nodenumber/main.go index 0382c2d4..a4b03fec 100644 --- a/examples/nodenumber/main.go +++ b/examples/nodenumber/main.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/kube-scheduler-wasm-extension/examples/nodenumber/plugin" "sigs.k8s.io/kube-scheduler-wasm-extension/guest/enqueue" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/host" "sigs.k8s.io/kube-scheduler-wasm-extension/guest/prescore" "sigs.k8s.io/kube-scheduler-wasm-extension/guest/score" ) @@ -34,7 +35,10 @@ import ( // main is compiled to a WebAssembly function named "_start", called by the // wasm scheduler plugin during initialization. func main() { - plugin := &plugin.NodeNumber{} + plugin, err := plugin.New(host.Get()) + if err != nil { + panic(err) + } // Below is like `var _ api.EnqueueExtensions = plugin`, except it also // wires up functions the host should provide (go:wasmimport). enqueue.SetPlugin(plugin) diff --git a/examples/nodenumber/main.wasm b/examples/nodenumber/main.wasm index 4b440335..aaf00748 100755 Binary files a/examples/nodenumber/main.wasm and b/examples/nodenumber/main.wasm differ diff --git a/examples/nodenumber/plugin/plugin.go b/examples/nodenumber/plugin/plugin.go index b6300fce..977ec424 100644 --- a/examples/nodenumber/plugin/plugin.go +++ b/examples/nodenumber/plugin/plugin.go @@ -22,7 +22,6 @@ // - Doesn't return an error if state has the wrong type, as it is // impossible: this panics instead with the default message. // - TODO: logging -// - TODO: config // // See https://github.com/kubernetes-sigs/kube-scheduler-simulator/blob/simulator/v0.1.0/simulator/docs/sample/nodenumber/plugin.go // @@ -30,6 +29,9 @@ package plugin import ( + "encoding/json" + "fmt" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api" "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api/proto" ) @@ -50,6 +52,21 @@ type NodeNumber struct { reverse bool } +// New creates a new NodeNumber plugin for the given host or returns an error. +func New(host api.Host) (*NodeNumber, error) { + var args nodeNumberArgs + if config := host.GetConfig(); config != nil { + if err := json.Unmarshal(config, &args); err != nil { + return nil, fmt.Errorf("decode arg into NodeNumberArgs: %w", err) + } + } + return &NodeNumber{reverse: args.Reverse}, nil +} + +type nodeNumberArgs struct { + Reverse bool `json:"reverse"` +} + const ( // Name is the name of the plugin used in the plugin registry and configurations. Name = "NodeNumber" diff --git a/guest/api/host.go b/guest/api/host.go new file mode 100644 index 00000000..740298e6 --- /dev/null +++ b/guest/api/host.go @@ -0,0 +1,12 @@ +package api + +// Host is the WebAssembly host side of the scheduler. Specifically, this +// scheduler framework.Plugin written in Go, which runs the Plugin this SDK +// compiles to Wasm. +type Host interface { + // GetConfig reads any configuration set by the host. + // + // Note: This is not updated dynamically. + GetConfig() []byte + // ^-- Note: This is a []byte, not a string, for json.Unmarshaler. +} diff --git a/guest/host/host.go b/guest/host/host.go new file mode 100644 index 00000000..b8ebb3cd --- /dev/null +++ b/guest/host/host.go @@ -0,0 +1,57 @@ +/* + Copyright 2023 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package host imports an api.Host from the WebAssembly host. +package host + +import ( + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api" + "sigs.k8s.io/kube-scheduler-wasm-extension/guest/internal/mem" +) + +// Get can be called at any time, to use features exported by the host. +// +// For example: +// +// func main() { +// config := host.Get().GetConfig() +// // decode yaml +// } +func Get() api.Host { + return currentHost +} + +var currentHost api.Host = &host{} + +type host struct { + config []byte +} + +func (n *host) GetConfig() []byte { + return n.lazyConfig() +} + +func (n *host) lazyConfig() []byte { + if config := n.config; config != nil { + return config + } + + // Wrap to avoid TinyGo 0.28: cannot use an exported function as value + n.config = mem.GetBytes(func(ptr uint32, limit mem.BufLimit) (len uint32) { + return getConfig(ptr, limit) + }) + return n.config +} diff --git a/guest/host/imports.go b/guest/host/imports.go new file mode 100644 index 00000000..f3ffdc84 --- /dev/null +++ b/guest/host/imports.go @@ -0,0 +1,24 @@ +//go:build tinygo.wasm + +/* + Copyright 2023 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package host + +import "sigs.k8s.io/kube-scheduler-wasm-extension/guest/internal/mem" + +//go:wasmimport k8s.io/scheduler get_config +func getConfig(ptr uint32, limit mem.BufLimit) (len uint32) diff --git a/guest/host/imports_stub.go b/guest/host/imports_stub.go new file mode 100644 index 00000000..69c66452 --- /dev/null +++ b/guest/host/imports_stub.go @@ -0,0 +1,24 @@ +//go:build !tinygo.wasm + +/* + Copyright 2023 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package host + +import "sigs.k8s.io/kube-scheduler-wasm-extension/guest/internal/mem" + +// getConfig is stubbed for compilation outside TinyGo. +func getConfig(ptr uint32, limit mem.BufLimit) (len uint32) { return } diff --git a/guest/internal/mem/mem.go b/guest/internal/mem/mem.go index 142a28ec..2f4e86a6 100644 --- a/guest/internal/mem/mem.go +++ b/guest/internal/mem/mem.go @@ -68,6 +68,27 @@ func Update( return updater(readBuf) } +func GetBytes(fn func(ptr uint32, limit BufLimit) (len uint32)) []byte { + size := fn(uint32(readBufPtr), readBufLimit) + if size == 0 { + return nil + } + + buf := make([]byte, size) + + // If the function result fit in our read buffer, copy it out. + if size <= readBufLimit { + copy(buf, readBuf[:size]) + return buf + } + + // If the size returned from the function was larger than our read buffer, + // we need to execute it again. + ptr := unsafe.Pointer(&buf[0]) + _ = fn(uint32(uintptr(ptr)), size) + return buf +} + func GetString(fn func(ptr uint32, limit BufLimit) (len uint32)) string { size := fn(uint32(readBufPtr), readBufLimit) if size == 0 { diff --git a/internal/e2e/scheduler/scheduler_test.go b/internal/e2e/scheduler/scheduler_test.go index 767d7258..a11caacf 100644 --- a/internal/e2e/scheduler/scheduler_test.go +++ b/internal/e2e/scheduler/scheduler_test.go @@ -18,6 +18,7 @@ package scheduler_test import ( "context" + "fmt" "io" "testing" @@ -52,7 +53,7 @@ func TestCycleStateCoherence(t *testing.T) { func TestExample_NodeNumber(t *testing.T) { ctx := context.Background() - plugin := newPlugin(ctx, t) + plugin := newNodeNumberPlugin(ctx, t, false) defer plugin.(io.Closer).Close() pod := &v1.Pod{Spec: v1.PodSpec{NodeName: "happy8"}} @@ -74,6 +75,17 @@ func TestExample_NodeNumber(t *testing.T) { t.Fatalf("unexpected score: want %v, have %v", want, have) } }) + + t.Run("Reverse means score zero on match", func(t *testing.T) { + // This proves we can read configuration. + reversed := newNodeNumberPlugin(ctx, t, true) + defer reversed.(io.Closer).Close() + + score := e2e.RunAll(ctx, t, reversed, pod, nodeInfoWithName("glad8")) + if want, have := int64(0), score; want != have { + t.Fatalf("unexpected score: want %v, have %v", want, have) + } + }) } func BenchmarkExample_NodeNumber(b *testing.B) { @@ -81,11 +93,11 @@ func BenchmarkExample_NodeNumber(b *testing.B) { b.Run("New", func(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - newPlugin(ctx, b).(io.Closer).Close() + newNodeNumberPlugin(ctx, b, false).(io.Closer).Close() } }) - plugin := newPlugin(ctx, b) + plugin := newNodeNumberPlugin(ctx, b, false) defer plugin.(io.Closer).Close() pod := *test.PodReal // copy @@ -102,8 +114,11 @@ func BenchmarkExample_NodeNumber(b *testing.B) { }) } -func newPlugin(ctx context.Context, t e2e.Testing) framework.Plugin { - plugin, err := wasm.NewFromConfig(ctx, wasm.PluginConfig{GuestPath: test.PathExampleNodeNumber}) +func newNodeNumberPlugin(ctx context.Context, t e2e.Testing, reverse bool) framework.Plugin { + plugin, err := wasm.NewFromConfig(ctx, wasm.PluginConfig{ + GuestPath: test.PathExampleNodeNumber, + GuestConfig: fmt.Sprintf(`{"reverse": %v}`, reverse), + }) if err != nil { t.Fatalf("failed to create plugin: %v", err) } diff --git a/scheduler/plugin/config.go b/scheduler/plugin/config.go index 24eea954..f33d31b2 100644 --- a/scheduler/plugin/config.go +++ b/scheduler/plugin/config.go @@ -20,6 +20,9 @@ type PluginConfig struct { // GuestPath is the path to the guest wasm. GuestPath string `json:"guestPath"` + // GuestConfig is any configuration to give to the guest. + GuestConfig string `json:"guestConfig"` + // Args are the os.Args the guest will receive, exposed for tests. Args []string } diff --git a/scheduler/plugin/host.go b/scheduler/plugin/host.go index d74ba762..bd148cf4 100644 --- a/scheduler/plugin/host.go +++ b/scheduler/plugin/host.go @@ -34,6 +34,7 @@ const ( k8sApiNodeName = "nodeName" k8sApiPod = "pod" k8sScheduler = "k8s.io/scheduler" + k8sSchedulerGetConfig = "get_config" k8sSchedulerResultClusterEvents = "result.cluster_events" k8sSchedulerResultNodeNames = "result.node_names" k8sSchedulerResultStatusReason = "result.status_reason" @@ -56,8 +57,12 @@ func instantiateHostApi(ctx context.Context, runtime wazero.Runtime) (wazeroapi. Instantiate(ctx) } -func instantiateHostScheduler(ctx context.Context, runtime wazero.Runtime) (wazeroapi.Module, error) { +func instantiateHostScheduler(ctx context.Context, runtime wazero.Runtime, guestConfig string) (wazeroapi.Module, error) { + host := &host{guestConfig: guestConfig} return runtime.NewHostModuleBuilder(k8sScheduler). + NewFunctionBuilder(). + WithGoModuleFunction(wazeroapi.GoModuleFunc(host.k8sSchedulerGetConfigFn), []wazeroapi.ValueType{i32, i32}, []wazeroapi.ValueType{i32}). + WithParameterNames("buf", "buf_limit").Export(k8sSchedulerGetConfig). NewFunctionBuilder(). WithGoModuleFunction(wazeroapi.GoModuleFunc(k8sSchedulerResultClusterEventsFn), []wazeroapi.ValueType{i32, i32}, []wazeroapi.ValueType{}). WithParameterNames("buf", "buf_len").Export(k8sSchedulerResultClusterEvents). @@ -154,6 +159,19 @@ func k8sApiPodFn(ctx context.Context, mod wazeroapi.Module, stack []uint64) { stack[0] = uint64(marshalIfUnderLimit(mod.Memory(), pod, buf, bufLimit)) } +type host struct { + guestConfig string +} + +func (h host) k8sSchedulerGetConfigFn(_ context.Context, mod wazeroapi.Module, stack []uint64) { + buf := uint32(stack[0]) + bufLimit := bufLimit(stack[1]) + + config := h.guestConfig + + stack[0] = uint64(writeStringIfUnderLimit(mod.Memory(), config, buf, bufLimit)) +} + // k8sSchedulerResultClusterEventsFn is a function used by the wasm guest to set the // cluster events result from guestExportEnqueue. func k8sSchedulerResultClusterEventsFn(ctx context.Context, mod wazeroapi.Module, stack []uint64) { diff --git a/scheduler/plugin/plugin.go b/scheduler/plugin/plugin.go index 04d9f72c..5f496bd9 100644 --- a/scheduler/plugin/plugin.go +++ b/scheduler/plugin/plugin.go @@ -56,7 +56,7 @@ func NewFromConfig(ctx context.Context, config PluginConfig) (framework.Plugin, return nil, fmt.Errorf("wasm: error reading guest binary at %s: %w", config.GuestPath, err) } - runtime, guestModule, err := prepareRuntime(ctx, guestBin) + runtime, guestModule, err := prepareRuntime(ctx, guestBin, config.GuestConfig) if err != nil { return nil, err } diff --git a/scheduler/plugin/plugin_test.go b/scheduler/plugin/plugin_test.go index 4f1e4cb2..a52ebebd 100644 --- a/scheduler/plugin/plugin_test.go +++ b/scheduler/plugin/plugin_test.go @@ -347,6 +347,7 @@ func TestPreFilter(t *testing.T) { tests := []struct { name string guestPath string + guestConfig string args []string globals map[string]int32 pod *v1.Pod @@ -391,6 +392,26 @@ wasm error: unreachable wasm stack trace: panic_on_prefilter.$1() i32`, }, + { + name: "panic no guestConfig", + guestPath: test.PathErrorPanicOnGetConfig, + pod: test.PodSmall, + expectedStatusCode: framework.Error, + expectedStatusMessage: `wasm: prefilter error: wasm error: unreachable +wasm stack trace: + panic_on_get_config.$2() i32`, + }, + { // This only tests that configuration gets assigned + name: "panic guestConfig", + guestPath: test.PathErrorPanicOnGetConfig, + guestConfig: "hello", + pod: test.PodSmall, + expectedStatusCode: framework.Error, + expectedStatusMessage: `wasm: prefilter error: hello +wasm error: unreachable +wasm stack trace: + panic_on_get_config.$2() i32`, + }, } for _, tc := range tests { @@ -400,7 +421,7 @@ wasm stack trace: guestPath = test.PathTestFilter } - p, err := wasm.NewFromConfig(ctx, wasm.PluginConfig{GuestPath: guestPath, Args: tc.args}) + p, err := wasm.NewFromConfig(ctx, wasm.PluginConfig{GuestPath: guestPath, Args: tc.args, GuestConfig: tc.guestConfig}) if err != nil { t.Fatal(err) } diff --git a/scheduler/plugin/runtime.go b/scheduler/plugin/runtime.go index 53a74742..abc2b240 100644 --- a/scheduler/plugin/runtime.go +++ b/scheduler/plugin/runtime.go @@ -26,7 +26,7 @@ import ( ) // prepareRuntime compiles the guest and instantiates any host modules it needs. -func prepareRuntime(ctx context.Context, guestBin []byte) (runtime wazero.Runtime, guest wazero.CompiledModule, err error) { +func prepareRuntime(ctx context.Context, guestBin []byte, guestConfig string) (runtime wazero.Runtime, guest wazero.CompiledModule, err error) { // Create the runtime, which when closed releases any resources associated with it. runtime = wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig(). // Here are settings required by the wasm profiler wzprof: @@ -63,7 +63,7 @@ func prepareRuntime(ctx context.Context, guestBin []byte) (runtime wazero.Runtim } fallthrough // proceed to more imports case imports&importK8sScheduler != 0: - if _, err = instantiateHostScheduler(ctx, runtime); err != nil { + if _, err = instantiateHostScheduler(ctx, runtime, guestConfig); err != nil { err = fmt.Errorf("wasm: error instantiating scheduler host functions: %w", err) return } diff --git a/scheduler/test/testdata.go b/scheduler/test/testdata.go index 10f3af3a..773b9a92 100644 --- a/scheduler/test/testdata.go +++ b/scheduler/test/testdata.go @@ -18,6 +18,8 @@ import ( var PathErrorNotPlugin = pathWatError("not_plugin") +var PathErrorPanicOnGetConfig = pathWatError("panic_on_get_config") + var PathErrorPanicOnEnqueue = pathWatError("panic_on_enqueue") var PathErrorPanicOnPreFilter = pathWatError("panic_on_prefilter") diff --git a/scheduler/test/testdata/error/panic_on_enqueue.wat b/scheduler/test/testdata/error/panic_on_enqueue.wat index fd81ff2d..d3014510 100644 --- a/scheduler/test/testdata/error/panic_on_enqueue.wat +++ b/scheduler/test/testdata/error/panic_on_enqueue.wat @@ -1,5 +1,5 @@ ;; panic_on_enqueue is a enqueue which issues an unreachable instruction -;; after writing and error to stdout. This simulates a panic in TinyGo. +;; after writing an error to stdout. This simulates a panic in TinyGo. (module $panic_on_enqueue ;; Import the fd_write function from wasi, used in TinyGo for println. (import "wasi_snapshot_preview1" "fd_write" diff --git a/scheduler/test/testdata/error/panic_on_filter.wat b/scheduler/test/testdata/error/panic_on_filter.wat index 2342dc40..54226a5d 100644 --- a/scheduler/test/testdata/error/panic_on_filter.wat +++ b/scheduler/test/testdata/error/panic_on_filter.wat @@ -1,5 +1,5 @@ ;; panic_on_filter is a filter which issues an unreachable instruction -;; after writing and error to stdout. This simulates a panic in TinyGo. +;; after writing an error to stdout. This simulates a panic in TinyGo. (module $panic_on_filter ;; Import the fd_write function from wasi, used in TinyGo for println. (import "wasi_snapshot_preview1" "fd_write" diff --git a/scheduler/test/testdata/error/panic_on_get_config.wasm b/scheduler/test/testdata/error/panic_on_get_config.wasm new file mode 100644 index 00000000..29ff3dfc Binary files /dev/null and b/scheduler/test/testdata/error/panic_on_get_config.wasm differ diff --git a/scheduler/test/testdata/error/panic_on_get_config.wat b/scheduler/test/testdata/error/panic_on_get_config.wat new file mode 100644 index 00000000..801213ec --- /dev/null +++ b/scheduler/test/testdata/error/panic_on_get_config.wat @@ -0,0 +1,49 @@ +;; $panic_on_get_config is a prefilter which issues an unreachable instruction +;; after writing config to stdout. This is a way to prove configuration got to +;; the guest. +(module $panic_on_get_config + ;; Import the fd_write function from wasi, used in TinyGo for println. + (import "wasi_snapshot_preview1" "fd_write" + (func $wasi.fd_write (param $fd i32) (param $iovs i32) (param $iovs_len i32) (param $result.size i32) (result (;errno;) i32))) + + ;; get_config writes configuration from the host to memory if it exists and + ;; isn't larger than $buf_limit. The result is its length in bytes. + (import "k8s.io/scheduler" "get_config" (func $get_config + (param $buf i32) (param $buf_limit i32) + (result (; len ;) i32))) + + ;; Allocate the minimum amount of memory, 1 page (64KB). + (memory (export "memory") 1 1) + + ;; config_limit is the max size config to read. + (global $config_limit i32 (i32.const 1024)) + + ;; On prefilter, write "panic!" to stdout and crash. + (func (export "prefilter") (result i32) + (local $config_len i32) + + ;; Write config to offset 8, which is the location where the data for + ;; stdout begins + (i32.store (i32.const 0) (i32.const 8)) ;; iovs[0].offset + (local.set $config_len + (call $get_config (i32.const 8) (global.get $config_limit))) ;; iovs[0] + + ;; if config_len > config_limit { panic } + (if (i32.gt_u (local.get $config_len) (global.get $config_limit)) + (then unreachable)) + + ;; Write the length of configuration read. + (i32.store (i32.const 4) (local.get $config_len)) ;; iovs[0].length + + ;; Write the panic to stdout via its iovec [offset, len]. + (call $wasi.fd_write + (i32.const 1) ;; stdout + (i32.const 0) ;; where's the iovec + (i32.const 1) ;; only one iovec + (i32.const 0) ;; overwrite the iovec with the ignored result. + ) + drop ;; ignore the errno returned + + ;; Issue the unreachable instruction instead of returning a code + (unreachable)) +) diff --git a/scheduler/test/testdata/error/panic_on_prefilter.wat b/scheduler/test/testdata/error/panic_on_prefilter.wat index d1ee334a..57275376 100644 --- a/scheduler/test/testdata/error/panic_on_prefilter.wat +++ b/scheduler/test/testdata/error/panic_on_prefilter.wat @@ -1,5 +1,5 @@ ;; panic_on_prefilter is a prefilter which issues an unreachable instruction -;; after writing and error to stdout. This simulates a panic in TinyGo. +;; after writing an error to stdout. This simulates a panic in TinyGo. (module $panic_on_prefilter ;; Import the fd_write function from wasi, used in TinyGo for println. (import "wasi_snapshot_preview1" "fd_write" diff --git a/scheduler/test/testdata/error/panic_on_prescore.wat b/scheduler/test/testdata/error/panic_on_prescore.wat index e7217f76..14fa4a26 100644 --- a/scheduler/test/testdata/error/panic_on_prescore.wat +++ b/scheduler/test/testdata/error/panic_on_prescore.wat @@ -1,5 +1,5 @@ ;; panic_on_prescore is a prescore which issues an unreachable instruction -;; after writing and error to stdout. This simulates a panic in TinyGo. +;; after writing an error to stdout. This simulates a panic in TinyGo. (module $panic_on_prescore ;; Import the fd_write function from wasi, used in TinyGo for println. (import "wasi_snapshot_preview1" "fd_write" diff --git a/scheduler/test/testdata/error/panic_on_score.wat b/scheduler/test/testdata/error/panic_on_score.wat index fbab0934..72600495 100644 --- a/scheduler/test/testdata/error/panic_on_score.wat +++ b/scheduler/test/testdata/error/panic_on_score.wat @@ -1,5 +1,5 @@ ;; panic_on_score is a score which issues an unreachable instruction -;; after writing and error to stdout. This simulates a panic in TinyGo. +;; after writing an error to stdout. This simulates a panic in TinyGo. (module $panic_on_score ;; Import the fd_write function from wasi, used in TinyGo for println. (import "wasi_snapshot_preview1" "fd_write" diff --git a/scheduler/test/testdata/error/panic_on_start.wat b/scheduler/test/testdata/error/panic_on_start.wat index 6a879fe6..497f24d5 100644 --- a/scheduler/test/testdata/error/panic_on_start.wat +++ b/scheduler/test/testdata/error/panic_on_start.wat @@ -1,5 +1,5 @@ ;; panic_on_start is a WASI command which issues an unreachable instruction -;; after writing and error to stdout. This simulates a panic in TinyGo. +;; after writing an error to stdout. This simulates a panic in TinyGo. (module $panic_on_start ;; Import the fd_write function from wasi, used in TinyGo for println. (import "wasi_snapshot_preview1" "fd_write"