diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b1d010..34ecee3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.22' - name: Build run: go build -v ./... diff --git a/README.md b/README.md index 57f9a4e..2b93fc9 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,42 @@ config := PluginConfig{ _, err := NewPlugin(ctx, manifest, config, []HostFunction{}) ``` +### Integrate with Dylibso Observe SDK +Dylibso provides [observability SDKs](https://github.com/dylibso/observe-sdk) for WebAssembly (Wasm), enabling continuous monitoring of WebAssembly code as it executes within a runtime. It provides developers with the tools necessary to capture and emit telemetry data from Wasm code, including function execution and memory allocation traces, logs, and metrics. + +While Observe SDK has adapters for many popular observability platforms, it also ships with an stdout adapter: + +``` +ctx := context.Background() + +adapter := stdout.NewStdoutAdapter() +adapter.Start(ctx) + +manifest := manifest("nested.c.instr.wasm") + +config := PluginConfig{ + ModuleConfig: wazero.NewModuleConfig().WithSysWalltime(), + EnableWasi: true, + ObserveAdapter: adapter.AdapterBase, +} + +plugin, err := NewPlugin(ctx, manifest, config, []HostFunction{}) +if err != nil { + panic(err) +} + +meta := map[string]string{ + "http.url": "https://example.com/my-endpoint", + "http.status_code": "200", + "http.client_ip": "192.168.1.0", +} + +plugin.TraceCtx.Metadata(meta) + +_, _, _ = plugin.Call("_start", []byte("hello world")) +plugin.Close() +``` + ### Enable filesystem access WASM plugins can read/write files outside the runtime. To do this we add `AllowedPaths` mapping of "HOST:PLUGIN" to the `extism.Manifest` of our plugin. diff --git a/extism.go b/extism.go index 0fd7e94..f1fc40b 100644 --- a/extism.go +++ b/extism.go @@ -15,12 +15,18 @@ import ( "strings" "time" + observe "github.com/dylibso/observe-sdk/go" "github.com/tetratelabs/wazero" "github.com/tetratelabs/wazero/api" "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" "github.com/tetratelabs/wazero/sys" ) +type module struct { + module api.Module + wasm []byte +} + //go:embed extism-runtime.wasm var extismRuntimeWasm []byte @@ -41,10 +47,12 @@ type Runtime struct { // PluginConfig contains configuration options for the Extism plugin. type PluginConfig struct { - ModuleConfig wazero.ModuleConfig - RuntimeConfig wazero.RuntimeConfig - EnableWasi bool - LogLevel LogLevel + ModuleConfig wazero.ModuleConfig + RuntimeConfig wazero.RuntimeConfig + EnableWasi bool + LogLevel LogLevel + ObserveAdapter *observe.AdapterBase + ObserveOptions *observe.Options } // HttpRequest represents an HTTP request to be made by the plugin. @@ -87,8 +95,8 @@ func (l LogLevel) String() string { // Plugin is used to call WASM functions type Plugin struct { Runtime *Runtime - Modules map[string]api.Module - Main api.Module + Modules map[string]module + Main module Timeout time.Duration Config map[string]string // NOTE: maybe we can have some nice methods for getting/setting vars @@ -101,6 +109,8 @@ type Plugin struct { log func(LogLevel, string) logLevel LogLevel guestRuntime guestRuntime + Adapter *observe.AdapterBase + TraceCtx *observe.TraceCtx } func logStd(level LogLevel, message string) { @@ -383,7 +393,7 @@ func NewPlugin( return nil, fmt.Errorf("Manifest can't be empty.") } - modules := map[string]api.Module{} + modules := map[string]module{} // NOTE: this is only necessary for guest modules because // host modules have the same access privileges as the host itself @@ -419,6 +429,7 @@ func NewPlugin( // - If there is only one module in the manifest then that is the main module by default // - Otherwise the last module listed is the main module + var trace *observe.TraceCtx for i, wasm := range manifest.Wasm { data, err := wasm.ToWasmData(ctx) if err != nil { @@ -430,6 +441,15 @@ func NewPlugin( data.Name = "main" } + if data.Name == "main" && config.ObserveAdapter != nil { + trace, err = config.ObserveAdapter.NewTraceCtx(ctx, c.Wazero, data.Data, config.ObserveOptions) + if err != nil { + return nil, fmt.Errorf("Failed to initialize Observe Adapter: %v", err) + } + + trace.Finish() + } + _, okh := hostModules[data.Name] _, okm := modules[data.Name] @@ -449,7 +469,7 @@ func NewPlugin( return nil, err } - modules[data.Name] = m + modules[data.Name] = module{module: m, wasm: data.Data} } logLevel := LogLevelWarn @@ -468,7 +488,7 @@ func NewPlugin( varMax = int64(manifest.Memory.MaxVarBytes) } for _, m := range modules { - if m.Name() == "main" { + if m.module.Name() == "main" { p := &Plugin{ Runtime: &c, Modules: modules, @@ -483,6 +503,8 @@ func NewPlugin( MaxVarBytes: varMax, log: logStd, logLevel: logLevel, + Adapter: config.ObserveAdapter, + TraceCtx: trace, } p.guestRuntime = detectGuestRuntime(ctx, p) @@ -574,7 +596,7 @@ func (plugin *Plugin) GetErrorWithContext(ctx context.Context) string { // FunctionExists returns true when the named function is present in the plugin's main module func (plugin *Plugin) FunctionExists(name string) bool { - return plugin.Main.ExportedFunction(name) != nil + return plugin.Main.module.ExportedFunction(name) != nil } // Call a function by name with the given input, returning the output @@ -599,7 +621,7 @@ func (plugin *Plugin) CallWithContext(ctx context.Context, name string, data []b ctx = context.WithValue(ctx, "inputOffset", intputOffset) - var f = plugin.Main.ExportedFunction(name) + var f = plugin.Main.module.ExportedFunction(name) if f == nil { return 1, []byte{}, errors.New(fmt.Sprintf("Unknown function: %s", name)) @@ -620,6 +642,10 @@ func (plugin *Plugin) CallWithContext(ctx context.Context, name string, data []b res, err := f.Call(ctx) + if plugin.TraceCtx != nil { + defer plugin.TraceCtx.Finish() + } + // Try to extact WASI exit code if exitErr, ok := err.(*sys.ExitError); ok { exitCode := exitErr.ExitCode() diff --git a/extism_test.go b/extism_test.go index 9c44f32..cdcb8c7 100644 --- a/extism_test.go +++ b/extism_test.go @@ -11,6 +11,8 @@ import ( "testing" "time" + observe "github.com/dylibso/observe-sdk/go" + "github.com/dylibso/observe-sdk/go/adapter/stdout" "github.com/stretchr/testify/assert" "github.com/tetratelabs/wazero" "github.com/tetratelabs/wazero/experimental" @@ -763,6 +765,79 @@ func TestInputOffset(t *testing.T) { } } +func TestObserve(t *testing.T) { + ctx := context.Background() + + var buf bytes.Buffer + log.SetOutput(&buf) + + adapter := stdout.NewStdoutAdapter() + adapter.Start(ctx) + + manifest := manifest("nested.c.instr.wasm") + + config := PluginConfig{ + ModuleConfig: wazero.NewModuleConfig().WithSysWalltime(), + EnableWasi: true, + ObserveAdapter: adapter.AdapterBase, + ObserveOptions: &observe.Options{ + SpanFilter: &observe.SpanFilter{MinDuration: 1 * time.Nanosecond}, + ChannelBufferSize: 1024, + }, + } + + // Plugin 1 + plugin, err := NewPlugin(ctx, manifest, config, []HostFunction{}) + if err != nil { + panic(err) + } + + meta := map[string]string{ + "http.url": "https://example.com/my-endpoint", + "http.status_code": "200", + "http.client_ip": "192.168.1.0", + } + + plugin.TraceCtx.Metadata(meta) + + _, _, _ = plugin.Call("_start", []byte("hello world")) + plugin.Close() + + // HACK: make sure we give enough time for the events to get flushed + time.Sleep(100 * time.Millisecond) + + actual := buf.String() + assert.Contains(t, actual, " Call to _start took") + assert.Contains(t, actual, " Call to main took") + assert.Contains(t, actual, " Call to one took") + assert.Contains(t, actual, " Call to two took") + assert.Contains(t, actual, " Call to three took") + assert.Contains(t, actual, " Call to printf took") + + // Reset underlying buffer + buf.Reset() + + // Plugin 2 + plugin2, err := NewPlugin(ctx, manifest, config, []HostFunction{}) + if err != nil { + panic(err) + } + + _, _, _ = plugin2.Call("_start", []byte("hello world")) + plugin2.Close() + + // HACK: make sure we give enough time for the events to get flushed + time.Sleep(100 * time.Millisecond) + + actual2 := buf.String() + assert.Contains(t, actual2, " Call to _start took") + assert.Contains(t, actual2, " Call to main took") + assert.Contains(t, actual2, " Call to one took") + assert.Contains(t, actual2, " Call to two took") + assert.Contains(t, actual2, " Call to three took") + assert.Contains(t, actual2, " Call to printf took") +} + // make sure cancelling the context given to NewPlugin doesn't affect plugin calls func TestContextCancel(t *testing.T) { manifest := manifest("sleep.wasm") diff --git a/go.mod b/go.mod index 332bf23..52787eb 100644 --- a/go.mod +++ b/go.mod @@ -1,15 +1,20 @@ module github.com/extism/go-sdk -go 1.20 +go 1.22 require ( + github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a github.com/gobwas/glob v0.2.3 - github.com/stretchr/testify v1.8.4 - github.com/tetratelabs/wazero v1.7.3 + github.com/stretchr/testify v1.9.0 + github.com/tetratelabs/wazero v1.8.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/ianlancetaylor/demangle v0.0.0-20240805132620-81f5be970eca // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 8383fa2..8dae262 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,29 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dylibso/observe-sdk/go v0.0.0-20240815143955-7e0389165219 h1:ISvdktS6sAspgbQ15M/eagCWo8TqOTRB5SlnH5BlPWQ= +github.com/dylibso/observe-sdk/go v0.0.0-20240815143955-7e0389165219/go.mod h1:C8DzXehI4zAbrdlbtOByKX6pfivJTBiV9Jjqv56Yd9Q= +github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a h1:UwSIFv5g5lIvbGgtf3tVwC7Ky9rmMFBp0RMs+6f6YqE= +github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a/go.mod h1:C8DzXehI4zAbrdlbtOByKX6pfivJTBiV9Jjqv56Yd9Q= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/ianlancetaylor/demangle v0.0.0-20240805132620-81f5be970eca h1:T54Ema1DU8ngI+aef9ZhAhNGQhcRTrWxVeG07F+c/Rw= +github.com/ianlancetaylor/demangle v0.0.0-20240805132620-81f5be970eca/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tetratelabs/wazero v1.7.3 h1:PBH5KVahrt3S2AHgEjKu4u+LlDbbk+nsGE3KLucy6Rw= -github.com/tetratelabs/wazero v1.7.3/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 h1:ZF+QBjOI+tILZjBaFj3HgFonKXUcwgJ4djLb6i42S3Q= +github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834/go.mod h1:m9ymHTgNSEjuxvw8E7WWe4Pl4hZQHXONY8wE6dMLaRk= +github.com/tetratelabs/wazero v1.8.0 h1:iEKu0d4c2Pd+QSRieYbnQC9yiFlMS9D+Jr0LsRmcF4g= +github.com/tetratelabs/wazero v1.8.0/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/runtime.go b/runtime.go index 8013b93..9fa8c00 100644 --- a/runtime.go +++ b/runtime.go @@ -25,12 +25,12 @@ type guestRuntime struct { func detectGuestRuntime(ctx context.Context, p *Plugin) guestRuntime { m := p.Main - runtime, ok := haskellRuntime(ctx, p, m) + runtime, ok := haskellRuntime(ctx, p, m.module) if ok { return runtime } - runtime, ok = wasiRuntime(ctx, p, m) + runtime, ok = wasiRuntime(ctx, p, m.module) if ok { return runtime } diff --git a/wasm/nested.c.instr.wasm b/wasm/nested.c.instr.wasm new file mode 100644 index 0000000..7f74c7f Binary files /dev/null and b/wasm/nested.c.instr.wasm differ