diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2a3955e..ac07922 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -36,11 +36,17 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: 1.17 + go-version: 1.19 + - name: Init API module + working-directory: go + run: | + sudo apt update -qq && sudo apt install -yqq libdevmapper-dev + go mod init github.com/sysflow-telemetry/sf-apis/go || true + go mod tidy - name: Lint go API uses: golangci/golangci-lint-action@v3 with: - version: v1.47.1 + version: v1.51.1 working-directory: go args: --disable=errcheck --build-tags=exclude_graphdriver_btrfs lint-python-api: diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d05699..9d231fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,18 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ## [Unreleased] +## [0.6.0] - 2023-09-30 + +### Changed + +- Make Python packages globally installed in base sfnb notebook +- Go 1.19 and refactor of plugin interfaces to support generics + +### Security + +- CVE-2022-32149: Denial of service in golang.org/x/text/language (updated to 0.3.8) +- CVE-2022-29526: golang.org/x/sys/unix has Incorrect privilege reporting in syscall (updated to 0.0.0-20220412211240-33da011f77ad) + ## [0.5.1] - 2023-06-07 ### Added @@ -214,7 +226,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - First release candidate with basic set of SysFlow APIs (C++ and Python). -[Unreleased]: https://github.com/sysflow-telemetry/sf-apis/compare/0.5.1...HEAD +[Unreleased]: https://github.com/sysflow-telemetry/sf-apis/compare/0.6.0...HEAD +[0.6.0]: https://github.com/sysflow-telemetry/sf-apis/compare/0.5.1...0.6.0 [0.5.1]: https://github.com/sysflow-telemetry/sf-apis/compare/0.5.0...0.5.1 [0.5.0]: https://github.com/sysflow-telemetry/sf-apis/compare/0.4.4...0.5.0 [0.4.4]: https://github.com/sysflow-telemetry/sf-apis/compare/0.4.3...0.4.4 diff --git a/Dockerfile.sfnb b/Dockerfile.sfnb index a59ecfa..d12b0b8 100644 --- a/Dockerfile.sfnb +++ b/Dockerfile.sfnb @@ -44,4 +44,4 @@ COPY --chown=$NB_UID:$NB_UID py3 /tmp/build COPY --chown=$NB_UID:$NB_UID pynb/requirements.txt /tmp/build/requirements.pynb.txt # Install sysflow API -RUN cd /tmp/build && pip install --user -r requirements.pynb.txt && pip install --user . && rm -r /tmp/build +RUN cd /tmp/build && pip install -r requirements.pynb.txt && pip install . && rm -r /tmp/build diff --git a/README.md b/README.md index 7067e63..9f5cd39 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ # Supported tags and respective `Dockerfile` links -- [`0.5.1`, `latest`](https://github.com/sysflow-telemetry/sf-apis/tree/0.5.1), [`edge`](https://github.com/sysflow-telemetry/sf-apis/tree/master), [`dev`](https://github.com/sysflow-telemetry/sf-apis/tree/dev) +- [`0.6.0`, `latest`](https://github.com/sysflow-telemetry/sf-apis/tree/0.6.0), [`edge`](https://github.com/sysflow-telemetry/sf-apis/tree/master), [`dev`](https://github.com/sysflow-telemetry/sf-apis/tree/dev) # Quick reference @@ -82,6 +82,12 @@ Note: If classic Jupyter notebook is preferred, run: docker run --rm -d --name sfnb -v $(pwd)/pynb:/home/jovyan/work -p 8888:8888 -e DOCKER_STACKS_JUPYTER_CMD=notebook sysflowtelemetry/sfnb ``` +Permission-specific configurations when volume mounting a directory: + +``` +docker run --rm -d --name sfnb -e NB_USER= --user root -e CHOWN_HOME=yes -e NB_UID=$(id -u) -e NB_GID=$(id -g) -e CHOWN_HOME_OPTS=‘-R’ -w /home// -v $(pwd)/pynb:/home//work sysflowtelemetry/sfnb +``` + # License View [license information](https://github.com/sysflow-telemetry/sf-apis/blob/master/LICENSE.md) for the software contained in this image. diff --git a/go/go.mod b/go/go.mod index 99dd511..bd42815 100644 --- a/go/go.mod +++ b/go/go.mod @@ -1,6 +1,6 @@ module github.com/sysflow-telemetry/sf-apis/go -go 1.17 +go 1.19 require ( github.com/actgardner/gogen-avro/v7 v7.3.1 @@ -22,9 +22,9 @@ require ( github.com/spf13/cast v1.4.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/subosito/gotenv v1.2.0 // indirect - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect - golang.org/x/text v0.3.7 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect + golang.org/x/text v0.3.8 // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go/ioutils/ioutils.go b/go/ioutils/ioutils.go index 75b31a1..f20697f 100644 --- a/go/ioutils/ioutils.go +++ b/go/ioutils/ioutils.go @@ -21,7 +21,6 @@ package ioutils import ( - "io/ioutil" "os" "path/filepath" ) @@ -29,32 +28,83 @@ import ( // ListFilePaths lists file paths with extension fileExt in path if // path is a valid directory, otherwise, it returns path if path is // a valid path and has extension fileExt. -func ListFilePaths(path string, fileExt string) ([]string, error) { +func ListFilePaths(path string, fileExts ...string) ([]string, error) { var paths []string if fi, err := os.Stat(path); os.IsNotExist(err) { return paths, err } else if fi.IsDir() { - var files []os.FileInfo + var entries []os.DirEntry var err error - if files, err = ioutil.ReadDir(path); err != nil { + if entries, err = os.ReadDir(path); err != nil { return paths, err } - for _, file := range files { - if filepath.Ext(file.Name()) == fileExt { - f := path + "/" + file.Name() + for _, entry := range entries { + if entry.IsDir() { + continue + } + if len(fileExts) > 0 { + for _, fileExt := range fileExts { + if filepath.Ext(entry.Name()) == fileExt { + f := path + "/" + entry.Name() + paths = append(paths, f) + } + } + } else { + f := path + "/" + entry.Name() paths = append(paths, f) } } - return paths, nil } else { - if filepath.Ext(path) == fileExt { - return append(paths, path), nil + for _, fileExt := range fileExts { + if filepath.Ext(path) == fileExt { + return append(paths, path), nil + } + } + } + return paths, nil +} + +// ListRecursiveFilePaths recursively lists file paths with extension +// fileExt in path if path is a valid directory, otherwise, it returns +// path if path is a valid path and has extension fileExt. +func ListRecursiveFilePaths(path string, fileExts ...string) ([]string, error) { + var paths []string + if fi, err := os.Stat(path); os.IsNotExist(err) { + return paths, err + } else if fi.IsDir() { + err := filepath.Walk(path, + func(p string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + if len(fileExts) > 0 { + for _, fileExt := range fileExts { + if filepath.Ext(info.Name()) == fileExt { + paths = append(paths, p) + } + } + } else { + paths = append(paths, p) + } + return nil + }) + if err != nil { + return paths, err + } + } else { + for _, fileExt := range fileExts { + if filepath.Ext(path) == fileExt { + return append(paths, path), nil + } } - return paths, nil } + return paths, nil } -//FileExists checks whether a file exists and whether it is a directory. +// FileExists checks whether a file exists and whether it is a directory. func FileExists(filename string) (bool, bool) { info, err := os.Stat(filename) if os.IsNotExist(err) { diff --git a/go/logger/logger.go b/go/logger/logger.go index 517ed0a..00a4362 100644 --- a/go/logger/logger.go +++ b/go/logger/logger.go @@ -23,7 +23,6 @@ package logger import ( "fmt" "io" - "io/ioutil" "log" "os" "strings" @@ -42,6 +41,9 @@ const ( QUIET ) +// Perf logger string. +const perf string = "Perf" + func (d LogLevel) String() string { return [...]string{"Trace", "Info", "Warn", "Error", "Health", "Quiet"}[d] } @@ -62,6 +64,7 @@ var ( Warn *log.Logger Error *log.Logger Health *log.Logger + Perf *log.Logger ) // InitLoggers initialize utility loggers with default i/o streams. @@ -70,17 +73,17 @@ func InitLoggers(level LogLevel) { case TRACE: initLoggers(os.Stdout, os.Stdout, os.Stdout, os.Stderr, os.Stdout) case INFO: - initLoggers(ioutil.Discard, os.Stdout, os.Stdout, os.Stderr, os.Stdout) + initLoggers(io.Discard, os.Stdout, os.Stdout, os.Stderr, os.Stdout) case WARN: - initLoggers(ioutil.Discard, ioutil.Discard, os.Stdout, os.Stderr, os.Stdout) + initLoggers(io.Discard, io.Discard, os.Stdout, os.Stderr, os.Stdout) case ERROR: - initLoggers(ioutil.Discard, ioutil.Discard, ioutil.Discard, os.Stderr, os.Stdout) + initLoggers(io.Discard, io.Discard, io.Discard, os.Stderr, os.Stdout) case HEALTH: - initLoggers(ioutil.Discard, ioutil.Discard, ioutil.Discard, ioutil.Discard, os.Stdout) + initLoggers(io.Discard, io.Discard, io.Discard, io.Discard, os.Stdout) case QUIET: - initLoggers(ioutil.Discard, ioutil.Discard, ioutil.Discard, ioutil.Discard, ioutil.Discard) + initLoggers(io.Discard, io.Discard, io.Discard, io.Discard, io.Discard) default: - initLoggers(ioutil.Discard, os.Stdout, os.Stdout, os.Stderr, os.Stdout) + initLoggers(io.Discard, os.Stdout, os.Stdout, os.Stderr, os.Stdout) } } @@ -110,4 +113,24 @@ func initLoggers( Health = log.New(healthHandle, fmt.Sprintf("[%s] ", HEALTH), log.Ldate|log.Ltime|log.Lshortfile) + + SetPerfLogger(false) +} + +// SetPerfLogger changes the state of the performance logger. This logger is independent of log levels (default: disabled). +func SetPerfLogger(enabled bool) { + var iowriter io.Writer + if enabled { + iowriter = os.Stdout + } else { + iowriter = io.Discard + } + Perf = log.New(iowriter, + fmt.Sprintf("[%s] ", perf), + log.Ldate|log.Ltime|log.Lshortfile) +} + +// IsEnabled checks whether a logger is enabled. +func IsEnabled(logger *log.Logger) bool { + return logger != nil && logger.Writer() != io.Discard } diff --git a/go/plugins/pipeline.go b/go/plugins/pipeline.go index ff0ee37..0c6189f 100644 --- a/go/plugins/pipeline.go +++ b/go/plugins/pipeline.go @@ -32,6 +32,6 @@ type SFPipeline interface { GetNumProcessors() int GetNumHandlers() int GetPluginCache() SFPluginCache - GetChannel(name string)(interface{}, error) + GetChannel(name string) (interface{}, error) Print() } diff --git a/go/plugins/types.go b/go/plugins/types.go index e57f6dc..6558785 100644 --- a/go/plugins/types.go +++ b/go/plugins/types.go @@ -38,12 +38,7 @@ type CtxSysFlow struct { GraphletID uint64 } -// CtxSFChannel defines a Contextual SysFlow channel for data transfer. -type CtxSFChannel struct { - In chan *CtxSysFlow -} - -// SFChannel defines a SysFlow channel for data transfer. -type SFChannel struct { - In chan *sfgo.SysFlow +// Channel type +type Channel[R any] struct { + In chan R } diff --git a/go/secrets/secrets.go b/go/secrets/secrets.go index 0cb97aa..7a5f14e 100644 --- a/go/secrets/secrets.go +++ b/go/secrets/secrets.go @@ -23,7 +23,6 @@ package secrets import ( "encoding/base64" "fmt" - "io/ioutil" "os" "strings" @@ -76,7 +75,7 @@ func (s *Secrets) read(secret string) (string, error) { if v, ok := s.secrets[secret]; ok { return v, nil } - buf, err := ioutil.ReadFile(s.secretsDir + "/" + secret) + buf, err := os.ReadFile(s.secretsDir + "/" + secret) if err != nil { return sfgo.Zeros.String, fmt.Errorf("secret %s does not exist or cannot be read: %v", secret, err) } @@ -88,9 +87,9 @@ func (s *Secrets) read(secret string) (string, error) { // Checks if the given path is a directory. Returns nil if directory. func isDir(path string) error { if fi, err := os.Stat(path); os.IsNotExist(err) { - return fmt.Errorf("Path %s not found", path) + return fmt.Errorf("path %s not found", path) } else if !fi.Mode().IsDir() { - return fmt.Errorf("Path %s is not a directory", path) + return fmt.Errorf("path %s is not a directory", path) } return nil } diff --git a/py3/setup.cfg b/py3/setup.cfg index c1c4899..92c1ae1 100644 --- a/py3/setup.cfg +++ b/py3/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = sysflow-tools -version = 0.5.1 +version = 0.6.0-rc1 description = SysFlow APIs and utilities long_description = file:README.md long_description_content_type = text/markdown