diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 34321b61161..147ca45ced1 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -1,5 +1,137 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_RHEL9: "family/core-rhel-9" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + steps: - - label: "Example test" - command: echo "Hello!" + - group: "Auditbeat Mandatory Testing" + key: "mandatory-tests" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" + + steps: + - label: ":ubuntu: Unit Tests" + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "auditbeat: Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":rhel: Unit Tests" + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "auditbeat: Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":windows:-{{matrix.image}} Unit Tests" + command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" + notify: + - github_commit_status: + context: "auditbeat: Unit Tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":linux: Crosscompile" + command: + - ".buildkite/auditbeat/scripts/crosscompile.sh" + env: + GOX_FLAGS: "-arch amd64" + notify: + - github_commit_status: + context: "auditbeat: Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + + - group: "Extended Testing" + key: "extended-tests" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for extended support" + + steps: + - label: ":linux: ARM64 Unit Tests" + key: "arm-extended" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "auditbeat/Extended: Unit Tests ARM" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.large" + artifact_paths: "auditbeat/build/*.xml" + + - label: ":mac: MacOS Unit Tests" + key: "macos-extended" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "auditbeat/Extended: MacOS Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "auditbeat/build/*.xml" + + - group: "Windows Extended Testing" + key: "extended-tests-win" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ + + steps: + - label: ":windows: Win 2019 Unit Tests" + key: "win-extended-2019" + command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" + notify: + - github_commit_status: + context: "auditbeat/Extended: Win-2019 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - group: "Packaging" + key: "packaging" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: + - "mandatory-tests" + + steps: + - label: Package pipeline + commands: ".buildkite/auditbeat/scripts/package-step.sh | buildkite-agent pipeline upload" diff --git a/.buildkite/auditbeat/scripts/crosscompile.sh b/.buildkite/auditbeat/scripts/crosscompile.sh new file mode 100755 index 00000000000..866d6be4223 --- /dev/null +++ b/.buildkite/auditbeat/scripts/crosscompile.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo "--- Executing Crosscompile" +make -C auditbeat crosscompile diff --git a/.buildkite/auditbeat/scripts/package-step.sh b/.buildkite/auditbeat/scripts/package-step.sh new file mode 100755 index 00000000000..02124058992 --- /dev/null +++ b/.buildkite/auditbeat/scripts/package-step.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/util.sh + +changeset="^auditbeat/ +^go.mod +^pytest.ini +^dev-tools/ +^libbeat/ +^testing/ +^\.buildkite/auditbeat/" + +if are_files_changed "$changeset"; then + cat <<-EOF + steps: + - label: ":ubuntu: Packaging Linux X86" + key: "package-linux-x86" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + command: + - ".buildkite/auditbeat/scripts/package.sh" + notify: + - github_commit_status: + context: "Auditbeat/Packaging: Linux X86" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + + - label: ":linux: Packaging Linux ARM" + key: "package-linux-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + command: + - ".buildkite/auditbeat/scripts/package.sh" + notify: + - github_commit_status: + context: "Auditbeat/Packaging: ARM" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.large" +EOF +fi diff --git a/.buildkite/auditbeat/scripts/package.sh b/.buildkite/auditbeat/scripts/package.sh new file mode 100755 index 00000000000..71872ca15a3 --- /dev/null +++ b/.buildkite/auditbeat/scripts/package.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo "--- Docker Version: $(docker --version)" + +echo "--- Start Packaging" +cd auditbeat +umask 0022 +mage package + diff --git a/.buildkite/auditbeat/scripts/unit-tests-win.ps1 b/.buildkite/auditbeat/scripts/unit-tests-win.ps1 new file mode 100644 index 00000000000..200627d518f --- /dev/null +++ b/.buildkite/auditbeat/scripts/unit-tests-win.ps1 @@ -0,0 +1,51 @@ +$ErrorActionPreference = "Stop" # set -e +$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually + +# Forcing to checkout again all the files with a correct autocrlf. +# Doing this here because we cannot set git clone options before. +function fixCRLF() { + Write-Host "--- Fixing CRLF in git checkout --" + git config core.autocrlf false + git rm --quiet --cached -r . + git reset --quiet --hard +} + +function withGolang() { + Write-Host "--- Install golang $GoVersion --" + choco install golang -y --version $GoVersion + + $choco = Convert-Path "$((Get-Command choco).Path)\..\.." + Import-Module "$choco\helpers\chocolateyProfile.psm1" + refreshenv + go version + go env +} + +function installGoDependencies() { + $installPackages = @( + "github.com/magefile/mage" + "github.com/elastic/go-licenser" + "golang.org/x/tools/cmd/goimports" + "github.com/jstemmer/go-junit-report" + "github.com/tebeka/go2xunit" + ) + foreach ($pkg in $installPackages) { + go install "$pkg" + } +} + +fixCRLF + +$ErrorActionPreference = "Continue" # set +e + +Set-Location -Path auditbeat +New-Item -ItemType Directory -Force -Path "build" +withGolang +installGoDependencies + +mage build unitTest + +$EXITCODE=$LASTEXITCODE +$ErrorActionPreference = "Stop" + +Exit $EXITCODE diff --git a/.buildkite/auditbeat/scripts/unit-tests.sh b/.buildkite/auditbeat/scripts/unit-tests.sh new file mode 100755 index 00000000000..c1f5685c77f --- /dev/null +++ b/.buildkite/auditbeat/scripts/unit-tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo "--- Running Unit Tests" +sudo chmod -R go-w auditbeat/ + +cd auditbeat +umask 0022 +mage build unitTest diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh index d94d03aad53..4dfc01bafc3 100644 --- a/.buildkite/env-scripts/env.sh +++ b/.buildkite/env-scripts/env.sh @@ -5,9 +5,15 @@ WORKSPACE="$(pwd)" BIN="${WORKSPACE}/bin" HW_TYPE="$(uname -m)" PLATFORM_TYPE="$(uname)" +REPO="beats" +TMP_FOLDER="tmp.${REPO}" +DOCKER_REGISTRY="docker.elastic.co" export SETUP_GVM_VERSION export WORKSPACE export BIN export HW_TYPE export PLATFORM_TYPE +export REPO +export TMP_FOLDER +export DOCKER_REGISTRY diff --git a/.buildkite/env-scripts/linux-env.sh b/.buildkite/env-scripts/linux-env.sh index edaf1a3100c..5e6e5f7cbf0 100644 --- a/.buildkite/env-scripts/linux-env.sh +++ b/.buildkite/env-scripts/linux-env.sh @@ -1,24 +1,47 @@ #!/usr/bin/env bash +set -euo pipefail + source .buildkite/env-scripts/util.sh DEBIAN_FRONTEND="noninteractive" -export DEBIAN_FRONTEND - sudo mkdir -p /etc/needrestart echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null -# Remove this code once beats specific agent is set up +echo "--- PLATFORM TYPE $PLATFORM_TYPE" + if [[ $PLATFORM_TYPE == "Linux" ]]; then - echo ":: Installing libs ::" - sudo apt-get update - sudo apt-get install -y libsystemd-dev - sudo apt install -y python3-pip - sudo apt-get install -y python3-venv + # Remove this code once beats specific agent is set up + if grep -q 'Ubuntu' /etc/*release; then + export DEBIAN_FRONTEND + + echo "--- Ubuntu - Installing libs" + sudo apt-get update + sudo apt-get install -y libsystemd-dev + sudo apt install -y python3-pip + sudo apt-get install -y python3-venv + fi + + # Remove this code once beats specific agent is set up + if grep -q 'Red Hat' /etc/*release; then + echo "--- RHL - Installing libs" + sudo yum update -y + sudo yum install -y systemd-devel + sudo yum install -y python3-pip + sudo yum install -y python3 + pip3 install virtualenv + fi +fi + +if [[ $PLATFORM_TYPE == Darwin* ]]; then + echo "--- Setting larger ulimit on MacOS" + # To bypass file descriptor errors like "Too many open files error" on MacOS + ulimit -Sn 50000 + echo "--- ULIMIT: $(ulimit -n)" fi -echo ":: Setting up environment ::" +echo "--- Setting up environment" add_bin_path with_go with_mage diff --git a/.buildkite/env-scripts/macos-env.sh b/.buildkite/env-scripts/macos-env.sh deleted file mode 100644 index ac1486b64fd..00000000000 --- a/.buildkite/env-scripts/macos-env.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -if [[ $PLATFORM_TYPE == Darwin* ]]; then - echo ":: Setting larger ulimit on MacOS ::" - # To bypass file descriptor errors like "Too many open files error" on MacOS - ulimit -Sn 50000 - echo ":: ULIMIT :: $(ulimit -n)" -fi diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh index 157a5aff37a..7aef69cff38 100644 --- a/.buildkite/env-scripts/util.sh +++ b/.buildkite/env-scripts/util.sh @@ -89,3 +89,17 @@ are_files_changed() { return 1; fi } + +cleanup() { + echo "Deleting temporary files..." + rm -rf ${BIN}/${TMP_FOLDER}.* + echo "Done." +} + +unset_secrets () { + for var in $(printenv | sed 's;=.*;;' | sort); do + if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then + unset "$var" + fi + done +} diff --git a/.buildkite/env-scripts/win-env.sh b/.buildkite/env-scripts/win-env.sh index aa5f67ca4ce..ccf5479b46e 100644 --- a/.buildkite/env-scripts/win-env.sh +++ b/.buildkite/env-scripts/win-env.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -install_python_win() { - if [[ ${PLATFORM_TYPE} = MINGW* ]]; then - choco install mingw -y - choco install python --version=3.11.0 -y - fi -} +echo "--- PLATFORM TYPE: ${PLATFORM_TYPE}" +if [[ ${PLATFORM_TYPE} = MINGW* ]]; then + echo "--- Installing Python on Win" + choco install mingw -y + choco install python --version=3.11.0 -y +fi diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index e3d7384a71e..eda9fb93a66 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -134,8 +134,6 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: - "mandatory-tests" - - "extended-tests" - - "extended-tests-win" steps: - label: Package pipeline diff --git a/.buildkite/filebeat/scripts/integration-gotests.sh b/.buildkite/filebeat/scripts/integration-gotests.sh index a3eabf70c0d..d64ce7c98eb 100755 --- a/.buildkite/filebeat/scripts/integration-gotests.sh +++ b/.buildkite/filebeat/scripts/integration-gotests.sh @@ -4,7 +4,7 @@ set -euo pipefail source .buildkite/env-scripts/linux-env.sh -echo ":: Execute Integration Tests ::" +echo "--- Executing Integration Tests" sudo chmod -R go-w filebeat/ cd filebeat diff --git a/.buildkite/filebeat/scripts/integration-pytests.sh b/.buildkite/filebeat/scripts/integration-pytests.sh index 5e2e403dda8..b51e8ae18a6 100755 --- a/.buildkite/filebeat/scripts/integration-pytests.sh +++ b/.buildkite/filebeat/scripts/integration-pytests.sh @@ -4,7 +4,7 @@ set -euo pipefail source .buildkite/env-scripts/linux-env.sh -echo ":: Execute Integration Tests ::" +echo "--- Executing Integration Tests" sudo chmod -R go-w filebeat/ cd filebeat diff --git a/.buildkite/filebeat/scripts/package-step.sh b/.buildkite/filebeat/scripts/package-step.sh index a4127c3cd1d..985125433ce 100755 --- a/.buildkite/filebeat/scripts/package-step.sh +++ b/.buildkite/filebeat/scripts/package-step.sh @@ -5,12 +5,12 @@ set -euo pipefail source .buildkite/env-scripts/util.sh changeset="^filebeat/ - ^go.mod - ^pytest.ini - ^dev-tools/ - ^libbeat/ - ^testing/ - ^\.buildkite/filebeat/" +^go.mod +^pytest.ini +^dev-tools/ +^libbeat/ +^testing/ +^\.buildkite/filebeat/" if are_files_changed "$changeset"; then cat <<-EOF diff --git a/.buildkite/filebeat/scripts/package.sh b/.buildkite/filebeat/scripts/package.sh index 2ae226eb739..0bb03250348 100755 --- a/.buildkite/filebeat/scripts/package.sh +++ b/.buildkite/filebeat/scripts/package.sh @@ -4,9 +4,7 @@ set -euo pipefail source .buildkite/env-scripts/linux-env.sh -echo ":: Evaluate Filebeat Changes ::" - -echo ":: Start Packaging ::" +echo "--- Start Packaging" cd filebeat umask 0022 mage package diff --git a/.buildkite/filebeat/scripts/unit-tests.sh b/.buildkite/filebeat/scripts/unit-tests.sh index cda1dd85aea..08ce9d4ea1c 100755 --- a/.buildkite/filebeat/scripts/unit-tests.sh +++ b/.buildkite/filebeat/scripts/unit-tests.sh @@ -3,9 +3,8 @@ set -euo pipefail source .buildkite/env-scripts/linux-env.sh -source .buildkite/env-scripts/macos-env.sh -echo ":: Execute Unit Tests ::" +echo "--- Executing Unit Tests" sudo chmod -R go-w filebeat/ umask 0022 diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index e10f15de7b6..b6cc7ad60bd 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index ef38478a432..c448a710251 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,15 +2,11 @@ set -euo pipefail -if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" ]]; then source .buildkite/env-scripts/env.sh source .buildkite/env-scripts/util.sh source .buildkite/env-scripts/win-env.sh - if [[ ${PLATFORM_TYPE} = MINGW* ]]; then - install_python_win - fi - if [[ -z "${GOLANG_VERSION-""}" ]]; then export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version") fi diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 4607a0576d0..66c508e252c 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -57,8 +57,8 @@ "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, - "trigger_comment_regex": "^/test auditbeat$", - "always_trigger_comment_regex": "^/test auditbeat$", + "trigger_comment_regex": "^/test auditbeat(for (arm|macos|windows|extended support))?$", + "always_trigger_comment_regex": "^/test auditbeat(for (arm|macos|windows|extended support))?$", "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], @@ -143,6 +143,22 @@ "skip_target_branches": [ ], "skip_ci_on_only_changed": ["^x-pack/elastic-agent/README.md", "^x-pack/elastic-agent/docs/.*", "^x-pack/elastic-agent/devtools/.*" ], "always_require_ci_on_changed": ["^x-pack/elastic-agent/.*", ".buildkite/x-pack/elastic-agent/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] + }, + { + "enabled": true, + "pipelineSlug": "beats-winlogbeat", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^/test winlogbeat$", + "always_trigger_comment_regex": "^/test winlogbeat$", + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^winlogbeat/.*", ".buildkite/winlogbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] } ] } diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml new file mode 100644 index 00000000000..34321b61161 --- /dev/null +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + +steps: + - label: "Example test" + command: echo "Hello!" diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 348aab26709..3c5a568569a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -145,6 +145,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d *Auditbeat* - Added `add_session_metadata` processor, which enables session viewer on Auditbeat data. {pull}37640[37640] +- Add linux capabilities to processes in the system/process. {pull}37453[37453] *Filebeat* @@ -189,6 +190,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Relax TCP/UDP metric polling expectations to improve metric collection. {pull}37714[37714] - Add support for PEM-based Okta auth in HTTPJSON. {pull}37772[37772] - Prevent complete loss of long request trace data. {issue}37826[37826] {pull}37836[37836] +- Add support for PEM-based Okta auth in CEL. {pull}37813[37813] *Auditbeat* diff --git a/auditbeat/docs/fields.asciidoc b/auditbeat/docs/fields.asciidoc index bd4db4ce5b6..9eee5f008fc 100644 --- a/auditbeat/docs/fields.asciidoc +++ b/auditbeat/docs/fields.asciidoc @@ -18925,6 +18925,28 @@ type: keyword -- +*`process.thread.capabilities.effective`*:: ++ +-- +This is the set of capabilities used by the kernel to perform permission checks for the thread. + +type: keyword + +example: ["CAP_BPF", "CAP_SYS_ADMIN"] + +-- + +*`process.thread.capabilities.permitted`*:: ++ +-- +This is a limiting superset for the effective capabilities that the thread may assume. + +type: keyword + +example: ["CAP_BPF", "CAP_SYS_ADMIN"] + +-- + [float] === hash diff --git a/catalog-info.yaml b/catalog-info.yaml index f3dd3094788..037d92e415c 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -413,9 +413,54 @@ spec: cancel_intermediate_builds: true cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-beats-winlogbeat + description: "Beats winlogbeat pipeline" + links: + - title: Pipeline + url: https://buildkite.com/elastic/beats-winlogbeat + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: beats-winlogbeat + description: "Beats winlogbeat pipeline" + spec: +# branch_configuration: "main 7.17 8.*" TODO: temporarily commented to build PRs from forks + pipeline_file: ".buildkite/winlogbeat/pipeline.winlogbeat.yml" +# maximum_timeout_in_minutes: 120 TODO: uncomment when pipeline is ready + provider_settings: + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_tags: true + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) + repository: elastic/beats + cancel_intermediate_builds: true + cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds: true + skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + # env: + # ELASTIC_PR_COMMENTS_ENABLED: "true" TODO: uncomment when pipeline is ready teams: ingest-fp: access_level: MANAGE_BUILD_AND_READ everyone: - access_level: READ_ONLY \ No newline at end of file + access_level: READ_ONLY diff --git a/filebeat/input/journald/pkg/journalfield/conv.go b/filebeat/input/journald/pkg/journalfield/conv.go index bd7403ae142..94447b773b7 100644 --- a/filebeat/input/journald/pkg/journalfield/conv.go +++ b/filebeat/input/journald/pkg/journalfield/conv.go @@ -19,11 +19,11 @@ package journalfield import ( "fmt" - "math/bits" "regexp" "strconv" "strings" + "github.com/elastic/beats/v7/libbeat/common/capabilities" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -190,72 +190,13 @@ func expandCapabilities(fields mapstr.M) { if !ok { return } - w, err := strconv.ParseUint(c, 16, 64) - if err != nil { - return - } - if w == 0 { + caps, err := capabilities.FromString(c, 16) + if err != nil || len(caps) == 0 { return } - caps := make([]string, 0, bits.OnesCount64(w)) - for i := 0; w != 0; i++ { - if w&1 != 0 { - if i < len(capTable) { - caps = append(caps, capTable[i]) - } else { - caps = append(caps, strconv.Itoa(i)) - } - } - w >>= 1 - } fields.Put("process.thread.capabilities.effective", caps) } -// include/uapi/linux/capability.h -var capTable = [...]string{ - 0: "CAP_CHOWN", - 1: "CAP_DAC_OVERRIDE", - 2: "CAP_DAC_READ_SEARCH", - 3: "CAP_FOWNER", - 4: "CAP_FSETID", - 5: "CAP_KILL", - 6: "CAP_SETGID", - 7: "CAP_SETUID", - 8: "CAP_SETPCAP", - 9: "CAP_LINUX_IMMUTABLE", - 10: "CAP_NET_BIND_SERVICE", - 11: "CAP_NET_BROADCAST", - 12: "CAP_NET_ADMIN", - 13: "CAP_NET_RAW", - 14: "CAP_IPC_LOCK", - 15: "CAP_IPC_OWNER", - 16: "CAP_SYS_MODULE", - 17: "CAP_SYS_RAWIO", - 18: "CAP_SYS_CHROOT", - 19: "CAP_SYS_PTRACE", - 20: "CAP_SYS_PACCT", - 21: "CAP_SYS_ADMIN", - 22: "CAP_SYS_BOOT", - 23: "CAP_SYS_NICE", - 24: "CAP_SYS_RESOURCE", - 25: "CAP_SYS_TIME", - 26: "CAP_SYS_TTY_CONFIG", - 27: "CAP_MKNOD", - 28: "CAP_LEASE", - 29: "CAP_AUDIT_WRITE", - 30: "CAP_AUDIT_CONTROL", - 31: "CAP_SETFCAP", - 32: "CAP_MAC_OVERRIDE", - 33: "CAP_MAC_ADMIN", - 34: "CAP_SYSLOG", - 35: "CAP_WAKE_ALARM", - 36: "CAP_BLOCK_SUSPEND", - 37: "CAP_AUDIT_READ", - 38: "CAP_PERFMON", - 39: "CAP_BPF", - 40: "CAP_CHECKPOINT_RESTORE", -} - func getStringFromFields(key string, fields mapstr.M) string { value, _ := fields.GetValue(key) str, _ := value.(string) diff --git a/filebeat/input/journald/pkg/journalfield/conv_expand_test.go b/filebeat/input/journald/pkg/journalfield/conv_expand_test.go index c43e57a1c49..09daf7c8f5b 100644 --- a/filebeat/input/journald/pkg/journalfield/conv_expand_test.go +++ b/filebeat/input/journald/pkg/journalfield/conv_expand_test.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +//go:build linux && cgo + package journalfield import ( @@ -228,8 +230,8 @@ var expandCapabilitiesTests = []struct { "CAP_PERFMON", "CAP_BPF", "CAP_CHECKPOINT_RESTORE", - "41", - "42", + "CAP_41", + "CAP_42", }, }, }, diff --git a/libbeat/common/capabilities/capabilities_linux.go b/libbeat/common/capabilities/capabilities_linux.go new file mode 100644 index 00000000000..715b86d9bc7 --- /dev/null +++ b/libbeat/common/capabilities/capabilities_linux.go @@ -0,0 +1,161 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package capabilities + +import ( + "errors" + "math/bits" + "strconv" + "strings" + + "kernel.org/pub/linux/libs/security/libcap/cap" +) + +var ( + // errInvalidCapability expresses an invalid capability ID: x < 0 || x >= 64. + errInvalidCapability = errors.New("invalid capability") +) + +// The capability set flag/vector, re-exported from +// libcap(3). Inherit, Bound & Ambient not exported since we have no +// use for it yet. +type Flag = cap.Flag + +const ( + // aka CapEff + Effective = cap.Effective + // aka CapPrm + Permitted = cap.Permitted +) + +// Fetch the capabilities of pid for a given flag/vector and convert +// it to the representation used in ECS. cap.GetPID() fetches it with +// SYS_CAPGET. +// Returns errors.ErrUnsupported on "not linux". +func FromPid(flag Flag, pid int) ([]string, error) { + set, err := cap.GetPID(pid) + if err != nil { + return nil, err + } + empty, err := isEmpty(flag, set) + if err != nil { + return nil, err + } + if empty { + return []string{}, nil + } + + sl := make([]string, 0, cap.MaxBits()) + for i := 0; i < int(cap.MaxBits()); i++ { + c := cap.Value(i) + enabled, err := set.GetFlag(flag, c) + if err != nil { + return nil, err + } + if !enabled { + continue + } + s, err := toECS(i) + // impossible since MaxBits <= 64 + if err != nil { + return nil, err + } + sl = append(sl, s) + } + + return sl, err +} + +// Convert a uint64 to the capabilities representation used in ECS. +// Returns errors.ErrUnsupported on "not linux". +func FromUint64(w uint64) ([]string, error) { + sl := make([]string, 0, bits.OnesCount64(w)) + for i := 0; w != 0; i++ { + if w&1 != 0 { + s, err := toECS(i) + // impossible since MaxBits <= 64 + if err != nil { + return nil, err + } + sl = append(sl, s) + } + w >>= 1 + } + + return sl, nil +} + +// Convert a string to the capabilities representation used in +// ECS. Example input: "1ffffffffff", 16. +// Returns errors.ErrUnsupported on "not linux". +func FromString(s string, base int) ([]string, error) { + w, err := strconv.ParseUint(s, 16, 64) + if err != nil { + return nil, err + } + + return FromUint64(w) +} + +// True if sets are equal for the given flag/vector, errors out in +// case any of the sets is malformed. +func isEqual(flag Flag, a *cap.Set, b *cap.Set) (bool, error) { + d, err := a.Cf(b) + if err != nil { + return false, err + } + + return !d.Has(flag), nil +} + +// Convert the capability ID to a string suitable to be used in +// ECS. +// If capabiliy ID X is unknown, but valid (0 <= X < 64), "CAP_X" +// will be returned instead. Fetches from an internal table built at +// startup. +var toECS = makeToECS() + +// Make toECS() which creates a map of every possible valid capability +// ID on startup. Returns errInvalidCapabilty for an invalid ID. +func makeToECS() func(int) (string, error) { + ecsNames := make(map[int]string) + + for i := 0; i < 64; i++ { + c := cap.Value(i) + if i < int(cap.MaxBits()) { + ecsNames[i] = strings.ToUpper(c.String()) + } else { + ecsNames[i] = strings.ToUpper("CAP_" + c.String()) + } + } + + return func(b int) (string, error) { + s, ok := ecsNames[b] + if !ok { + return "", errInvalidCapability + } + return s, nil + } +} + +// Like isAll(), but for the empty set, here for symmetry. +func isEmpty(flag Flag, set *cap.Set) (bool, error) { + return isEqual(flag, set, cap.NewSet()) +} diff --git a/libbeat/common/capabilities/capabilities_linux_test.go b/libbeat/common/capabilities/capabilities_linux_test.go new file mode 100644 index 00000000000..1481fc5679b --- /dev/null +++ b/libbeat/common/capabilities/capabilities_linux_test.go @@ -0,0 +1,87 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package capabilities + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "kernel.org/pub/linux/libs/security/libcap/cap" +) + +func TestEmpty(t *testing.T) { + sl, err := FromString("0", 16) + assert.Nil(t, err) + assert.Equal(t, len(sl), 0) + + sl, err = FromUint64(0) + assert.Nil(t, err) + assert.Equal(t, len(sl), 0) + + // assumes non root has no capabilities + if os.Geteuid() != 0 { + empty := cap.NewSet() + self := cap.GetProc() + d, err := self.Cf(empty) + assert.Nil(t, err) + assert.False(t, d.Has(cap.Effective)) + assert.False(t, d.Has(cap.Permitted)) + assert.False(t, d.Has(cap.Inheritable)) + } +} + +func TestOverflow(t *testing.T) { + sl, err := FromUint64(^uint64(0)) + assert.Nil(t, err) + assert.Equal(t, len(sl), 64) + + for _, cap := range []string{ + "CAP_CHOWN", + "CAP_DAC_OVERRIDE", + "CAP_DAC_READ_SEARCH", + "CAP_FOWNER", + "CAP_FSETID", + "CAP_KILL", + "CAP_SETGID", + "CAP_SYS_MODULE", + "CAP_SYS_RAWIO", + "CAP_IPC_LOCK", + "CAP_MAC_OVERRIDE", + } { + assertHasCap(t, sl, cap) + } + if cap.MaxBits() <= 62 { + assertHasCap(t, sl, "CAP_62") + } + if cap.MaxBits() <= 63 { + assertHasCap(t, sl, "CAP_63") + } +} + +func assertHasCap(t *testing.T, sl []string, s string) { + var found int + + for _, s2 := range sl { + if s2 == s { + found++ + } + } + + assert.Equal(t, found, 1, s) +} diff --git a/libbeat/common/capabilities/capabilities_other.go b/libbeat/common/capabilities/capabilities_other.go new file mode 100644 index 00000000000..fbd7e879772 --- /dev/null +++ b/libbeat/common/capabilities/capabilities_other.go @@ -0,0 +1,47 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !linux + +package capabilities + +import "errors" + +// Dummy value on "not linux". +type Flag = uint + +const ( + // Meaningless on "not linux". + Effective = Flag(0) + // Meaningless on "not linux". + Permitted = Flag(1) +) + +// Returns errors.ErrUnsupported on "not linux". +func FromPid(flag Flag, pid int) ([]string, error) { + return nil, errors.ErrUnsupported +} + +// Returns errors.ErrUnsupported on "not linux". +func FromUint64(w uint64) ([]string, error) { + return nil, errors.ErrUnsupported +} + +// Returns errors.ErrUnsupported on "not linux". +func FromString(s string, base int) ([]string, error) { + return nil, errors.ErrUnsupported +} diff --git a/libbeat/common/capabilities_linux.go b/libbeat/common/capabilities_linux.go deleted file mode 100644 index b2992c251ef..00000000000 --- a/libbeat/common/capabilities_linux.go +++ /dev/null @@ -1,66 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build linux - -package common - -import ( - "errors" - "fmt" - - "github.com/elastic/go-sysinfo" - "github.com/elastic/go-sysinfo/types" -) - -// Capabilities contains the capability sets of a process -type Capabilities types.CapabilityInfo - -// Check performs a permission check for a given capabilities set -func (c Capabilities) Check(set []string) bool { - for _, capability := range set { - found := false - for _, effective := range c.Effective { - if capability == effective { - found = true - break - } - } - if !found { - return false - } - } - return true -} - -// GetCapabilities gets the capabilities of this process -func GetCapabilities() (Capabilities, error) { - p, err := sysinfo.Self() - if err != nil { - return Capabilities{}, fmt.Errorf("failed to read self process information: %w", err) - } - - if c, ok := p.(types.Capabilities); ok { - capabilities, err := c.Capabilities() - if err != nil { - return Capabilities{}, fmt.Errorf("failed to read process capabilities: %w", err) - } - return Capabilities(*capabilities), nil - } - - return Capabilities{}, errors.New("capabilities not available") -} diff --git a/libbeat/common/seccomp/policy_linux_386.go b/libbeat/common/seccomp/policy_linux_386.go index 72466698720..ac2a93a5c74 100644 --- a/libbeat/common/seccomp/policy_linux_386.go +++ b/libbeat/common/seccomp/policy_linux_386.go @@ -31,6 +31,7 @@ func init() { "_llseek", "access", "brk", + "capget", "chmod", "chown", "clock_gettime", diff --git a/libbeat/common/seccomp/policy_linux_amd64.go b/libbeat/common/seccomp/policy_linux_amd64.go index 0a05bdde927..624f48c890a 100644 --- a/libbeat/common/seccomp/policy_linux_amd64.go +++ b/libbeat/common/seccomp/policy_linux_amd64.go @@ -34,6 +34,7 @@ func init() { "arch_prctl", "bind", "brk", + "capget", "chmod", "chown", "clock_gettime", diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 100ccd3f013..4360aa0c192 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -46,7 +46,7 @@ export PATH := ./bin:$(PATH) GOFILES = $(shell find . -type f -name '*.go' 2>/dev/null) GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "*/vendor/*" 2>/dev/null) GOFILES_ALL = $(GOFILES) $(shell find $(ES_BEATS) -type f -name '*.go' 2>/dev/null) -GOPACKAGES_STRESSTESTS=$(shell find . -name '*.go' 2>/dev/null | xargs grep -l '\+build.*stresstest' | xargs -n1 dirname | uniq) +GOPACKAGES_STRESSTESTS=$(shell find . -type d \( -name "stress" \) 2>/dev/null) SHELL=bash ES_HOST?=elasticsearch ES_PORT?=9200 @@ -87,7 +87,7 @@ SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run un STRESS_TESTS?=false ## @testing if true, "make test" and "make testsuite" run also run the stress tests STRESS_TEST_OPTIONS?=-timeout=20m -race -v GOX_OS?=linux darwin windows freebsd netbsd openbsd ## @Building List of all OS to be supported by "make crosscompile". -GOX_OSARCH?=!darwin/arm !darwin/arm64 ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile". +GOX_OSARCH?=!darwin/arm !darwin/386 !linux/386 !windows/386 !freebsd/386 !netbsd/386 !openbsd/386 !linux/ppc64 ## @building Space-separated list of GOOS/GOARCH pairs to exclude (unsupported by GO and generated by GOX) in the "make crosscompile" build. GOX_FLAGS?= ## @building Additional flags to append to the gox command used by "make crosscompile". # XXX: Should be switched back to `snapshot` once the Elasticsearch # snapshots are working. https://github.com/elastic/beats/pull/6416 diff --git a/metricbeat/helper/socket/ptable_linux.go b/metricbeat/helper/socket/ptable_linux.go index 88fff488bc2..ffe585f7094 100644 --- a/metricbeat/helper/socket/ptable_linux.go +++ b/metricbeat/helper/socket/ptable_linux.go @@ -20,17 +20,22 @@ package socket import ( - "github.com/elastic/beats/v7/libbeat/common" + "kernel.org/pub/linux/libs/security/libcap/cap" ) -var requiredCapabilities = []string{"sys_ptrace", "dac_read_search"} - // isPrivileged checks if this process has privileges to read sockets // of all users func isPrivileged() (bool, error) { - capabilities, err := common.GetCapabilities() + set := cap.GetProc() + + ptrace, err := set.GetFlag(cap.Effective, cap.SYS_PTRACE) + if err != nil { + return false, err + } + dac_read_search, err := set.GetFlag(cap.Effective, cap.DAC_READ_SEARCH) if err != nil { return false, err } - return capabilities.Check(requiredCapabilities), nil + + return ptrace && dac_read_search, nil } diff --git a/metricbeat/tests/system/test_reload.py b/metricbeat/tests/system/test_reload.py index 29d82bbf82b..99aa8e2c2f2 100644 --- a/metricbeat/tests/system/test_reload.py +++ b/metricbeat/tests/system/test_reload.py @@ -42,7 +42,8 @@ def test_reload(self): self.wait_until(lambda: self.output_lines() > 0) proc.check_kill_and_wait() - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") + # windows is disabled, see https://github.com/elastic/beats/issues/37841 + @unittest.skipUnless(re.match("(?i)linux|darwin|freebsd|openbsd", sys.platform), "os") def test_start_stop(self): """ Test if module is properly started and stopped diff --git a/x-pack/auditbeat/module/system/_meta/fields.yml b/x-pack/auditbeat/module/system/_meta/fields.yml index 61908a6ce29..43101839c0a 100644 --- a/x-pack/auditbeat/module/system/_meta/fields.yml +++ b/x-pack/auditbeat/module/system/_meta/fields.yml @@ -30,6 +30,25 @@ - name: process type: group fields: + - name: thread.capabilities.effective + level: extended + type: keyword + ignore_above: 1024 + description: This is the set of capabilities used by the kernel to perform permission + checks for the thread. + example: '["CAP_BPF", "CAP_SYS_ADMIN"]' + pattern: ^(CAP_[A-Z_]+|\d+)$ + default_field: false + - name: thread.capabilities.permitted + level: extended + type: keyword + ignore_above: 1024 + description: This is a limiting superset for the effective capabilities that + the thread may assume. + example: '["CAP_BPF", "CAP_SYS_ADMIN"]' + pattern: ^(CAP_[A-Z_]+|\d+)$ + default_field: false + - name: hash type: group description: > diff --git a/x-pack/auditbeat/module/system/fields.go b/x-pack/auditbeat/module/system/fields.go index 7711dffe2c0..4b0a95d23b0 100644 --- a/x-pack/auditbeat/module/system/fields.go +++ b/x-pack/auditbeat/module/system/fields.go @@ -19,5 +19,5 @@ func init() { // AssetSystem returns asset data. // This is the base64 encoded zlib format compressed contents of module/system. func AssetSystem() string { - return "eJy0WV1v2zoSffevGPSlCeAqiNsEhR8WSJtiE2y7DdYp0DebEscSNxSpS1JJ1F9/QerDkk3ZlqMroEDNkOec+eBoSH2AJyzmoAttMJ0AGGY4zuHdwg28mwBQ1JFimWFSzOFfEwCAxwQ1AlEIJkFYM+RUQ4wCFTFIISzceIkJqaQ5x2ACoJAj0TiHEA2ZQLVwPpkAfABBUpwDPqMwjsMUGc4hVjLP3O96sv1/PVsqFjPhhuoFT1i8SEWrMY92+/x060CunU7HGcBjwjRERECIQGDNOEJGTAJnGMQBrC6eibrgMrb/gsvV+bRBk8rBWEk1ZGV6JNNMChQGTEIM6DzLOEPqplBiSI0t0HAmnlbnQdsXuUZ1tCtQGGaKJaPDvXF/C7lgf+XIC2DUAq0LJmKn0moAKYBAIrUJ4N6A9ZJMs9xGmmggsLi7+TC7uoaE6GTjlNIRdhXc305LIPsfImj5w+oOOjYYVCkThA834bFaWdNago4vMyUj1Ppod7Zs2Z7eK+KO6AR1k1WvGOWGhBxtaqG1Q7stQ3gsFTNJ6qi0c4hd8Ex4jm5Kg+g8iK+AIpIUKVAWozbVTGfftv6NBSEnTzgLl7Or6w2ex6Nb5nz5fvOfb7OwCajHnEkP08fPn05h+vj501Cmq8vZKUxXl7NjmXRCZrNB5izubmazoy3RCRnorsXdzQBPWfzlcAvcmmEcw9Kr5Dg+txzHCZ5aDvXVwJRyHMPy6epydkJEri5nF8Ni4ngGR8XxHB+X19fkepApv39f7zWiMcC9OQOSU+bvAzzFt1sAW0VcatMM+gp5D179rCzACiIpDGGi7nB4+VJjYi1VSuy6oLVqu8epn22Nrdd8ZliKHeJSKZci7gyXhHOguXK8nT8ykeVmWU8RREiNkRRUd2bJ3LSnEX1LCu+MTGHEtHPKZefve/xln1/OGmCiLSHwmB1KaXoMp8TgEM4vUhqwWD6eKnqo2B+kHrJQSo5EDOFboAG2rtLAdkANh0+AFfZHCgzsT4+A7W1zhID/tlrNGr7dcU3B9ZVfFo97Bcn1WqMJNEbHZN8BTY8bHRbVZsCe6FuV4/njrkLzMTFf0E/kgPtbHwVRUcIMRiZXIxrUga1OCq+fr5fXn859IlLii+IJ3D9uvgKhVKHW6I0dyzxEW4MHOO4f9lNI7aHYrtwHWFZSt2p3q1wDCWVu3GaRmT2y2sNO9d7p1tudmt0uKxR3Enif1w/65OeiAZ3a8kJEUUVdG4UmSs4Dr5KME2NtG1VJDVopiFAYqaeQh7kw+RRemKDyRfcoGt0v7mhdKvlBIjvyu4d6TVLGi1HJS8iKXiFNiJkCxZARMYW1Qgw1PeSRZ1R6+4X9Vl0Vpp/wCZVAPh7fo2ezvNcVzX4plnVUw+1qONOI8O3rAqQO7EDL8c3GINETifFNHWCFsbeQEAFMaEM4RwpSgcJUPiOt+d/WHW7f6xxy4F737bvpqdUevOKpGo2dyDRXPRUSlJWMiMYTvjzprRknmvjQIvfx+HbiG6n2WFXFe0y2CrKvDxmTqt2A+Pg4i1CMa10F6W07yj02ypmhpqswew8Pmv056mR2FJkF85LkaUpUcQJgudCHmSs+Zlh+/e/7bn1t7qfbFEOKqwU42KLZSbq8gt7t0Y6vp/9UdwLwq3uZveMlto34drbuMWTDFY/L9W8by14yytTYhr3XkMgULTRGRnZTu33JhXzE3gbgQclYkRSMBJULIAa4jFlPP2MTctnK1VE9Xt0wuQ8k7Rsm+CngOxP56xRMwrR9Q9vNEWMkdZntPRmxc2aqFcrw/xiZYQJXDu5AM1SUpHrz/YhpyIgytnE4C7GQ1QePvIx4ppitYuWqrf7Zv5Nh/24+FIWjIgFN/u9ubdi75Tb0TBiMcXuXDKTv234Z0dpjXN9R+XBsa8D94W2iVs2GMyFN1UBWI8xo5OvBkfScE2CsSN7syLawATxIrVnI2x/fYKUTQuXLsvFHD+ZZx2jXGduNKcoPwA7DfUU+n258u6RMk5AjXU17UFdCbpgtR7nZKRExKplr14+LQgp036q5jIGJc9dm9yFGqshMG/QlQdENmYuN1X6BJrpwwxQ0Yqp7QI2ss8Qef1A4DnfmKRF3ot/qGok2yyixBvVvnZ12rnyOCvaj+7pedGpMbegL0U4AVAKCyd8BAAD//yDbzZE=" + return "eJy8Wm1v27oV/u5fcRAMaILrKotvExT+MMBt7pZg7W0wp0C3uzubEo8lLhSpkVQSFfvxA6kXSzZlW4k7A0VjmXyec57z4kPJb+EBiynoQhtMRwCGGY5TOJm7CycjAIo6UiwzTIop/GkEAHCfoEYgCsEkCCuGnGqIUaAiBimEhbteYkIqac4xGAEo5Eg0TiFEQ0ZQbZyORgBvQZAUp4CPKIzjMEWGU4iVzDP3vl5s/65XS8ViJtylesMDFk9S0eqax3b7+uL2gVw5Ox1nAPcJ0xARASECgRXjCBkxCZxiEAewPH8k6pzL2P4LLpZn4wZNKgdjTaohK9cjmWZSoDBgEmJA51nGGVK3hBJDamyBhjPxsDwL2lrkGtXBUqAwzBQLRoercXsNuWD/yZEXwKgFWhVMxM5KawNIAQQSqU0AtwasSjLNchtpooHA/Gb2dnJ5BQnRyVqUUgi7C26vxyWQ/YMIWr6xdgcdHwyqlAnCh7twX+2saS1BR8tMyQi1PlhOkygkNIhIRkLGmWGoA1ytMDLsEStajo/Ip4DPBgXFXcKzWEiFCxLKR5zCxR8n73zuuARkukwgNNaXNr91qqmtB1QCORgJGaqVVKn9P2VaMykaVaIEowcNqypBK5+qj/GZpJkt9Te/nXyc3S0+3P35ZAzuz/nf54vZ9efbX09+f1OtzogxqMQU/nVqV/w2e/uPxe8//fef9KezPzS+rEjOzcLJOYUV4Rr3auqsNqZR70dpSoCzlBmb1jrPUFl9a12auHbltiXbSLnWD1JSANE6r7P3/yVlR8tWrW2mc2+R3BCdoG663jNGuSEhR9v6bEYV2rV0wmOpmElSR6VdwdoNj4Tn6JZ0VEnwGVBEkiIFymLUploZjKp17fpaexBy8oCTcDG5vFrjeeK84c6HT7O//jIJm4bjcWfUw/Tz+3cvYfr5/buhTJcXk5cwXV5MDmXSCZlMBrkzv5lNJgd7ohMyUK75zWyAUhZ/MdwDt2cYx7D0KjkOzy3H8QKlFkO1GphSjmNYPl1eTF4QkcuLyfmwmDiewVFxPIfH5fk5uRrkyrdvVzudaBxwk11Acsr8c6qn+XYbYKuJS73+hvE18h68+rW0AEuIpDCEiXoC5+XQxYQdC4jdF7R2bc7g9WvTxtYYmhmWYoe4tJRLEXcul4RToLlyvJ0Pmchys6iXCCKkxkgKqjurZG7ay4i+JoV3RaYwYtqJctH5fIde9vXVeQNMtE0IPG6HUpoexykxOITzg5QGLJaPp4oeKvYdqYcslJIjEUP45miArao0sLNPw+EzwBr2XQoM7FuPAZtlc4ABv7aOQjV8+0QwBnfu+TC/32mQXK00mkBjdEj27bHpfm2HRbUZsCP61srj6XFTofmYmC/oL+SA22sfBVFRwgxGJldHdKgDW51kn99fLa7enfmMSIkvii/g/jz7CIRShVqjN3Ys8xBtXNzDcXu3m0JqD8Vm597DspS61btb7RpIKHPjikVmaFupPbWU3zvdfrvVs9ttheJWAu9Sfa8mX+YN6Ni2FyKKKuraKDRRchZ4Lck4Mda3o1pSg1YWRCiM1GPIw1yYfAxPTFD5pHssOrou7tZPaclnEtkr33qoVyRlvDgqeQlZ0SukCTFjoBgyIsawUoihpvsUeUSlN7+wX2tXheknLO9fHI/v3lMsb3R9m2SnKZb1qI7b3XCqEeGXj3OQOrAXWsI3hUGiBxLjqybACmNnIyECmNCGcI4UpAKFqXxEWvO/bjrcvO+4T8Cd8u26E1lbu/cWZDVobEWmuRVZIUHZyYholPDlSW/PeKGLdy1yH4+vEl9JtcOrKt7HZKsg++aQY1K1BxAfH2cRiuN6V0F6x46yxo5yZqjpKszew4Nm3w86mR1EZsG8JHmaElW8ALDc6MPMFT9mWL7+7dN2f22en7QphjRXC7B3RLOLdPmIZHtGO7yf/qjpBOBr92HLlkpsE/H1bN1jyJorPi7XX2wse8koU8d27I2GRKZooTEyspva7ZtcyI842wDcKRkrkoKRoHIBxACXMeuZZ2xCLlq5elTFqztM7gFe+w4TfBHwiYn8eQym9RgrxkjqMtt7MmLrzFRbKMN/Y2SGGbh0cHuGoaIk1evnm0xDRpR74HYaYiGrBx55GfFMMdvFyl0b87O/kmF3Ne+LwkGRgCb/t0sbdpbcmp4JgzFuVslA+r7yy4jWHuf6jsr7Y1sD7g5vE7VqNZwKaaoBsrrCjEa+GhxJzzkBjhXJ2ZbZFjaAO6k1C3n74RssdUKofFo0evRgnnacdpOxLUxRPl92GO5XDmfjtbYLyjQJOdLluAd1KeSa2XKUxU6JiFHJXLt5XBRSoPstBZcxMHHmxuw+xEgVmWmDPiUouiFzsbG2n6OJzt1lChox1T2gRtZZYo8/KByHO/OUiFvRb02NRJtFlFiH+ktna5wrXwcF+979+qPo9Jja0SeinQFQGRCM/hcAAP//VUidAg==" } diff --git a/x-pack/auditbeat/module/system/process/process.go b/x-pack/auditbeat/module/system/process/process.go index d2dfae06598..08a72fe562e 100644 --- a/x-pack/auditbeat/module/system/process/process.go +++ b/x-pack/auditbeat/module/system/process/process.go @@ -18,6 +18,7 @@ import ( "github.com/elastic/beats/v7/auditbeat/datastore" "github.com/elastic/beats/v7/auditbeat/helper/hasher" + "github.com/elastic/beats/v7/libbeat/common/capabilities" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/x-pack/auditbeat/cache" @@ -101,12 +102,14 @@ type MetricSet struct { // Process represents information about a process. type Process struct { - Info types.ProcessInfo - UserInfo *types.UserInfo - User *user.User - Group *user.Group - Hashes map[hasher.HashType]hasher.Digest - Error error + Info types.ProcessInfo + UserInfo *types.UserInfo + User *user.User + Group *user.Group + CapEffective []string + CapPermitted []string + Hashes map[hasher.HashType]hasher.Digest + Error error } // Hash creates a hash for Process. @@ -376,6 +379,13 @@ func (ms *MetricSet) processEvent(process *Process, eventType string, action eve event.RootFields.Put("user.group.name", process.Group.Name) } + if len(process.CapEffective) > 0 { + event.RootFields.Put("process.thread.capabilities.effective", process.CapEffective) + } + if len(process.CapPermitted) > 0 { + event.RootFields.Put("process.thread.capabilities.permitted", process.CapPermitted) + } + if process.Hashes != nil { for hashType, digest := range process.Hashes { fieldName := "process.hash." + string(hashType) @@ -489,8 +499,20 @@ func (ms *MetricSet) getProcesses() ([]*Process, error) { } // Exclude Linux kernel processes, they are not very interesting. - if runtime.GOOS == "linux" && userInfo.UID == "0" && process.Info.Exe == "" { - continue + if runtime.GOOS == "linux" { + if userInfo.UID == "0" && process.Info.Exe == "" { + continue + } + + // Fetch Effective and Permitted capabilities + process.CapEffective, err = capabilities.FromPid(capabilities.Effective, pInfo.PID) + if err != nil && process.Error == nil { + process.Error = err + } + process.CapPermitted, err = capabilities.FromPid(capabilities.Permitted, pInfo.PID) + if err != nil && process.Error == nil { + process.Error = err + } } processes = append(processes, process) diff --git a/x-pack/filebeat/docs/inputs/input-cel.asciidoc b/x-pack/filebeat/docs/inputs/input-cel.asciidoc index 837ea80ea1e..b6eaa9ad744 100644 --- a/x-pack/filebeat/docs/inputs/input-cel.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-cel.asciidoc @@ -580,6 +580,13 @@ The RSA JWK Private Key JSON for your Okta Service App which is used for interac NOTE: Only one of the credentials settings can be set at once. For more information please refer to https://developer.okta.com/docs/guides/implement-oauth-for-okta-serviceapp/main/ +[float] +==== `auth.oauth2.okta.jwk_pem` + +The RSA JWK private key PEM block for your Okta Service App which is used for interacting with Okta Org Auth Server to mint tokens with okta.* scopes. + +NOTE: Only one of the credentials settings can be set at once. For more information please refer to https://developer.okta.com/docs/guides/implement-oauth-for-okta-serviceapp/main/ + [[resource-parameters]] [float] ==== `resource.url` diff --git a/x-pack/filebeat/input/cel/config_auth.go b/x-pack/filebeat/input/cel/config_auth.go index e550a9635d5..d6b35d633e6 100644 --- a/x-pack/filebeat/input/cel/config_auth.go +++ b/x-pack/filebeat/input/cel/config_auth.go @@ -6,6 +6,7 @@ package cel import ( "context" + "crypto/x509" "encoding/json" "errors" "fmt" @@ -141,6 +142,7 @@ type oAuth2Config struct { // okta specific RSA JWK private key OktaJWKFile string `config:"okta.jwk_file"` OktaJWKJSON common.JSONBlob `config:"okta.jwk_json"` + OktaJWKPEM string `config:"okta.jwk_pem"` } // isEnabled returns true if the `enable` field is set to true in the yaml. @@ -321,8 +323,26 @@ func (o *oAuth2Config) validateGoogleProvider() error { } func (o *oAuth2Config) validateOktaProvider() error { - if o.TokenURL == "" || o.ClientID == "" || len(o.Scopes) == 0 || (o.OktaJWKJSON == nil && o.OktaJWKFile == "") { - return errors.New("okta validation error: token_url, client_id, scopes and at least one of okta.jwk_json or okta.jwk_file must be provided") + if o.TokenURL == "" || o.ClientID == "" || len(o.Scopes) == 0 { + return errors.New("okta validation error: token_url, client_id, scopes must be provided") + } + var n int + if o.OktaJWKJSON != nil { + n++ + } + if o.OktaJWKFile != "" { + n++ + } + if o.OktaJWKPEM != "" { + n++ + } + if n != 1 { + return errors.New("okta validation error: one of okta.jwk_json, okta.jwk_file or okta.jwk_pem must be provided") + } + // jwk_pem + if o.OktaJWKPEM != "" { + _, err := x509.ParsePKCS1PrivateKey([]byte(o.OktaJWKPEM)) + return err } // jwk_file if o.OktaJWKFile != "" { diff --git a/x-pack/filebeat/input/cel/config_okta_auth.go b/x-pack/filebeat/input/cel/config_okta_auth.go index cf9003dee8a..74366afd3d5 100644 --- a/x-pack/filebeat/input/cel/config_okta_auth.go +++ b/x-pack/filebeat/input/cel/config_okta_auth.go @@ -5,10 +5,13 @@ package cel import ( + "bytes" "context" "crypto/rsa" + "crypto/x509" "encoding/base64" "encoding/json" + "encoding/pem" "fmt" "math/big" "net/http" @@ -43,9 +46,20 @@ func (o *oAuth2Config) fetchOktaOauthClient(ctx context.Context, _ *http.Client) }, } - oktaJWT, err := generateOktaJWT(o.OktaJWKJSON, conf) - if err != nil { - return nil, fmt.Errorf("oauth2 client: error generating Okta JWT: %w", err) + var ( + oktaJWT string + err error + ) + if len(o.OktaJWKPEM) != 0 { + oktaJWT, err = generateOktaJWTPEM(o.OktaJWKPEM, conf) + if err != nil { + return nil, fmt.Errorf("oauth2 client: error generating Okta JWT PEM: %w", err) + } + } else { + oktaJWT, err = generateOktaJWT(o.OktaJWKJSON, conf) + if err != nil { + return nil, fmt.Errorf("oauth2 client: error generating Okta JWT: %w", err) + } } token, err := exchangeForBearerToken(ctx, oktaJWT, conf) @@ -59,14 +73,16 @@ func (o *oAuth2Config) fetchOktaOauthClient(ctx context.Context, _ *http.Client) oktaJWK: o.OktaJWKJSON, token: token, } - // reuse the tokenSource to refresh the token (automatically calls the custom Token() method when token is no longer valid). + // reuse the tokenSource to refresh the token (automatically calls + // the custom Token() method when token is no longer valid). client := oauth2.NewClient(ctx, oauth2.ReuseTokenSource(token, tokenSource)) return client, nil } -// Token implements the oauth2.TokenSource interface and helps to implement custom token refresh logic. -// Parent context is passed via the customTokenSource struct since we cannot modify the function signature here. +// Token implements the oauth2.TokenSource interface and helps to implement +// custom token refresh logic. The parent context is passed via the +// customTokenSource struct since we cannot modify the function signature here. func (ts *oktaTokenSource) Token() (*oauth2.Token, error) { ts.mu.Lock() defer ts.mu.Unlock() @@ -85,70 +101,79 @@ func (ts *oktaTokenSource) Token() (*oauth2.Token, error) { } func generateOktaJWT(oktaJWK []byte, cnf *oauth2.Config) (string, error) { - // unmarshal the JWK into a map - var jwkData map[string]string + // Unmarshal the JWK into big ints. + var jwkData struct { + N base64int `json:"n"` + E base64int `json:"e"` + D base64int `json:"d"` + P base64int `json:"p"` + Q base64int `json:"q"` + Dp base64int `json:"dp"` + Dq base64int `json:"dq"` + Qinv base64int `json:"qi"` + } err := json.Unmarshal(oktaJWK, &jwkData) if err != nil { return "", fmt.Errorf("error decoding JWK: %w", err) } - // create an RSA private key from JWK components - decodeBase64 := func(key string) (*big.Int, error) { - data, err := base64.RawURLEncoding.DecodeString(jwkData[key]) - if err != nil { - return nil, fmt.Errorf("error decoding RSA JWK component %s: %w", key, err) - } - return new(big.Int).SetBytes(data), nil + // Create an RSA private key from JWK components. + key := &rsa.PrivateKey{ + PublicKey: rsa.PublicKey{ + N: &jwkData.N.Int, + E: int(jwkData.E.Int64()), + }, + D: &jwkData.D.Int, + Primes: []*big.Int{&jwkData.P.Int, &jwkData.Q.Int}, + Precomputed: rsa.PrecomputedValues{ + Dp: &jwkData.Dp.Int, + Dq: &jwkData.Dq.Int, + Qinv: &jwkData.Qinv.Int, + }, } - n, err := decodeBase64("n") - if err != nil { - return "", err - } - e, err := decodeBase64("e") - if err != nil { - return "", err - } - d, err := decodeBase64("d") - if err != nil { - return "", err - } - p, err := decodeBase64("p") - if err != nil { - return "", err + return signJWT(cnf, key) + +} + +// base64int is a JSON decoding shim for base64-encoded big.Int. +type base64int struct { + big.Int +} + +func (i *base64int) UnmarshalJSON(b []byte) error { + src, ok := bytes.CutPrefix(b, []byte{'"'}) + if !ok { + return fmt.Errorf("invalid JSON type: %s", b) } - q, err := decodeBase64("q") - if err != nil { - return "", err + src, ok = bytes.CutSuffix(src, []byte{'"'}) + if !ok { + return fmt.Errorf("invalid JSON type: %s", b) } - dp, err := decodeBase64("dp") + dst := make([]byte, base64.RawURLEncoding.DecodedLen(len(src))) + _, err := base64.RawURLEncoding.Decode(dst, src) if err != nil { - return "", err + return err } - dq, err := decodeBase64("dq") - if err != nil { - return "", err + i.SetBytes(dst) + return nil +} + +func generateOktaJWTPEM(pemdata string, cnf *oauth2.Config) (string, error) { + blk, rest := pem.Decode([]byte(pemdata)) + if rest := bytes.TrimSpace(rest); len(rest) != 0 { + return "", fmt.Errorf("PEM text has trailing data: %s", rest) } - qi, err := decodeBase64("qi") + key, err := x509.ParsePKCS8PrivateKey(blk.Bytes) if err != nil { return "", err } + return signJWT(cnf, key) +} - privateKeyRSA := &rsa.PrivateKey{ - PublicKey: rsa.PublicKey{ - N: n, - E: int(e.Int64()), - }, - D: d, - Primes: []*big.Int{p, q}, - Precomputed: rsa.PrecomputedValues{ - Dp: dp, - Dq: dq, - Qinv: qi, - }, - } - - // create a JWT token using required claims and sign it with the private key +// signJWT creates a JWT token using required claims and sign it with the +// private key. +func signJWT(cnf *oauth2.Config, key any) (string, error) { now := time.Now() tok, err := jwt.NewBuilder().Audience([]string{cnf.Endpoint.TokenURL}). Issuer(cnf.ClientID). @@ -159,11 +184,10 @@ func generateOktaJWT(oktaJWK []byte, cnf *oauth2.Config) (string, error) { if err != nil { return "", err } - signedToken, err := jwt.Sign(tok, jwt.WithKey(jwa.RS256, privateKeyRSA)) + signedToken, err := jwt.Sign(tok, jwt.WithKey(jwa.RS256, key)) if err != nil { return "", fmt.Errorf("failed to sign token: %w", err) } - return string(signedToken), nil } diff --git a/x-pack/filebeat/input/cel/config_okta_auth_test.go b/x-pack/filebeat/input/cel/config_okta_auth_test.go new file mode 100644 index 00000000000..fc02a2ec9e7 --- /dev/null +++ b/x-pack/filebeat/input/cel/config_okta_auth_test.go @@ -0,0 +1,88 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cel + +import ( + "testing" + + "github.com/lestrrat-go/jwx/v2/jwt" + "golang.org/x/oauth2" +) + +func TestGenerateOktaJWT(t *testing.T) { + // jwt is a JWT obtained from the Okta integration. + const jwtText = `{ "d": "Cmhokw2MnZfX6da36nnsnQ7IPX9vE6se8_D1NgyL9j9rarYpexhlp45hswcAIFNgWA03NV848Gc0e84AW6wMbyD2E8LPI0Bd8lhdmzRE6L4or2Rxqqjk2Pr2aqGnqs4A0uTijAA7MfPF1zFFdR3EOVx499fEeTiMcLjO83IJCoNiOySDoQgt3KofX5bCbaDy2eiB83rzf0fEcWrWfTY65_Hc2c5lek-1uuF7NpELVzX80p5H-b9MOfLn0BdOGe-mJ2j5bXi-UCQ45Wxj2jdkoA_Qwb4MEtXZjp5LjcM75SrlGfVd99acML2wGZgYLGweJ0sAPDlKzGvj4ve-JT8nNw", "p": "8-UBb4psN0wRPktkh3S48L3ng4T5zR08t7nwXDYNajROrS2j7oq60dtlGY4IwgwcC0c9GDQP7NiN2IpU2uahYkGQ7lDyM_h7UfQWL5fMrsYiKgn2pUgSy5TTT8smkSLbJAD35nAH6PknsQ2PuvOlb4laiC0MXw1Rw4vT9HAEB9M", "q": "0DJkPEN0bECG_6lorlNJgIfoNahVevGKK-Yti1YZ5K-nQCuffPCwPG0oZZo_55y5LODe9W7psxnAt7wxkpAY4lK2hpHTWJSkPjqXWFYIP8trn4RZDShnJXli0i1XqPOqkiVzBZGx5nLtj2bUtmXfIU7-kneHGvLQ5EXcyQW1ISM", "dp": "Ye1PWEPSE5ndSo_m-2RoZXE6pdocmrjkijiEQ-IIHN6HwI0Ux1C4lk5rF4mqBo_qKrUd2Lv-sPB6c7mHPKVhoxwEX0vtE-TvTwacadufeYVgblS1zcNUmJ1XAzDkeV3vc1NYNhRBeM-hmjuBvGTbxh72VLsRvpCQhd186yaW17U", "dq": "jvSK7vZCUrJb_-CLCGgX6DFpuK5FQ43mmg4K58nPLb-Oz_kkId4CpPsu6dToXFi4raAad9wYi-n68i4-u6xF6eFxgyVOQVyPCkug7_7i2ysKUxXFL8u2R3z55edMca4eSQt91y0bQmlXxUeOd0-rzms3UcrQ8igYVyXBXCaXIJE", "qi": "iIY1Y4bzMYIFG7XH7gNP7C-mWi6QH4l9aGRTzPB_gPaFThvc0XKW0S0l82bfp_PPPWg4D4QpDCp7rZ6KhEA8BlNi86Vt3V6F3Hz5XiDa4ikgQNsAXiXLqf83R-y1-cwHjW70PP3U89hmalCRRFfVXcLHV77AVHqbrp9rAIo-X-I", "kty": "RSA", "e": "AQAB", "kid": "koeFQjkyiav_3Qwr3aRinCqCD2LaEHOjFnje7XlkbdI", "n": "xloTY8bAuI5AEo8JursCd7w0LmELCae7JOFaVo9njGrG8tRNqgIdjPyoGY_ABwKkmjcCMLGMA29llFDbry8rB4LTWai-h_jX4_uUUnl52mLX-lO6merL5HEPZF438Ql9Hrxs5yGzT8n865-E_3uwYSBrhTjvlZJeXYUeVHfKo8pJSSsw3RZEjBW4Tt0eFmCZnFErtTyk3oUPaYVP-8YLLAenhUDV4Lm1dC4dxqUj0Oh6XrWgIb-eYHGolMY9g9xbgyd4ir39RodA_1DOjzHWpNfCM-J5ZOtfpuKCAe5__u7L8FT0m56XOxcDoVVsz1J1VNrACWAGbhDWNjyHfL5E2Q" }` + cnf := &oauth2.Config{ + ClientID: "0oaajljpeokFZLyKU5d7", + Scopes: []string{"okta.logs.read"}, + } + got, err := generateOktaJWT([]byte(jwtText), cnf) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + tok, err := jwt.Parse([]byte(got), jwt.WithVerify(false)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tok.Issuer() != cnf.ClientID { + t.Errorf("unexpected issuer: got:%s want:%s", tok.Issuer(), cnf.ClientID) + } + if tok.Subject() != cnf.ClientID { + t.Errorf("unexpected subject: got:%s want:%s", tok.Subject(), cnf.ClientID) + } +} + +func TestGenerateOktaJWTPEM(t *testing.T) { + // jwtText is generated by https://mkjwk.org/ using the instructions at + // https://developer.okta.com/docs/guides/dpop/nonoktaresourceserver/main/#create-the-json-web-token + const jwtText = ` +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCOuef3HMRhohVT +5kSoAJgV+atpDjkwTwkOq+ImnbBlv75GaApG90w8VpjXjhqN/1KJmwfyrKiquiMq +OPu+o/672Dys5rUAaWSbT7wRF1GjLDDZrM0GHRdV4DGxM/LKI8I5yE1Mx3EzV+D5 +ZLmcRc5U4oEoMwtGpr0zRZ7uUr6a28UQwcUsVIPItc1/9rERlo1WTv8dcaj4ECC3 +2Sc0y/F+9XqwJvLd4Uv6ckzP0Sv4tbDA+7jpD9MneAIUiZ4LVj2cwbBd+YRY6jXx +MkevcCSmSX60clBY1cIFkw1DYHqtdHEwAQcQHLGMoi72xRP2qrdzIPsaTKVYoHVo +WA9vADdHAgMBAAECggEAIlx7jjCsztyYyeQsL05FTzUWoWo9NnYwtgmHnshkCXsK +MiUmJEOxZO1sSqj5l6oakupyFWigCspZYPbrFNCiqVK7+NxqQzkccY/WtT6p9uDS +ufUyPwCN96zMCd952lSVlBe3FH8Hr9a+YQxw60CbFjCZ67WuR0opTsi6JKJjJSDb +TQQZ4qJR97D05I1TgfmO+VO7G/0/dDaNHnnlYz0AnOgZPSyvrU2G5cYye4842EMB +ng81xjHD+xp55JNui/xYkhmYspYhrB2KlEjkKb08OInUjBeaLEAgA1r9yOHsfV/3 +DQzDPRO9iuqx5BfJhdIqUB1aifrye+sbxt9uMBtUgQKBgQDVdfO3GYT+ZycOQG9P +QtdMn6uiSddchVCGFpk331u6M6yafCKjI/MlJDl29B+8R5sVsttwo8/qnV/xd3cn +pY14HpKAsE4l6/Ciagzoj+0NqfPEDhEzbo8CyArcd7pSxt3XxECAfZe2+xivEPHe +gFO60vSFjFtvlLRMDMOmqX3kYQKBgQCrK1DISyQTnD6/axsgh2/ESOmT7n+JRMx/ +YzA7Lxu3zGzUC8/sRDa1C41t054nf5ZXJueYLDSc4kEAPddzISuCLxFiTD2FQ75P +lHWMgsEzQObDm4GPE9cdKOjoAvtAJwbvZcjDa029CDx7aCaDzbNvdmplZ7EUrznR +55U8Wsm8pwKBgBytxTmzZwfbCgdDJvFKNKzpwuCB9TpL+v6Y6Kr2Clfg+26iAPFU +MiWqUUInGGBuamqm5g6jI5sM28gQWeTsvC4IRXyes1Eq+uCHSQax15J/Y+3SSgNT +9kjUYYkvWMwoRcPobRYWSZze7XkP2L8hFJ7EGvAaZGqAWxzgliS9HtnhAoGAONZ/ +UqMw7Zoac/Ga5mhSwrj7ZvXxP6Gqzjofj+eKqrOlB5yMhIX6LJATfH6iq7cAMxxm +Fu/G4Ll4oB3o5wACtI3wldV/MDtYfJBtoCTjBqPsfNOsZ9hMvBATlsc2qwzKjsAb +tFhzTevoOYpSD75EcSS/G8Ec2iN9bagatBnpl00CgYBVqAOFZelNfP7dj//lpk8y +EUAw7ABOq0S9wkpFWTXIVPoBQUipm3iAUqGNPmvr/9ShdZC9xeu5AwKram4caMWJ +ExRhcDP1hFM6CdmSkIYEgBKvN9N0O4Lx1ba34gk74Hm65KXxokjJHOC0plO7c7ok +LNV/bIgMHOMoxiGrwyjAhg== +-----END PRIVATE KEY----- +` + cnf := &oauth2.Config{ + ClientID: "0oaajljpeokFZLyKU5d7", + Scopes: []string{"okta.logs.read"}, + } + got, err := generateOktaJWTPEM(jwtText, cnf) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + tok, err := jwt.Parse([]byte(got), jwt.WithVerify(false)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tok.Issuer() != cnf.ClientID { + t.Errorf("unexpected issuer: got:%s want:%s", tok.Issuer(), cnf.ClientID) + } + if tok.Subject() != cnf.ClientID { + t.Errorf("unexpected subject: got:%s want:%s", tok.Subject(), cnf.ClientID) + } +} diff --git a/x-pack/filebeat/input/cel/config_test.go b/x-pack/filebeat/input/cel/config_test.go index 0cd404705e2..e4c98b78dc5 100644 --- a/x-pack/filebeat/input/cel/config_test.go +++ b/x-pack/filebeat/input/cel/config_test.go @@ -489,8 +489,8 @@ var oAuth2ValidationTests = []struct { }, }, { - name: "okta requires token_url, client_id, scopes and at least one of okta.jwk_json or okta.jwk_file to be provided", - wantErr: errors.New("okta validation error: token_url, client_id, scopes and at least one of okta.jwk_json or okta.jwk_file must be provided accessing 'auth.oauth2'"), + name: "unique_okta_jwk_token", + wantErr: errors.New("okta validation error: one of okta.jwk_json, okta.jwk_file or okta.jwk_pem must be provided accessing 'auth.oauth2'"), input: map[string]interface{}{ "auth.oauth2": map[string]interface{}{ "provider": "okta", @@ -501,7 +501,7 @@ var oAuth2ValidationTests = []struct { }, }, { - name: "okta oauth2 validation fails if jwk_json is not a valid JSON", + name: "invalid_okta_jwk_json", wantErr: errors.New("the field can't be converted to valid JSON accessing 'auth.oauth2.okta.jwk_json'"), input: map[string]interface{}{ "auth.oauth2": map[string]interface{}{ @@ -514,7 +514,7 @@ var oAuth2ValidationTests = []struct { }, }, { - name: "okta successful oauth2 validation", + name: "okta_successful_oauth2_validation", input: map[string]interface{}{ "auth.oauth2": map[string]interface{}{ "provider": "okta",