diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 75f267c0849..b9d2440e677 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -404,6 +404,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Introduce ignore older and start timestamp filters for AWS S3 input. {pull}41804[41804] - Journald input now can report its status to Elastic-Agent {issue}39791[39791] {pull}42462[42462] - Publish events progressively in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}42567[42567] +- Journald `include_matches.match` now accepts `+` to represent a logical disjunction (OR) {issue}40185[40185] {pull}42517[42517] - The journald input is now generally available. {pull}42107[42107] *Auditbeat* diff --git a/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl index f9c70377950..9fc2eb0ef83 100644 --- a/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl +++ b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl @@ -810,11 +810,19 @@ filebeat.inputs: #- 2 # You may wish to have separate inputs for each service. You can use - # include_matches.or to specify a list of filter expressions that are - # applied as a logical OR. + # include_matches.match to define a list of filters, each event needs + # to match all filters defined. #include_matches.match: #- _SYSTEMD_UNIT=foo.service + # To create a disjunction (logical OR) use the `+` character between + # the filters + #include_matches: + #match: + #- "systemd.transport=kernel" + #- "+" + #- "journald.process.name=systemd" + # Uses the original hostname of the entry instead of the one # from the host running jounrald #save_remote_hostname: false diff --git a/filebeat/docs/inputs/input-journald.asciidoc b/filebeat/docs/inputs/input-journald.asciidoc index d161fb5bc28..80c29af9289 100644 --- a/filebeat/docs/inputs/input-journald.asciidoc +++ b/filebeat/docs/inputs/input-journald.asciidoc @@ -179,9 +179,13 @@ numeric code. ==== `include_matches` A collection of filter expressions used to match fields. The format of the expression -is `field=value`. {beatname_uc} fetches all events that exactly match the +is `field=value` or `+` representing disjunction (i.e. logical +OR). {beatname_uc} fetches all events that exactly match the expressions. Pattern matching is not supported. +When `+` is used, it will cause all matches before and after to be +combined in a disjunction (i.e. logical OR). + If you configured a filter expression, only entries with this field set will be iterated by the journald reader of Filebeat. If the filter expressions apply to different fields, only entries with all fields set will be iterated. If they apply to the same fields, only entries where the field takes one of the specified values will be iterated. @@ -213,6 +217,31 @@ include_matches: - "systemd.transport=syslog" ---- + +The following include matches configuration is the equivalent of the +following logical expression: + +``` +A=a OR (B=b AND C=c) OR (D=d AND B=1) +``` + +["source","yaml",subs="attributes"] +---- +include_matches: + match: + - A=a + - + + - B=b + - C=c + - + + - B=1 +---- + +`include_matches` translates to `journalctl` `MATCHES`, its +[documentation](https://www.man7.org/linux/man-pages/man1/journalctl.1.html) +is not clear about how multiple disjunctions are handled. The previous +example was tested with journalctl version 257. + To reference fields, use one of the following: * The field name used by the systemd journal. For example, diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index ba828e72de2..875f3f361cd 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1223,11 +1223,19 @@ filebeat.inputs: #- 2 # You may wish to have separate inputs for each service. You can use - # include_matches.or to specify a list of filter expressions that are - # applied as a logical OR. + # include_matches.match to define a list of filters, each event needs + # to match all filters defined. #include_matches.match: #- _SYSTEMD_UNIT=foo.service + # To create a disjunction (logical OR) use the `+` character between + # the filters + #include_matches: + #match: + #- "systemd.transport=kernel" + #- "+" + #- "journald.process.name=systemd" + # Uses the original hostname of the entry instead of the one # from the host running jounrald #save_remote_hostname: false diff --git a/filebeat/input/journald/pkg/journalfield/matcher.go b/filebeat/input/journald/pkg/journalfield/matcher.go index 8f44579f263..3c3421b8e4b 100644 --- a/filebeat/input/journald/pkg/journalfield/matcher.go +++ b/filebeat/input/journald/pkg/journalfield/matcher.go @@ -42,9 +42,7 @@ type MatcherBuilder struct { // IncludeMatches stores the advanced matching configuratio // provided by the user. type IncludeMatches struct { - Matches []Matcher `config:"match"` - AND []IncludeMatches `config:"and"` - OR []IncludeMatches `config:"or"` + Matches []Matcher `config:"match"` } var ( @@ -52,15 +50,42 @@ var ( ) func (i IncludeMatches) Validate() error { - if len(i.AND) != 0 || len(i.OR) != 0 { - return errors.New("'or' and 'and' are not supported at the moment") + var errs []error + for _, m := range i.Matches { + if err := m.validate(); err != nil { + errs = append(errs, err) + } + } + + return errors.Join(errs...) +} + +var errInvalidMatcher = errors.New("expression must be '+' or in the format 'field=value'") + +func (m Matcher) validate() error { + if len(m.str) == 1 { + if m.str != "+" { + return fmt.Errorf("'%s' is invalid, %w", m.str, errInvalidMatcher) + } + + return nil + } + + elems := strings.Split(m.str, "=") + if len(elems) != 2 { + return fmt.Errorf("'%s' is invalid, %w", m.str, errInvalidMatcher) } + return nil } // Build creates a new Matcher using the configured conversion table. // If no table has been configured the internal default table will be used. func (b MatcherBuilder) Build(in string) (Matcher, error) { + if in == "+" { + return Matcher{in}, nil + } + elems := strings.Split(in, "=") if len(elems) != 2 { return Matcher{}, fmt.Errorf("invalid match format: %s", in) diff --git a/filebeat/input/journald/pkg/journalfield/matcher_test.go b/filebeat/input/journald/pkg/journalfield/matcher_test.go index 589e6f4bb70..7efcc9ff093 100644 --- a/filebeat/input/journald/pkg/journalfield/matcher_test.go +++ b/filebeat/input/journald/pkg/journalfield/matcher_test.go @@ -27,24 +27,6 @@ func TestValidate(t *testing.T) { im IncludeMatches error bool }{ - { - name: "OR condition exists", - im: IncludeMatches{ - OR: []IncludeMatches{ - {}, - }, - }, - error: true, - }, - { - name: "AND condition exists", - im: IncludeMatches{ - AND: []IncludeMatches{ - {}, - }, - }, - error: true, - }, { name: "empty include matches succeeds validation", im: IncludeMatches{}, @@ -53,8 +35,10 @@ func TestValidate(t *testing.T) { name: "matches are allowed", im: IncludeMatches{ Matches: []Matcher{ - {"foo"}, - {"bar"}, + {"foo=bar"}, + {"+"}, + {"FOO=bar"}, + {"foo.bar=foo"}, }, }, }, diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 953ac8c8bb8..59c1897634d 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -2907,11 +2907,19 @@ filebeat.inputs: #- 2 # You may wish to have separate inputs for each service. You can use - # include_matches.or to specify a list of filter expressions that are - # applied as a logical OR. + # include_matches.match to define a list of filters, each event needs + # to match all filters defined. #include_matches.match: #- _SYSTEMD_UNIT=foo.service + # To create a disjunction (logical OR) use the `+` character between + # the filters + #include_matches: + #match: + #- "systemd.transport=kernel" + #- "+" + #- "journald.process.name=systemd" + # Uses the original hostname of the entry instead of the one # from the host running jounrald #save_remote_hostname: false