From 017f25db054d27b09f6cd0f6a16b1c774acd89f3 Mon Sep 17 00:00:00 2001 From: Alexander Yastrebov Date: Mon, 20 Jan 2025 18:27:49 +0100 Subject: [PATCH] dataclients/kubernetes: append filters to routes of Ingress/RouteGroup having specific annotation (#3376) Similar to #3328 this change adds new flags to configure filters appended to routes created for Kubernetes resources having specific annotation. Signed-off-by: Alexander Yastrebov --- config/config.go | 210 ++++++++------ config/config_test.go | 271 +++++++++++------- dataclients/kubernetes/annotations.go | 16 +- dataclients/kubernetes/ingress.go | 66 +++-- dataclients/kubernetes/ingress_test.go | 6 + dataclients/kubernetes/ingressv1.go | 6 +- dataclients/kubernetes/kube.go | 9 +- .../kubernetes/kubernetestest/fixtures.go | 52 ++-- dataclients/kubernetes/routegroup.go | 6 +- dataclients/kubernetes/routegroups_test.go | 6 +- .../annotation-filters-append.eskip | 43 +++ .../annotation-filters-append.kube | 48 ++++ .../annotation-filters-append.yaml | 90 ++++++ .../annotation-filters-append.eskip | 57 ++++ .../annotation-filters-append.kube | 46 +++ .../annotation-filters-append.yaml | 77 +++++ skipper.go | 77 ++--- 17 files changed, 810 insertions(+), 276 deletions(-) create mode 100644 dataclients/kubernetes/testdata/ingressV1/annotation-filters/annotation-filters-append.eskip create mode 100644 dataclients/kubernetes/testdata/ingressV1/annotation-filters/annotation-filters-append.kube create mode 100644 dataclients/kubernetes/testdata/ingressV1/annotation-filters/annotation-filters-append.yaml create mode 100644 dataclients/kubernetes/testdata/routegroups/annotation-filters/annotation-filters-append.eskip create mode 100644 dataclients/kubernetes/testdata/routegroups/annotation-filters/annotation-filters-append.kube create mode 100644 dataclients/kubernetes/testdata/routegroups/annotation-filters/annotation-filters-append.yaml diff --git a/config/config.go b/config/config.go index e945af39f5..8a4e2150c4 100644 --- a/config/config.go +++ b/config/config.go @@ -149,38 +149,42 @@ type Config struct { RefusePayload multiFlag `yaml:"refuse-payload"` // Kubernetes: - KubernetesIngress bool `yaml:"kubernetes"` - KubernetesInCluster bool `yaml:"kubernetes-in-cluster"` - KubernetesURL string `yaml:"kubernetes-url"` - KubernetesTokenFile string `yaml:"kubernetes-token-file"` - KubernetesHealthcheck bool `yaml:"kubernetes-healthcheck"` - KubernetesHTTPSRedirect bool `yaml:"kubernetes-https-redirect"` - KubernetesHTTPSRedirectCode int `yaml:"kubernetes-https-redirect-code"` - KubernetesDisableCatchAllRoutes bool `yaml:"kubernetes-disable-catchall-routes"` - KubernetesIngressClass string `yaml:"kubernetes-ingress-class"` - KubernetesRouteGroupClass string `yaml:"kubernetes-routegroup-class"` - WhitelistedHealthCheckCIDR string `yaml:"whitelisted-healthcheck-cidr"` - KubernetesPathModeString string `yaml:"kubernetes-path-mode"` - KubernetesPathMode kubernetes.PathMode `yaml:"-"` - KubernetesNamespace string `yaml:"kubernetes-namespace"` - KubernetesEnableEndpointSlices bool `yaml:"enable-kubernetes-endpointslices"` - KubernetesEnableEastWest bool `yaml:"enable-kubernetes-east-west"` - KubernetesEastWestDomain string `yaml:"kubernetes-east-west-domain"` - KubernetesEastWestRangeDomains *listFlag `yaml:"kubernetes-east-west-range-domains"` - KubernetesEastWestRangePredicatesString string `yaml:"kubernetes-east-west-range-predicates"` - KubernetesEastWestRangeAnnotationPredicatesString multiFlag `yaml:"kubernetes-east-west-range-annotation-predicates"` - KubernetesAnnotationPredicatesString multiFlag `yaml:"kubernetes-annotation-predicates"` - KubernetesEastWestRangeAnnotationPredicates []kubernetes.AnnotationPredicates `yaml:"-"` - KubernetesAnnotationPredicates []kubernetes.AnnotationPredicates `yaml:"-"` - KubernetesEastWestRangePredicates []*eskip.Predicate `yaml:"-"` - KubernetesOnlyAllowedExternalNames bool `yaml:"kubernetes-only-allowed-external-names"` - KubernetesAllowedExternalNames regexpListFlag `yaml:"kubernetes-allowed-external-names"` - KubernetesRedisServiceNamespace string `yaml:"kubernetes-redis-service-namespace"` - KubernetesRedisServiceName string `yaml:"kubernetes-redis-service-name"` - KubernetesRedisServicePort int `yaml:"kubernetes-redis-service-port"` - KubernetesBackendTrafficAlgorithmString string `yaml:"kubernetes-backend-traffic-algorithm"` - KubernetesBackendTrafficAlgorithm kubernetes.BackendTrafficAlgorithm `yaml:"-"` - KubernetesDefaultLoadBalancerAlgorithm string `yaml:"kubernetes-default-lb-algorithm"` + KubernetesIngress bool `yaml:"kubernetes"` + KubernetesInCluster bool `yaml:"kubernetes-in-cluster"` + KubernetesURL string `yaml:"kubernetes-url"` + KubernetesTokenFile string `yaml:"kubernetes-token-file"` + KubernetesHealthcheck bool `yaml:"kubernetes-healthcheck"` + KubernetesHTTPSRedirect bool `yaml:"kubernetes-https-redirect"` + KubernetesHTTPSRedirectCode int `yaml:"kubernetes-https-redirect-code"` + KubernetesDisableCatchAllRoutes bool `yaml:"kubernetes-disable-catchall-routes"` + KubernetesIngressClass string `yaml:"kubernetes-ingress-class"` + KubernetesRouteGroupClass string `yaml:"kubernetes-routegroup-class"` + WhitelistedHealthCheckCIDR string `yaml:"whitelisted-healthcheck-cidr"` + KubernetesPathModeString string `yaml:"kubernetes-path-mode"` + KubernetesPathMode kubernetes.PathMode `yaml:"-"` + KubernetesNamespace string `yaml:"kubernetes-namespace"` + KubernetesEnableEndpointSlices bool `yaml:"enable-kubernetes-endpointslices"` + KubernetesEnableEastWest bool `yaml:"enable-kubernetes-east-west"` + KubernetesEastWestDomain string `yaml:"kubernetes-east-west-domain"` + KubernetesEastWestRangeDomains *listFlag `yaml:"kubernetes-east-west-range-domains"` + KubernetesEastWestRangePredicatesString string `yaml:"kubernetes-east-west-range-predicates"` + KubernetesEastWestRangeAnnotationPredicatesString multiFlag `yaml:"kubernetes-east-west-range-annotation-predicates"` + KubernetesEastWestRangeAnnotationFiltersAppendString multiFlag `yaml:"kubernetes-east-west-range-annotation-filters-append"` + KubernetesAnnotationPredicatesString multiFlag `yaml:"kubernetes-annotation-predicates"` + KubernetesAnnotationFiltersAppendString multiFlag `yaml:"kubernetes-annotation-filters-append"` + KubernetesEastWestRangeAnnotationPredicates []kubernetes.AnnotationPredicates `yaml:"-"` + KubernetesEastWestRangeAnnotationFiltersAppend []kubernetes.AnnotationFilters `yaml:"-"` + KubernetesAnnotationPredicates []kubernetes.AnnotationPredicates `yaml:"-"` + KubernetesAnnotationFiltersAppend []kubernetes.AnnotationFilters `yaml:"-"` + KubernetesEastWestRangePredicates []*eskip.Predicate `yaml:"-"` + KubernetesOnlyAllowedExternalNames bool `yaml:"kubernetes-only-allowed-external-names"` + KubernetesAllowedExternalNames regexpListFlag `yaml:"kubernetes-allowed-external-names"` + KubernetesRedisServiceNamespace string `yaml:"kubernetes-redis-service-namespace"` + KubernetesRedisServiceName string `yaml:"kubernetes-redis-service-name"` + KubernetesRedisServicePort int `yaml:"kubernetes-redis-service-port"` + KubernetesBackendTrafficAlgorithmString string `yaml:"kubernetes-backend-traffic-algorithm"` + KubernetesBackendTrafficAlgorithm kubernetes.BackendTrafficAlgorithm `yaml:"-"` + KubernetesDefaultLoadBalancerAlgorithm string `yaml:"kubernetes-default-lb-algorithm"` // Default filters DefaultFiltersDir string `yaml:"default-filters-dir"` @@ -475,7 +479,9 @@ func NewConfig() *Config { flag.Var(cfg.KubernetesEastWestRangeDomains, "kubernetes-east-west-range-domains", "set the the cluster internal domains for east west traffic. Identified routes to such domains will include the -kubernetes-east-west-range-predicates") flag.StringVar(&cfg.KubernetesEastWestRangePredicatesString, "kubernetes-east-west-range-predicates", "", "set the predicates that will be appended to routes identified as to -kubernetes-east-west-range-domains") flag.Var(&cfg.KubernetesAnnotationPredicatesString, "kubernetes-annotation-predicates", "configures predicates appended to non east-west routes of annotated resources. E.g. -kubernetes-annotation-predicates='zone-a=true=Foo() && Bar()' will add 'Foo() && Bar()' predicates to all non east-west routes of ingress or routegroup annotated with 'zone-a: true'. For east-west routes use -kubernetes-east-west-range-annotation-predicates.") + flag.Var(&cfg.KubernetesAnnotationFiltersAppendString, "kubernetes-annotation-filters-append", "configures filters appended to non east-west routes of annotated resources. E.g. -kubernetes-annotation-filters-append='zone-a=true=foo() -> bar()' will add 'foo() -> bar()' filters to all non east-west routes of ingress or routegroup annotated with 'zone-a: true'. For east-west routes use -kubernetes-east-west-range-annotation-filters-append.") flag.Var(&cfg.KubernetesEastWestRangeAnnotationPredicatesString, "kubernetes-east-west-range-annotation-predicates", "similar to -kubernetes-annotation-predicates configures predicates appended to east-west routes of annotated resources. See also -kubernetes-east-west-range-domains.") + flag.Var(&cfg.KubernetesEastWestRangeAnnotationFiltersAppendString, "kubernetes-east-west-range-annotation-filters-append", "similar to -kubernetes-annotation-filters-append configures filters appended to east-west routes of annotated resources. See also -kubernetes-east-west-range-domains.") flag.BoolVar(&cfg.KubernetesOnlyAllowedExternalNames, "kubernetes-only-allowed-external-names", false, "only accept external name services, route group network backends and route group explicit LB endpoints from an allow list defined by zero or more -kubernetes-allowed-external-name flags") flag.Var(&cfg.KubernetesAllowedExternalNames, "kubernetes-allowed-external-name", "set zero or more regular expressions from which at least one should be matched by the external name services, route group network addresses and explicit endpoints domain names") flag.StringVar(&cfg.KubernetesRedisServiceNamespace, "kubernetes-redis-service-namespace", "", "Sets namespace for redis to be used to lookup endpoints") @@ -623,11 +629,21 @@ func validate(c *Config) error { return fmt.Errorf("invalid annotation predicates: %q, %w", c.KubernetesAnnotationPredicatesString, err) } + _, err = parseAnnotationFilters(c.KubernetesAnnotationFiltersAppendString) + if err != nil { + return fmt.Errorf("invalid annotation filters: %q, %w", c.KubernetesAnnotationFiltersAppendString, err) + } + _, err = parseAnnotationPredicates(c.KubernetesEastWestRangeAnnotationPredicatesString) if err != nil { return fmt.Errorf("invalid east-west annotation predicates: %q, %w", c.KubernetesEastWestRangeAnnotationPredicatesString, err) } + _, err = parseAnnotationFilters(c.KubernetesEastWestRangeAnnotationFiltersAppendString) + if err != nil { + return fmt.Errorf("invalid east-west annotation filters: %q, %w", c.KubernetesEastWestRangeAnnotationFiltersAppendString, err) + } + _, err = kubernetes.ParseBackendTrafficAlgorithm(c.KubernetesBackendTrafficAlgorithmString) if err != nil { return err @@ -691,7 +707,9 @@ func (c *Config) ParseArgs(progname string, args []string) error { c.KubernetesPathMode, _ = kubernetes.ParsePathMode(c.KubernetesPathModeString) c.KubernetesEastWestRangePredicates, _ = eskip.ParsePredicates(c.KubernetesEastWestRangePredicatesString) c.KubernetesAnnotationPredicates, _ = parseAnnotationPredicates(c.KubernetesAnnotationPredicatesString) + c.KubernetesAnnotationFiltersAppend, _ = parseAnnotationFilters(c.KubernetesAnnotationFiltersAppendString) c.KubernetesEastWestRangeAnnotationPredicates, _ = parseAnnotationPredicates(c.KubernetesEastWestRangeAnnotationPredicatesString) + c.KubernetesEastWestRangeAnnotationFiltersAppend, _ = parseAnnotationFilters(c.KubernetesEastWestRangeAnnotationFiltersAppendString) c.KubernetesBackendTrafficAlgorithm, _ = kubernetes.ParseBackendTrafficAlgorithm(c.KubernetesBackendTrafficAlgorithmString) c.HistogramMetricBuckets, _ = c.parseHistogramBuckets() @@ -834,33 +852,35 @@ func (c *Config) ToOptions() skipper.Options { WaitFirstRouteLoad: c.WaitFirstRouteLoad, // Kubernetes: - Kubernetes: c.KubernetesIngress, - KubernetesInCluster: c.KubernetesInCluster, - KubernetesURL: c.KubernetesURL, - KubernetesTokenFile: c.KubernetesTokenFile, - KubernetesHealthcheck: c.KubernetesHealthcheck, - KubernetesHTTPSRedirect: c.KubernetesHTTPSRedirect, - KubernetesHTTPSRedirectCode: c.KubernetesHTTPSRedirectCode, - KubernetesDisableCatchAllRoutes: c.KubernetesDisableCatchAllRoutes, - KubernetesIngressClass: c.KubernetesIngressClass, - KubernetesRouteGroupClass: c.KubernetesRouteGroupClass, - WhitelistedHealthCheckCIDR: whitelistCIDRS, - KubernetesPathMode: c.KubernetesPathMode, - KubernetesNamespace: c.KubernetesNamespace, - KubernetesEnableEndpointslices: c.KubernetesEnableEndpointSlices, - KubernetesEnableEastWest: c.KubernetesEnableEastWest, - KubernetesEastWestDomain: c.KubernetesEastWestDomain, - KubernetesEastWestRangeDomains: c.KubernetesEastWestRangeDomains.values, - KubernetesEastWestRangePredicates: c.KubernetesEastWestRangePredicates, - KubernetesEastWestRangeAnnotationPredicates: c.KubernetesEastWestRangeAnnotationPredicates, - KubernetesAnnotationPredicates: c.KubernetesAnnotationPredicates, - KubernetesOnlyAllowedExternalNames: c.KubernetesOnlyAllowedExternalNames, - KubernetesAllowedExternalNames: c.KubernetesAllowedExternalNames, - KubernetesRedisServiceNamespace: c.KubernetesRedisServiceNamespace, - KubernetesRedisServiceName: c.KubernetesRedisServiceName, - KubernetesRedisServicePort: c.KubernetesRedisServicePort, - KubernetesBackendTrafficAlgorithm: c.KubernetesBackendTrafficAlgorithm, - KubernetesDefaultLoadBalancerAlgorithm: c.KubernetesDefaultLoadBalancerAlgorithm, + Kubernetes: c.KubernetesIngress, + KubernetesInCluster: c.KubernetesInCluster, + KubernetesURL: c.KubernetesURL, + KubernetesTokenFile: c.KubernetesTokenFile, + KubernetesHealthcheck: c.KubernetesHealthcheck, + KubernetesHTTPSRedirect: c.KubernetesHTTPSRedirect, + KubernetesHTTPSRedirectCode: c.KubernetesHTTPSRedirectCode, + KubernetesDisableCatchAllRoutes: c.KubernetesDisableCatchAllRoutes, + KubernetesIngressClass: c.KubernetesIngressClass, + KubernetesRouteGroupClass: c.KubernetesRouteGroupClass, + WhitelistedHealthCheckCIDR: whitelistCIDRS, + KubernetesPathMode: c.KubernetesPathMode, + KubernetesNamespace: c.KubernetesNamespace, + KubernetesEnableEndpointslices: c.KubernetesEnableEndpointSlices, + KubernetesEnableEastWest: c.KubernetesEnableEastWest, + KubernetesEastWestDomain: c.KubernetesEastWestDomain, + KubernetesEastWestRangeDomains: c.KubernetesEastWestRangeDomains.values, + KubernetesEastWestRangePredicates: c.KubernetesEastWestRangePredicates, + KubernetesEastWestRangeAnnotationPredicates: c.KubernetesEastWestRangeAnnotationPredicates, + KubernetesEastWestRangeAnnotationFiltersAppend: c.KubernetesEastWestRangeAnnotationFiltersAppend, + KubernetesAnnotationPredicates: c.KubernetesAnnotationPredicates, + KubernetesAnnotationFiltersAppend: c.KubernetesAnnotationFiltersAppend, + KubernetesOnlyAllowedExternalNames: c.KubernetesOnlyAllowedExternalNames, + KubernetesAllowedExternalNames: c.KubernetesAllowedExternalNames, + KubernetesRedisServiceNamespace: c.KubernetesRedisServiceNamespace, + KubernetesRedisServiceName: c.KubernetesRedisServiceName, + KubernetesRedisServicePort: c.KubernetesRedisServicePort, + KubernetesBackendTrafficAlgorithm: c.KubernetesBackendTrafficAlgorithm, + KubernetesDefaultLoadBalancerAlgorithm: c.KubernetesDefaultLoadBalancerAlgorithm, // API Monitoring: ApiUsageMonitoringEnable: c.ApiUsageMonitoringEnable, @@ -1173,42 +1193,70 @@ func (c *Config) checkDeprecated(configKeys map[string]interface{}, options ...s } func parseAnnotationPredicates(s []string) ([]kubernetes.AnnotationPredicates, error) { - var annotationPredicates []kubernetes.AnnotationPredicates + return parseAnnotationConfig(s, func(annotationKey, annotationValue, value string) (kubernetes.AnnotationPredicates, error) { + predicates, err := eskip.ParsePredicates(value) + if err != nil { + var zero kubernetes.AnnotationPredicates + return zero, err + } + return kubernetes.AnnotationPredicates{ + Key: annotationKey, + Value: annotationValue, + Predicates: predicates, + }, nil + }) +} - for _, annotationPredicate := range s { - if annotationPredicate == "" { +func parseAnnotationFilters(s []string) ([]kubernetes.AnnotationFilters, error) { + return parseAnnotationConfig(s, func(annotationKey, annotationValue, value string) (kubernetes.AnnotationFilters, error) { + filters, err := eskip.ParseFilters(value) + if err != nil { + var zero kubernetes.AnnotationFilters + return zero, err + } + return kubernetes.AnnotationFilters{ + Key: annotationKey, + Value: annotationValue, + Filters: filters, + }, nil + }) +} + +// parseAnnotationConfig parses a slice of strings in the "annotationKey=annotationValue=value" format +// by calling parseValue function to convert (annotationKey, annotationValue, value) tuple into T. +// Empty input strings are skipped and duplicate annotationKey-annotationValue pairs are rejected with error. +func parseAnnotationConfig[T any](kvvs []string, parseValue func(annotationKey, annotationValue, value string) (T, error)) ([]T, error) { + var result []T + seenKVs := make(map[string]struct{}) + for _, kvv := range kvvs { + if kvv == "" { continue } - annotationKey, rest, found := strings.Cut(annotationPredicate, "=") + annotationKey, rest, found := strings.Cut(kvv, "=") if !found { - return nil, fmt.Errorf("invalid annotation predicate flag: %q, failed to get annotation key", annotationPredicate) + return nil, fmt.Errorf("invalid annotation flag: %q, failed to get annotation key", kvv) } - annotationValue, predicates, found := strings.Cut(rest, "=") + annotationValue, value, found := strings.Cut(rest, "=") if !found { - return nil, fmt.Errorf("invalid annotation predicate flag: %q, faild to get annotation value", annotationPredicate) + return nil, fmt.Errorf("invalid annotation flag: %q, failed to get annotation value", kvv) } - predicateList, err := eskip.ParsePredicates(predicates) + v, err := parseValue(annotationKey, annotationValue, value) if err != nil { - return nil, fmt.Errorf("invalid annotation predicate flag: %q, %w", annotationPredicate, err) + return nil, fmt.Errorf("invalid annotation flag value: %q, %w", kvv, err) } - // We throw an err because having duplicate annotation keys will override each others - for _, ap := range annotationPredicates { - if ap.Key == annotationKey && ap.Value == annotationValue { - return nil, fmt.Errorf("invalid annotation predicate flag: %q, duplicate annotation key and value", annotationPredicate) - } + // Reject duplicate annotation key-value pairs + kv := annotationKey + "=" + annotationValue + if _, ok := seenKVs[kv]; ok { + return nil, fmt.Errorf("invalid annotation flag: %q, duplicate annotation key-value %q", kvv, kv) + } else { + seenKVs[kv] = struct{}{} } - annotationPredicates = append(annotationPredicates, kubernetes.AnnotationPredicates{ - Key: annotationKey, - Value: annotationValue, - Predicates: predicateList, - }) + result = append(result, v) } - - return annotationPredicates, nil - + return result, nil } diff --git a/config/config_test.go b/config/config_test.go index cf49e2c42d..cb4cd03c1a 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -495,117 +495,190 @@ func TestMultiFlagYamlErr(t *testing.T) { } } -func TestAnnotationPredicatesParsingInvalid(t *testing.T) { - for _, tc := range []struct { - name string - input []string - }{ - { - name: "wrong predicate", - input: []string{`to-add-predicate=true="Fo_o("123")"`}, - }, - { - name: "wrong predicate and empty value", - input: []string{"", `to-add-predicate=true="Fo_o()"`}, - }, - { - name: "duplicate", - input: []string{`to-add-predicate=true=Foo("123")`, `to-add-predicate=true=Bar("456")`}, - }, - } { - t.Run(tc.name, func(t *testing.T) { - _, err := parseAnnotationPredicates(tc.input) - require.Error(t, err) - }) - } +func TestParseAnnotationConfigInvalid(t *testing.T) { + t.Run("parseAnnotationPredicates", func(t *testing.T) { + for _, tc := range []struct { + name string + input []string + }{ + { + name: "wrong predicate", + input: []string{`to-add-predicate=true="Fo_o("123")"`}, + }, + { + name: "wrong predicate and empty value", + input: []string{"", `to-add-predicate=true="Fo_o()"`}, + }, + { + name: "duplicate", + input: []string{`to-add-predicate=true=Foo("123")`, `to-add-predicate=true=Bar("456")`}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := parseAnnotationPredicates(tc.input) + assert.Error(t, err) + }) + } + }) + + t.Run("parseAnnotationFilters", func(t *testing.T) { + for _, tc := range []struct { + name string + input []string + }{ + { + name: "invalid filter", + input: []string{`foo=bar=invalid-filter()`}, + }, + { + name: "invalid filter and empty value", + input: []string{"", `foo=bar=invalid-filter()`}, + }, + { + name: "duplicate", + input: []string{`foo=bar=baz("123")`, `foo=bar=qux("456")`}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := parseAnnotationFilters(tc.input) + assert.Error(t, err) + }) + } + }) } -func TestAnnotationPredicatesParsingValid(t *testing.T) { - for _, tc := range []struct { - name string - input []string - expected []kubernetes.AnnotationPredicates - }{ - { - name: "empty", - input: []string{}, - expected: nil, - }, - { - name: "empty string", - input: []string{""}, - expected: nil, - }, - { - name: "empty string and a valid value", - input: []string{ - "", - `to-add-predicate=true=Foo("123")`, +func TestParseAnnotationConfig(t *testing.T) { + t.Run("parseAnnotationPredicates", func(t *testing.T) { + + for _, tc := range []struct { + name string + input []string + expected []kubernetes.AnnotationPredicates + }{ + { + name: "empty", + input: []string{}, + expected: nil, }, - expected: []kubernetes.AnnotationPredicates{ - { - Key: "to-add-predicate", - Value: "true", - Predicates: []*eskip.Predicate{ - { - Name: "Foo", - Args: []any{"123"}, - }, + { + name: "empty string", + input: []string{""}, + expected: nil, + }, + { + name: "empty string and a valid value", + input: []string{ + "", + `to-add-predicate=true=Foo("123")`, + }, + expected: []kubernetes.AnnotationPredicates{ + { + Key: "to-add-predicate", + Value: "true", + Predicates: eskip.MustParsePredicates(`Foo("123")`), }, }, }, - }, - { - name: "single", - input: []string{`to-add-predicate=true=Foo("123")`}, - expected: []kubernetes.AnnotationPredicates{ - { - Key: "to-add-predicate", - Value: "true", - Predicates: []*eskip.Predicate{ - { - Name: "Foo", - Args: []any{"123"}, - }, + { + name: "single", + input: []string{`to-add-predicate=true=Foo("123")`}, + expected: []kubernetes.AnnotationPredicates{ + { + Key: "to-add-predicate", + Value: "true", + Predicates: eskip.MustParsePredicates(`Foo("123")`), }, }, }, - }, - { - name: "multiple", - input: []string{`to-add-predicate=true=Foo("123")`, `to-add-predicate=false=Bar("456") && Foo("789")`}, - expected: []kubernetes.AnnotationPredicates{ - { - Key: "to-add-predicate", - Value: "true", - Predicates: []*eskip.Predicate{ - { - Name: "Foo", - Args: []any{"123"}, - }, + { + name: "multiple", + input: []string{`to-add-predicate=true=Foo("123")`, `to-add-predicate=false=Bar("456") && Foo("789")`}, + expected: []kubernetes.AnnotationPredicates{ + { + Key: "to-add-predicate", + Value: "true", + Predicates: eskip.MustParsePredicates(`Foo("123")`), + }, + { + Key: "to-add-predicate", + Value: "false", + Predicates: eskip.MustParsePredicates(`Bar("456") && Foo("789")`), + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + val, err := parseAnnotationPredicates(tc.input) + require.NoError(t, err) + assert.Equal(t, tc.expected, val) + }) + } + }) + + t.Run("parseAnnotationFilters", func(t *testing.T) { + + for _, tc := range []struct { + name string + input []string + expected []kubernetes.AnnotationFilters + }{ + { + name: "empty", + input: []string{}, + expected: nil, + }, + { + name: "empty string", + input: []string{""}, + expected: nil, + }, + { + name: "empty string and a valid value", + input: []string{ + "", + `foo=true=baz("123=456")`, + }, + expected: []kubernetes.AnnotationFilters{ + { + Key: "foo", + Value: "true", + Filters: eskip.MustParseFilters(`baz("123=456")`), }, }, - { - Key: "to-add-predicate", - Value: "false", - Predicates: []*eskip.Predicate{ - { - Name: "Bar", - Args: []any{"456"}, - }, - { - Name: "Foo", - Args: []any{"789"}, - }, + }, + { + name: "single", + input: []string{`foo=true=baz("123=456")`}, + expected: []kubernetes.AnnotationFilters{ + { + Key: "foo", + Value: "true", + Filters: eskip.MustParseFilters(`baz("123=456")`), }, }, }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - val, err := parseAnnotationPredicates(tc.input) - require.NoError(t, err) - assert.Equal(t, tc.expected, val) - }) - } + { + name: "multiple", + input: []string{`foo=true=baz("123=456")`, `foo=false=bar("456") -> foo("789")`}, + expected: []kubernetes.AnnotationFilters{ + { + Key: "foo", + Value: "true", + Filters: eskip.MustParseFilters(`baz("123=456")`), + }, + { + Key: "foo", + Value: "false", + Filters: eskip.MustParseFilters(`bar("456") -> foo("789")`), + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + val, err := parseAnnotationFilters(tc.input) + require.NoError(t, err) + assert.Equal(t, tc.expected, val) + }) + } + }) } diff --git a/dataclients/kubernetes/annotations.go b/dataclients/kubernetes/annotations.go index 3e0dd95ea1..716a6e65aa 100644 --- a/dataclients/kubernetes/annotations.go +++ b/dataclients/kubernetes/annotations.go @@ -10,7 +10,13 @@ type AnnotationPredicates struct { Predicates []*eskip.Predicate } -func addAnnotationPredicates(annotationPredicates []AnnotationPredicates, annotations map[string]string, r *eskip.Route) { +type AnnotationFilters struct { + Key string + Value string + Filters []*eskip.Filter +} + +func appendAnnotationPredicates(annotationPredicates []AnnotationPredicates, annotations map[string]string, r *eskip.Route) { for _, ap := range annotationPredicates { if objAnnotationVal, ok := annotations[ap.Key]; ok && ap.Value == objAnnotationVal { // since this annotation is managed by skipper operator, we can safely assume that the predicate is valid @@ -19,3 +25,11 @@ func addAnnotationPredicates(annotationPredicates []AnnotationPredicates, annota } } } + +func appendAnnotationFilters(annotationFilters []AnnotationFilters, annotations map[string]string, r *eskip.Route) { + for _, af := range annotationFilters { + if objAnnotationVal, ok := annotations[af.Key]; ok && af.Value == objAnnotationVal { + r.Filters = append(r.Filters, af.Filters...) + } + } +} diff --git a/dataclients/kubernetes/ingress.go b/dataclients/kubernetes/ingress.go index 27896408ec..2315503a24 100644 --- a/dataclients/kubernetes/ingress.go +++ b/dataclients/kubernetes/ingress.go @@ -46,20 +46,22 @@ type ingressContext struct { } type ingress struct { - eastWestRangeDomains []string - eastWestRangePredicates []*eskip.Predicate - allowedExternalNames []*regexp.Regexp - kubernetesEastWestDomain string - pathMode PathMode - httpsRedirectCode int - kubernetesEnableEastWest bool - provideHTTPSRedirect bool - disableCatchAllRoutes bool - forceKubernetesService bool - backendTrafficAlgorithm BackendTrafficAlgorithm - defaultLoadBalancerAlgorithm string - kubernetesAnnotationPredicates []AnnotationPredicates - kubernetesEastWestRangeAnnotationPredicates []AnnotationPredicates + eastWestRangeDomains []string + eastWestRangePredicates []*eskip.Predicate + allowedExternalNames []*regexp.Regexp + kubernetesEastWestDomain string + pathMode PathMode + httpsRedirectCode int + kubernetesEnableEastWest bool + provideHTTPSRedirect bool + disableCatchAllRoutes bool + forceKubernetesService bool + backendTrafficAlgorithm BackendTrafficAlgorithm + defaultLoadBalancerAlgorithm string + kubernetesAnnotationPredicates []AnnotationPredicates + kubernetesAnnotationFiltersAppend []AnnotationFilters + kubernetesEastWestRangeAnnotationPredicates []AnnotationPredicates + kubernetesEastWestRangeAnnotationFiltersAppend []AnnotationFilters } var nonWord = regexp.MustCompile(`\W`) @@ -72,20 +74,22 @@ func (ic *ingressContext) addHostRoute(host string, route *eskip.Route) { func newIngress(o Options) *ingress { return &ingress{ - provideHTTPSRedirect: o.ProvideHTTPSRedirect, - httpsRedirectCode: o.HTTPSRedirectCode, - disableCatchAllRoutes: o.DisableCatchAllRoutes, - pathMode: o.PathMode, - kubernetesEnableEastWest: o.KubernetesEnableEastWest, - kubernetesEastWestDomain: o.KubernetesEastWestDomain, - eastWestRangeDomains: o.KubernetesEastWestRangeDomains, - eastWestRangePredicates: o.KubernetesEastWestRangePredicates, - allowedExternalNames: o.AllowedExternalNames, - forceKubernetesService: o.ForceKubernetesService, - backendTrafficAlgorithm: o.BackendTrafficAlgorithm, - defaultLoadBalancerAlgorithm: o.DefaultLoadBalancerAlgorithm, - kubernetesAnnotationPredicates: o.KubernetesAnnotationPredicates, - kubernetesEastWestRangeAnnotationPredicates: o.KubernetesEastWestRangeAnnotationPredicates, + provideHTTPSRedirect: o.ProvideHTTPSRedirect, + httpsRedirectCode: o.HTTPSRedirectCode, + disableCatchAllRoutes: o.DisableCatchAllRoutes, + pathMode: o.PathMode, + kubernetesEnableEastWest: o.KubernetesEnableEastWest, + kubernetesEastWestDomain: o.KubernetesEastWestDomain, + eastWestRangeDomains: o.KubernetesEastWestRangeDomains, + eastWestRangePredicates: o.KubernetesEastWestRangePredicates, + allowedExternalNames: o.AllowedExternalNames, + forceKubernetesService: o.ForceKubernetesService, + backendTrafficAlgorithm: o.BackendTrafficAlgorithm, + defaultLoadBalancerAlgorithm: o.DefaultLoadBalancerAlgorithm, + kubernetesAnnotationPredicates: o.KubernetesAnnotationPredicates, + kubernetesAnnotationFiltersAppend: o.KubernetesAnnotationFiltersAppend, + kubernetesEastWestRangeAnnotationPredicates: o.KubernetesEastWestRangeAnnotationPredicates, + kubernetesEastWestRangeAnnotationFiltersAppend: o.KubernetesEastWestRangeAnnotationFiltersAppend, } } @@ -203,9 +207,11 @@ func (ing *ingress) addExtraRoutes(ic *ingressContext, ruleHost, path, pathType setPathV1(ic.pathMode, &route, pathType, path) if n := countPathPredicates(&route); n <= 1 { if ewHost { - addAnnotationPredicates(ing.kubernetesEastWestRangeAnnotationPredicates, ic.ingressV1.Metadata.Annotations, &route) + appendAnnotationPredicates(ing.kubernetesEastWestRangeAnnotationPredicates, ic.ingressV1.Metadata.Annotations, &route) + appendAnnotationFilters(ing.kubernetesEastWestRangeAnnotationFiltersAppend, ic.ingressV1.Metadata.Annotations, &route) } else { - addAnnotationPredicates(ing.kubernetesAnnotationPredicates, ic.ingressV1.Metadata.Annotations, &route) + appendAnnotationPredicates(ing.kubernetesAnnotationPredicates, ic.ingressV1.Metadata.Annotations, &route) + appendAnnotationFilters(ing.kubernetesAnnotationFiltersAppend, ic.ingressV1.Metadata.Annotations, &route) } ic.addHostRoute(ruleHost, &route) ic.redirect.updateHost(ruleHost) diff --git a/dataclients/kubernetes/ingress_test.go b/dataclients/kubernetes/ingress_test.go index 9c7f5a1c68..3c05bd5e7c 100644 --- a/dataclients/kubernetes/ingress_test.go +++ b/dataclients/kubernetes/ingress_test.go @@ -20,6 +20,12 @@ func TestIngressV1Fixtures(t *testing.T) { "testdata/ingressV1/traffic", "testdata/ingressV1/traffic-segment", "testdata/ingressV1/loadbalancer-algorithm", + ) +} + +func TestIngressV1AnnotationConfig(t *testing.T) { + kubernetestest.FixturesToTest(t, "testdata/ingressV1/annotation-predicates", + "testdata/ingressV1/annotation-filters", ) } diff --git a/dataclients/kubernetes/ingressv1.go b/dataclients/kubernetes/ingressv1.go index fed5caeed6..8b525fe2b4 100644 --- a/dataclients/kubernetes/ingressv1.go +++ b/dataclients/kubernetes/ingressv1.go @@ -230,9 +230,11 @@ func (ing *ingress) addEndpointsRuleV1(ic *ingressContext, host string, prule *d redirect.setHost(host) } - addAnnotationPredicates(ing.kubernetesAnnotationPredicates, meta.Annotations, endpointsRoute) + appendAnnotationPredicates(ing.kubernetesAnnotationPredicates, meta.Annotations, endpointsRoute) + appendAnnotationFilters(ing.kubernetesAnnotationFiltersAppend, meta.Annotations, endpointsRoute) } else { - addAnnotationPredicates(ing.kubernetesEastWestRangeAnnotationPredicates, meta.Annotations, endpointsRoute) + appendAnnotationPredicates(ing.kubernetesEastWestRangeAnnotationPredicates, meta.Annotations, endpointsRoute) + appendAnnotationFilters(ing.kubernetesEastWestRangeAnnotationFiltersAppend, meta.Annotations, endpointsRoute) } if ing.kubernetesEnableEastWest { diff --git a/dataclients/kubernetes/kube.go b/dataclients/kubernetes/kube.go index 671c4e2788..a66d0fe237 100644 --- a/dataclients/kubernetes/kube.go +++ b/dataclients/kubernetes/kube.go @@ -212,9 +212,16 @@ type Options struct { // routes that has KubernetesEastWestRangeDomains suffix. KubernetesEastWestRangeAnnotationPredicates []AnnotationPredicates - // KubernetesAnnotationPredicates set a list predicates for each annotation key and value + // KubernetesEastWestRangeAnnotationFiltersAppend same as KubernetesAnnotationFiltersAppend but will append to + // routes that has KubernetesEastWestRangeDomains suffix. + KubernetesEastWestRangeAnnotationFiltersAppend []AnnotationFilters + + // KubernetesAnnotationPredicates sets predicates to append for each annotation key and value KubernetesAnnotationPredicates []AnnotationPredicates + // KubernetesAnnotationFiltersAppend sets filters to append for each annotation key and value + KubernetesAnnotationFiltersAppend []AnnotationFilters + // DefaultFiltersDir enables default filters mechanism and sets the location of the default filters. // The provided filters are then applied to all routes. DefaultFiltersDir string diff --git a/dataclients/kubernetes/kubernetestest/fixtures.go b/dataclients/kubernetes/kubernetestest/fixtures.go index a17aa7ea40..d45f7dc3be 100644 --- a/dataclients/kubernetes/kubernetestest/fixtures.go +++ b/dataclients/kubernetes/kubernetestest/fixtures.go @@ -32,29 +32,31 @@ type fixtureSet struct { } type kubeOptionsParser struct { - IngressV1 bool `yaml:"ingressv1"` - EastWest bool `yaml:"eastWest"` - EastWestDomain string `yaml:"eastWestDomain"` - EastWestRangeDomains []string `yaml:"eastWestRangeDomains"` - EastWestRangePredicates []*eskip.Predicate `yaml:"eastWestRangePredicatesAppend"` - HTTPSRedirect bool `yaml:"httpsRedirect"` - HTTPSRedirectCode int `yaml:"httpsRedirectCode"` - DisableCatchAllRoutes bool `yaml:"disableCatchAllRoutes"` - BackendNameTracingTag bool `yaml:"backendNameTracingTag"` - OnlyAllowedExternalNames bool `yaml:"onlyAllowedExternalNames"` - AllowedExternalNames []string `yaml:"allowedExternalNames"` - IngressClass string `yaml:"kubernetes-ingress-class"` - KubernetesEnableEndpointSlices bool `yaml:"enable-kubernetes-endpointslices"` - KubernetesEnableTLS bool `yaml:"kubernetes-enable-tls"` - IngressesLabels map[string]string `yaml:"kubernetes-ingresses-label-selector"` - ServicesLabels map[string]string `yaml:"kubernetes-services-label-selector"` - EndpointsLabels map[string]string `yaml:"kubernetes-endpoints-label-selector"` - EndpointsliceLabels map[string]string `yaml:"kubernetes-endpointslice-label-selector"` - ForceKubernetesService bool `yaml:"force-kubernetes-service"` - BackendTrafficAlgorithm string `yaml:"backend-traffic-algorithm"` - DefaultLoadBalancerAlgorithm string `yaml:"default-lb-algorithm"` - KubernetesAnnotationPredicates []kubernetes.AnnotationPredicates `yaml:"kubernetesAnnotationPredicates"` - KubernetesEastWestRangeAnnotationPredicates []kubernetes.AnnotationPredicates `yaml:"kubernetesEastWestRangeAnnotationPredicates"` + IngressV1 bool `yaml:"ingressv1"` + EastWest bool `yaml:"eastWest"` + EastWestDomain string `yaml:"eastWestDomain"` + EastWestRangeDomains []string `yaml:"eastWestRangeDomains"` + EastWestRangePredicates []*eskip.Predicate `yaml:"eastWestRangePredicatesAppend"` + HTTPSRedirect bool `yaml:"httpsRedirect"` + HTTPSRedirectCode int `yaml:"httpsRedirectCode"` + DisableCatchAllRoutes bool `yaml:"disableCatchAllRoutes"` + BackendNameTracingTag bool `yaml:"backendNameTracingTag"` + OnlyAllowedExternalNames bool `yaml:"onlyAllowedExternalNames"` + AllowedExternalNames []string `yaml:"allowedExternalNames"` + IngressClass string `yaml:"kubernetes-ingress-class"` + KubernetesEnableEndpointSlices bool `yaml:"enable-kubernetes-endpointslices"` + KubernetesEnableTLS bool `yaml:"kubernetes-enable-tls"` + IngressesLabels map[string]string `yaml:"kubernetes-ingresses-label-selector"` + ServicesLabels map[string]string `yaml:"kubernetes-services-label-selector"` + EndpointsLabels map[string]string `yaml:"kubernetes-endpoints-label-selector"` + EndpointsliceLabels map[string]string `yaml:"kubernetes-endpointslice-label-selector"` + ForceKubernetesService bool `yaml:"force-kubernetes-service"` + BackendTrafficAlgorithm string `yaml:"backend-traffic-algorithm"` + DefaultLoadBalancerAlgorithm string `yaml:"default-lb-algorithm"` + KubernetesAnnotationPredicates []kubernetes.AnnotationPredicates `yaml:"kubernetesAnnotationPredicates"` + KubernetesAnnotationFiltersAppend []kubernetes.AnnotationFilters `yaml:"kubernetesAnnotationFiltersAppend"` + KubernetesEastWestRangeAnnotationPredicates []kubernetes.AnnotationPredicates `yaml:"kubernetesEastWestRangeAnnotationPredicates"` + KubernetesEastWestRangeAnnotationFiltersAppend []kubernetes.AnnotationFilters `yaml:"kubernetesEastWestRangeAnnotationFiltersAppend"` } func baseNoExt(n string) string { @@ -232,7 +234,9 @@ func testFixture(t *testing.T, f fixtureSet) { o.KubernetesEastWestRangeDomains = kop.EastWestRangeDomains o.KubernetesEastWestRangePredicates = kop.EastWestRangePredicates o.KubernetesAnnotationPredicates = kop.KubernetesAnnotationPredicates + o.KubernetesAnnotationFiltersAppend = kop.KubernetesAnnotationFiltersAppend o.KubernetesEastWestRangeAnnotationPredicates = kop.KubernetesEastWestRangeAnnotationPredicates + o.KubernetesEastWestRangeAnnotationFiltersAppend = kop.KubernetesEastWestRangeAnnotationFiltersAppend o.ProvideHTTPSRedirect = kop.HTTPSRedirect o.HTTPSRedirectCode = kop.HTTPSRedirectCode o.DisableCatchAllRoutes = kop.DisableCatchAllRoutes @@ -302,7 +306,7 @@ func testFixture(t *testing.T, f fixtureSet) { t.Logf("routes: %d, expected: %d", len(routes), len(expectedRoutes)) t.Logf("got:\n%s", eskip.String(eskip.CanonicalList(routes)...)) t.Logf("expected:\n%s", eskip.String(eskip.CanonicalList(expectedRoutes)...)) - t.Logf("diff\n%s:", cmp.Diff( + t.Logf("diff:\n%s", cmp.Diff( eskip.Print(eskip.PrettyPrintInfo{Pretty: true}, eskip.CanonicalList(expectedRoutes)...), eskip.Print(eskip.PrettyPrintInfo{Pretty: true}, eskip.CanonicalList(routes)...), )) diff --git a/dataclients/kubernetes/routegroup.go b/dataclients/kubernetes/routegroup.go index 6f4a3b3fa3..dfcbf18de9 100644 --- a/dataclients/kubernetes/routegroup.go +++ b/dataclients/kubernetes/routegroup.go @@ -592,7 +592,8 @@ func (r *routeGroups) convert(s *clusterState, df defaultFilters, loggingEnabled } for _, route := range ri { - addAnnotationPredicates(r.options.KubernetesAnnotationPredicates, rg.Metadata.Annotations, route) + appendAnnotationPredicates(r.options.KubernetesAnnotationPredicates, rg.Metadata.Annotations, route) + appendAnnotationFilters(r.options.KubernetesAnnotationFiltersAppend, rg.Metadata.Annotations, route) } rs = append(rs, ri...) @@ -634,7 +635,8 @@ func (r *routeGroups) convert(s *clusterState, df defaultFilters, loggingEnabled applyEastWestRangePredicates(internalRi, r.options.KubernetesEastWestRangePredicates) for _, route := range internalRi { - addAnnotationPredicates(r.options.KubernetesEastWestRangeAnnotationPredicates, rg.Metadata.Annotations, route) + appendAnnotationPredicates(r.options.KubernetesEastWestRangeAnnotationPredicates, rg.Metadata.Annotations, route) + appendAnnotationFilters(r.options.KubernetesEastWestRangeAnnotationFiltersAppend, rg.Metadata.Annotations, route) } if internalCtx.certificateRegistry != nil { diff --git a/dataclients/kubernetes/routegroups_test.go b/dataclients/kubernetes/routegroups_test.go index ac9f2c89bd..365ed82f4b 100644 --- a/dataclients/kubernetes/routegroups_test.go +++ b/dataclients/kubernetes/routegroups_test.go @@ -62,6 +62,8 @@ func TestRouteGroupTLS(t *testing.T) { kubernetestest.FixturesToTest(t, "testdata/routegroups/tls") } -func TestAnnotationPredicates(t *testing.T) { - kubernetestest.FixturesToTest(t, "testdata/routegroups/annotation-predicates") +func TestRouteGroupAnnotationConfig(t *testing.T) { + kubernetestest.FixturesToTest(t, + "testdata/routegroups/annotation-predicates", + "testdata/routegroups/annotation-filters") } diff --git a/dataclients/kubernetes/testdata/ingressV1/annotation-filters/annotation-filters-append.eskip b/dataclients/kubernetes/testdata/ingressV1/annotation-filters/annotation-filters-append.eskip new file mode 100644 index 0000000000..29b208fed1 --- /dev/null +++ b/dataclients/kubernetes/testdata/ingressV1/annotation-filters/annotation-filters-append.eskip @@ -0,0 +1,43 @@ +kube_default__myapp1__zone1_test____myapp: + Host("^(zone1[.]test[.]?(:[0-9]+)?)$") + && True() + -> comment("zalando.org/skipper-filter [1]") + -> comment("zalando.org/skipper-filter [2]") + -> comment("zalando.org/zone=zone1 [1]") + -> comment("foo [1]") + -> ; + +kube_default__myapp1_r1_0__zone1_test____: + Host("^(zone1[.]test[.]?(:[0-9]+)?)$") + && Path("/r1") + && True() + -> comment("zalando.org/skipper-routes [1]") + -> comment("zalando.org/zone=zone1 [1]") + -> comment("foo [1]") + -> "https://zone1.test"; + + +kube_default__myapp1__zone1_ingress_cluster_local____myapp: + Host("^(zone1[.]ingress[.]cluster[.]local[.]?(:[0-9]+)?)$") + && ClientIP("10.2.0.0/16") + && Weight(10) + -> comment("zalando.org/skipper-filter [1]") + -> comment("zalando.org/skipper-filter [2]") + -> comment("zalando.org/zone=zone1 [1] east-west") + -> comment("foo=bar [1] east-west") + -> ; + +kube_default__myapp1_r1_0__zone1_ingress_cluster_local____: + Host("^(zone1[.]ingress[.]cluster[.]local[.]?(:[0-9]+)?)$") + && Path("/r1") + && ClientIP("10.2.0.0/16") + && Weight(10) + -> comment("zalando.org/skipper-routes [1]") + -> comment("zalando.org/zone=zone1 [1] east-west") + -> comment("foo=bar [1] east-west") + -> "https://zone1.test"; + + +kube_default__myapp2__zone2_test____myapp: + Host("^(zone2[.]test[.]?(:[0-9]+)?)$") + -> ; diff --git a/dataclients/kubernetes/testdata/ingressV1/annotation-filters/annotation-filters-append.kube b/dataclients/kubernetes/testdata/ingressV1/annotation-filters/annotation-filters-append.kube new file mode 100644 index 0000000000..f7cc9eae04 --- /dev/null +++ b/dataclients/kubernetes/testdata/ingressV1/annotation-filters/annotation-filters-append.kube @@ -0,0 +1,48 @@ +# +# Test that multiple annotations: +# * append filters in the configured order +# * append different filters for east-west domain +# * append both filters and predicates +# * work with zalando.org/skipper-filter annotation +# * work with zalando.org/skipper-routes annotation +# +kubernetesAnnotationFiltersAppend: + - key: zalando.org/zone + value: zone1 + filters: + - name: comment + args: ["zalando.org/zone=zone1 [1]"] + - key: foo + value: bar + filters: + - name: comment + args: ["foo [1]"] +kubernetesEastWestRangeAnnotationFiltersAppend: + - key: zalando.org/zone + value: zone1 + filters: + - name: comment + args: ["zalando.org/zone=zone1 [1] east-west"] + - key: foo + value: bar + filters: + - name: comment + args: ["foo=bar [1] east-west"] + +kubernetesAnnotationPredicates: + - key: zalando.org/zone + value: zone1 + predicates: + - name: True +kubernetesEastWestRangeAnnotationPredicates: + - key: zalando.org/zone + value: zone1 + predicates: + - name: Weight + args: [10.0] + +eastWestRangeDomains: + - ingress.cluster.local +eastWestRangePredicatesAppend: + - name: ClientIP + args: ["10.2.0.0/16"] diff --git a/dataclients/kubernetes/testdata/ingressV1/annotation-filters/annotation-filters-append.yaml b/dataclients/kubernetes/testdata/ingressV1/annotation-filters/annotation-filters-append.yaml new file mode 100644 index 0000000000..3923845517 --- /dev/null +++ b/dataclients/kubernetes/testdata/ingressV1/annotation-filters/annotation-filters-append.yaml @@ -0,0 +1,90 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: myapp1 + namespace: default + annotations: + foo: bar + zalando.org/zone: zone1 + zalando.org/skipper-filter: | + comment("zalando.org/skipper-filter [1]") -> + comment("zalando.org/skipper-filter [2]") + zalando.org/skipper-routes: | + r1: Path("/r1") + -> comment("zalando.org/skipper-routes [1]") + -> "https://zone1.test"; +spec: + rules: + - host: zone1.test + http: + paths: + - backend: + service: + name: myapp + port: + number: 8080 + pathType: ImplementationSpecific + - host: zone1.ingress.cluster.local + http: + paths: + - backend: + service: + name: myapp + port: + number: 8080 + pathType: ImplementationSpecific +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: myapp2 + namespace: default + annotations: + zalando.org/zone: unknown +spec: + rules: + - host: zone2.test + http: + paths: + - backend: + service: + name: myapp + port: + number: 8080 + pathType: ImplementationSpecific +--- +apiVersion: v1 +kind: Service +metadata: + labels: + application: myapp + name: myapp +spec: + clusterIP: 10.3.190.97 + ports: + - name: foo + port: 8080 + protocol: TCP + targetPort: web + - name: web + port: 80 + protocol: TCP + targetPort: foo + selector: + application: myapp + type: ClusterIP +--- +apiVersion: v1 +kind: Endpoints +metadata: + labels: + application: myapp + name: myapp +subsets: + - addresses: + - ip: 10.2.9.103 + - ip: 10.2.9.104 + ports: + - name: foo + port: 8080 + protocol: TCP diff --git a/dataclients/kubernetes/testdata/routegroups/annotation-filters/annotation-filters-append.eskip b/dataclients/kubernetes/testdata/routegroups/annotation-filters/annotation-filters-append.eskip new file mode 100644 index 0000000000..1c9d7aca49 --- /dev/null +++ b/dataclients/kubernetes/testdata/routegroups/annotation-filters/annotation-filters-append.eskip @@ -0,0 +1,57 @@ +kube_rg__default__myapp1__all__0_0: + Host("^(zone1[.]test[.]?(:[0-9]+)?)$") + && PathSubtree("/") + && True() + -> comment("zalando.org/zone=zone1 [1]") + -> comment("foo [1]") + -> ; + +kube_rg__default__myapp1__all__1_0: + Host("^(zone1[.]test[.]?(:[0-9]+)?)$") + && PathSubtree("/shunt") + && True() + -> comment("zalando.org/zone=zone1 [1]") + -> comment("foo [1]") + -> ; + +kube_rg____zone1_test__catchall__0_0: + Host("^(zone1[.]test[.]?(:[0-9]+)?)$") + && True() + -> comment("zalando.org/zone=zone1 [1]") + -> comment("foo [1]") + -> ; + +kube_rg__internal_default__myapp1__all__0_0: + Host("^(zone1[.]ingress[.]cluster[.]local[.]?(:[0-9]+)?)$") + && PathSubtree("/") + && ClientIP("10.2.0.0/16") + && Weight(10) + -> comment("zalando.org/zone=zone1 [1] east-west") + -> comment("foo=bar [1] east-west") + -> ; + +kube_rg__internal_default__myapp1__all__1_0: + Host("^(zone1[.]ingress[.]cluster[.]local[.]?(:[0-9]+)?)$") + && PathSubtree("/shunt") + && ClientIP("10.2.0.0/16") + && Weight(10) + -> comment("zalando.org/zone=zone1 [1] east-west") + -> comment("foo=bar [1] east-west") + -> ; + +kube_rg__internal___zone1_ingress_cluster_local__catchall__0_0: + Host("^(zone1[.]ingress[.]cluster[.]local[.]?(:[0-9]+)?)$") + && ClientIP("10.2.0.0/16") + && Weight(10) + -> comment("zalando.org/zone=zone1 [1] east-west") + -> comment("foo=bar [1] east-west") + -> ; + +kube_rg__default__myapp2__all__0_0: + Host("^(zone2[.]test[.]?(:[0-9]+)?)$") + && PathSubtree("/") + -> ; + +kube_rg____zone2_test__catchall__0_0: + Host("^(zone2[.]test[.]?(:[0-9]+)?)$") + -> ; diff --git a/dataclients/kubernetes/testdata/routegroups/annotation-filters/annotation-filters-append.kube b/dataclients/kubernetes/testdata/routegroups/annotation-filters/annotation-filters-append.kube new file mode 100644 index 0000000000..f9ceedaea3 --- /dev/null +++ b/dataclients/kubernetes/testdata/routegroups/annotation-filters/annotation-filters-append.kube @@ -0,0 +1,46 @@ +# +# Test that multiple annotations: +# * append filters in the configured order +# * append different filters for east-west domain +# * append both filters and predicates +# +kubernetesAnnotationFiltersAppend: + - key: zalando.org/zone + value: zone1 + filters: + - name: comment + args: ["zalando.org/zone=zone1 [1]"] + - key: foo + value: bar + filters: + - name: comment + args: ["foo [1]"] +kubernetesEastWestRangeAnnotationFiltersAppend: + - key: zalando.org/zone + value: zone1 + filters: + - name: comment + args: ["zalando.org/zone=zone1 [1] east-west"] + - key: foo + value: bar + filters: + - name: comment + args: ["foo=bar [1] east-west"] + +kubernetesAnnotationPredicates: + - key: zalando.org/zone + value: zone1 + predicates: + - name: True +kubernetesEastWestRangeAnnotationPredicates: + - key: zalando.org/zone + value: zone1 + predicates: + - name: Weight + args: [10.0] + +eastWestRangeDomains: + - ingress.cluster.local +eastWestRangePredicatesAppend: + - name: ClientIP + args: ["10.2.0.0/16"] diff --git a/dataclients/kubernetes/testdata/routegroups/annotation-filters/annotation-filters-append.yaml b/dataclients/kubernetes/testdata/routegroups/annotation-filters/annotation-filters-append.yaml new file mode 100644 index 0000000000..a1f053c6b8 --- /dev/null +++ b/dataclients/kubernetes/testdata/routegroups/annotation-filters/annotation-filters-append.yaml @@ -0,0 +1,77 @@ +apiVersion: zalando.org/v1 +kind: RouteGroup +metadata: + name: myapp1 + annotations: + foo: bar + zalando.org/zone: zone1 +spec: + hosts: + - zone1.test + - zone1.ingress.cluster.local + backends: + - name: myapp + type: service + serviceName: myapp + servicePort: 80 + - name: shunt + type: shunt + defaultBackends: + - backendName: shunt + routes: + - pathSubtree: / + backends: + - backendName: myapp + - pathSubtree: /shunt +--- +apiVersion: zalando.org/v1 +kind: RouteGroup +metadata: + name: myapp2 + annotations: + zalando.org/zone: unknown +spec: + hosts: + - zone2.test + backends: + - name: myapp + type: service + serviceName: myapp + servicePort: 80 + routes: + - pathSubtree: / + backends: + - backendName: myapp +--- +apiVersion: v1 +kind: Service +metadata: + labels: + application: myapp + name: myapp +spec: + clusterIP: 10.3.190.97 + ports: + - name: main + port: 80 + protocol: TCP + targetPort: 7272 + selector: + application: myapp + type: ClusterIP +--- +apiVersion: v1 +kind: Endpoints +metadata: + labels: + application: myapp + name: myapp + namespace: default +subsets: + - addresses: + - ip: 10.2.9.103 + - ip: 10.2.9.104 + ports: + - name: main + port: 7272 + protocol: TCP diff --git a/skipper.go b/skipper.go index 0bdfd9e541..0cfd71bd90 100644 --- a/skipper.go +++ b/skipper.go @@ -272,9 +272,16 @@ type Options struct { // routes that has KubernetesEastWestRangeDomains suffix. KubernetesEastWestRangeAnnotationPredicates []kubernetes.AnnotationPredicates - // KubernetesAnnotationPredicates set a list predicates for each annotation key and value + // KubernetesEastWestRangeAnnotationFiltersAppend same as KubernetesAnnotationFiltersAppend but will append to + // routes that has KubernetesEastWestRangeDomains suffix. + KubernetesEastWestRangeAnnotationFiltersAppend []kubernetes.AnnotationFilters + + // KubernetesAnnotationPredicates sets predicates to append for each annotation key and value KubernetesAnnotationPredicates []kubernetes.AnnotationPredicates + // KubernetesAnnotationFiltersAppend sets filters to append for each annotation key and value + KubernetesAnnotationFiltersAppend []kubernetes.AnnotationFilters + // KubernetesOnlyAllowedExternalNames will enable validation of ingress external names and route groups network // backend addresses, explicit LB endpoints validation against the list of patterns in // AllowedExternalNames. @@ -958,39 +965,41 @@ type Options struct { func (o *Options) KubernetesDataClientOptions() kubernetes.Options { return kubernetes.Options{ - AllowedExternalNames: o.KubernetesAllowedExternalNames, - BackendNameTracingTag: o.OpenTracingBackendNameTag, - DefaultFiltersDir: o.DefaultFiltersDir, - KubernetesInCluster: o.KubernetesInCluster, - KubernetesURL: o.KubernetesURL, - TokenFile: o.KubernetesTokenFile, - KubernetesNamespace: o.KubernetesNamespace, - KubernetesEnableEastWest: o.KubernetesEnableEastWest, - KubernetesEnableEndpointslices: o.KubernetesEnableEndpointslices, - KubernetesEastWestDomain: o.KubernetesEastWestDomain, - KubernetesEastWestRangeDomains: o.KubernetesEastWestRangeDomains, - KubernetesEastWestRangePredicates: o.KubernetesEastWestRangePredicates, - KubernetesEastWestRangeAnnotationPredicates: o.KubernetesEastWestRangeAnnotationPredicates, - KubernetesAnnotationPredicates: o.KubernetesAnnotationPredicates, - HTTPSRedirectCode: o.KubernetesHTTPSRedirectCode, - DisableCatchAllRoutes: o.KubernetesDisableCatchAllRoutes, - IngressClass: o.KubernetesIngressClass, - IngressLabelSelectors: o.KubernetesIngressLabelSelectors, - ServicesLabelSelectors: o.KubernetesServicesLabelSelectors, - EndpointsLabelSelectors: o.KubernetesEndpointsLabelSelectors, - SecretsLabelSelectors: o.KubernetesSecretsLabelSelectors, - RouteGroupsLabelSelectors: o.KubernetesRouteGroupsLabelSelectors, - OnlyAllowedExternalNames: o.KubernetesOnlyAllowedExternalNames, - OriginMarker: o.EnableRouteCreationMetrics, - PathMode: o.KubernetesPathMode, - ProvideHealthcheck: o.KubernetesHealthcheck, - ProvideHTTPSRedirect: o.KubernetesHTTPSRedirect, - ReverseSourcePredicate: o.ReverseSourcePredicate, - RouteGroupClass: o.KubernetesRouteGroupClass, - WhitelistedHealthCheckCIDR: o.WhitelistedHealthCheckCIDR, - ForceKubernetesService: o.KubernetesForceService, - BackendTrafficAlgorithm: o.KubernetesBackendTrafficAlgorithm, - DefaultLoadBalancerAlgorithm: o.KubernetesDefaultLoadBalancerAlgorithm, + AllowedExternalNames: o.KubernetesAllowedExternalNames, + BackendNameTracingTag: o.OpenTracingBackendNameTag, + DefaultFiltersDir: o.DefaultFiltersDir, + KubernetesInCluster: o.KubernetesInCluster, + KubernetesURL: o.KubernetesURL, + TokenFile: o.KubernetesTokenFile, + KubernetesNamespace: o.KubernetesNamespace, + KubernetesEnableEastWest: o.KubernetesEnableEastWest, + KubernetesEnableEndpointslices: o.KubernetesEnableEndpointslices, + KubernetesEastWestDomain: o.KubernetesEastWestDomain, + KubernetesEastWestRangeDomains: o.KubernetesEastWestRangeDomains, + KubernetesEastWestRangePredicates: o.KubernetesEastWestRangePredicates, + KubernetesEastWestRangeAnnotationPredicates: o.KubernetesEastWestRangeAnnotationPredicates, + KubernetesEastWestRangeAnnotationFiltersAppend: o.KubernetesEastWestRangeAnnotationFiltersAppend, + KubernetesAnnotationPredicates: o.KubernetesAnnotationPredicates, + KubernetesAnnotationFiltersAppend: o.KubernetesAnnotationFiltersAppend, + HTTPSRedirectCode: o.KubernetesHTTPSRedirectCode, + DisableCatchAllRoutes: o.KubernetesDisableCatchAllRoutes, + IngressClass: o.KubernetesIngressClass, + IngressLabelSelectors: o.KubernetesIngressLabelSelectors, + ServicesLabelSelectors: o.KubernetesServicesLabelSelectors, + EndpointsLabelSelectors: o.KubernetesEndpointsLabelSelectors, + SecretsLabelSelectors: o.KubernetesSecretsLabelSelectors, + RouteGroupsLabelSelectors: o.KubernetesRouteGroupsLabelSelectors, + OnlyAllowedExternalNames: o.KubernetesOnlyAllowedExternalNames, + OriginMarker: o.EnableRouteCreationMetrics, + PathMode: o.KubernetesPathMode, + ProvideHealthcheck: o.KubernetesHealthcheck, + ProvideHTTPSRedirect: o.KubernetesHTTPSRedirect, + ReverseSourcePredicate: o.ReverseSourcePredicate, + RouteGroupClass: o.KubernetesRouteGroupClass, + WhitelistedHealthCheckCIDR: o.WhitelistedHealthCheckCIDR, + ForceKubernetesService: o.KubernetesForceService, + BackendTrafficAlgorithm: o.KubernetesBackendTrafficAlgorithm, + DefaultLoadBalancerAlgorithm: o.KubernetesDefaultLoadBalancerAlgorithm, } }