From 5b554978623b3d49d869b0a8e3d0fec7244a9931 Mon Sep 17 00:00:00 2001 From: B Pearson Date: Tue, 25 Feb 2025 10:58:20 -0500 Subject: [PATCH] adds pagerduty as an endpoint #781 Signed-off-by: B Pearson --- pkg/target/factory.go | 6 + pkg/target/factory/factory.go | 45 +++++++ pkg/target/pagerduty/pagerduty.go | 178 +++++++++++++++++++++++++ pkg/target/pagerduty/pagerduty_test.go | 167 +++++++++++++++++++++++ 4 files changed, 396 insertions(+) create mode 100644 pkg/target/pagerduty/pagerduty.go create mode 100644 pkg/target/pagerduty/pagerduty_test.go diff --git a/pkg/target/factory.go b/pkg/target/factory.go index 9f50ab6c..2d53da22 100644 --- a/pkg/target/factory.go +++ b/pkg/target/factory.go @@ -140,6 +140,11 @@ type GCSOptions struct { Bucket string `mapstructure:"bucket"` } +type PagerDutyOptions struct { + APIToken string `mapstructure:"apiToken"` + ServiceID string `mapstructure:"serviceId"` +} + type Targets struct { Loki *Config[LokiOptions] `mapstructure:"loki"` Elasticsearch *Config[ElasticsearchOptions] `mapstructure:"elasticsearch"` @@ -153,6 +158,7 @@ type Targets struct { Kinesis *Config[KinesisOptions] `mapstructure:"kinesis"` SecurityHub *Config[SecurityHubOptions] `mapstructure:"securityHub"` GCS *Config[GCSOptions] `mapstructure:"gcs"` + PagerDuty *Config[PagerDutyOptions] `mapstructure:"pagerduty"` } type Factory interface { diff --git a/pkg/target/factory/factory.go b/pkg/target/factory/factory.go index d398f1fd..0c928284 100644 --- a/pkg/target/factory/factory.go +++ b/pkg/target/factory/factory.go @@ -23,6 +23,7 @@ import ( "github.com/kyverno/policy-reporter/pkg/target/http" "github.com/kyverno/policy-reporter/pkg/target/kinesis" "github.com/kyverno/policy-reporter/pkg/target/loki" + "github.com/kyverno/policy-reporter/pkg/target/pagerduty" "github.com/kyverno/policy-reporter/pkg/target/provider/aws" gs "github.com/kyverno/policy-reporter/pkg/target/provider/gcs" "github.com/kyverno/policy-reporter/pkg/target/s3" @@ -93,6 +94,7 @@ func (f *TargetFactory) CreateClients(config *target.Targets) *target.Collection targets = append(targets, createClients("Kinesis", config.Kinesis, f.CreateKinesisTarget)...) targets = append(targets, createClients("SecurityHub", config.SecurityHub, f.CreateSecurityHubTarget)...) targets = append(targets, createClients("GoogleCloudStorage", config.GCS, f.CreateGCSTarget)...) + targets = append(targets, createClients("PagerDuty", config.PagerDuty, f.CreatePagerDutyTarget)...) return target.NewCollection(targets...) } @@ -741,6 +743,49 @@ func (f *TargetFactory) CreateGCSTarget(config, parent *target.Config[target.GCS } } +func (f *TargetFactory) CreatePagerDutyTarget(config, parent *target.Config[target.PagerDutyOptions]) *target.Target { + if config == nil || config.Config == nil { + return nil + } + + if (parent.SecretRef != "" && f.secretClient != nil) || parent.MountedSecret != "" { + f.mapSecretValues(parent, parent.SecretRef, parent.MountedSecret) + } + + if (config.SecretRef != "" && f.secretClient != nil) || config.MountedSecret != "" { + f.mapSecretValues(config, config.SecretRef, config.MountedSecret) + } + + if config.Config.APIToken == "" || config.Config.ServiceID == "" { + return nil + } + + setFallback(&config.Config.APIToken, parent.Config.APIToken) + setFallback(&config.Config.ServiceID, parent.Config.ServiceID) + + config.MapBaseParent(parent) + + zap.S().Infof("%s configured", config.Name) + + return &target.Target{ + ID: uuid.NewString(), + Type: target.PagerDuty, + Config: config, + ParentConfig: parent, + Client: pagerduty.NewClient(pagerduty.Options{ + ClientOptions: target.ClientOptions{ + Name: config.Name, + SkipExistingOnStartup: config.SkipExisting, + ResultFilter: f.createResultFilter(config.Filter, config.MinimumSeverity, config.Sources), + ReportFilter: createReportFilter(config.Filter), + }, + APIToken: config.Config.APIToken, + ServiceID: config.Config.ServiceID, + CustomFields: config.CustomFields, + }), + } +} + func (f *TargetFactory) createResultFilter(filter target.Filter, minimumSeverity string, sources []string) *report.ResultFilter { sourceFilter := filter.Sources if len(sources) > 0 { diff --git a/pkg/target/pagerduty/pagerduty.go b/pkg/target/pagerduty/pagerduty.go new file mode 100644 index 00000000..50b15f89 --- /dev/null +++ b/pkg/target/pagerduty/pagerduty.go @@ -0,0 +1,178 @@ +package pagerduty + +import ( + "fmt" + "strings" + "sync" + "time" + + "github.com/PagerDuty/go-pagerduty" + "go.uber.org/zap" + + "github.com/kyverno/policy-reporter/pkg/crd/api/policyreport/v1alpha2" + "github.com/kyverno/policy-reporter/pkg/target" + "github.com/kyverno/policy-reporter/pkg/target/formatting" +) + +// Options to configure the PagerDuty target +type Options struct { + target.ClientOptions + APIToken string + ServiceID string + CustomFields map[string]string +} + +type client struct { + target.BaseClient + client *pagerduty.Client + serviceID string + customFields map[string]string + // Track active incidents by policy+resource + incidents sync.Map +} + +// Create a unique key for tracking incidents +func incidentKey(result v1alpha2.PolicyReportResult) string { + key := result.Policy + if result.HasResource() { + res := result.GetResource() + key = fmt.Sprintf("%s/%s/%s/%s", + result.Policy, + res.Kind, + res.Namespace, + res.Name, + ) + } + return key +} + +func (p *client) Send(result v1alpha2.PolicyReportResult) { + key := incidentKey(result) + + if result.Result == v1alpha2.StatusPass { + // Check if we have an active incident to resolve + if incidentID, ok := p.incidents.Load(key); ok { + p.resolveIncident(incidentID.(string)) + p.incidents.Delete(key) + } + return + } + + if result.Result != v1alpha2.StatusFail { + // Only create incidents for failed policies + return + } + + // Check if we already have an incident for this policy/resource + if _, exists := p.incidents.Load(key); exists { + // Incident already exists, no need to create another + return + } + + details := map[string]interface{}{ + "policy": result.Policy, + "rule": result.Rule, + "message": result.Message, + "severity": result.Severity, + } + + if result.HasResource() { + res := result.GetResource() + details["resource"] = formatting.ResourceString(res) + } + + for k, v := range p.customFields { + details[k] = v + } + + for k, v := range result.Properties { + details[k] = v + } + + incident := pagerduty.CreateIncidentOptions{ + Type: "incident", + Title: fmt.Sprintf("Policy Violation: %s", result.Policy), + Service: &pagerduty.APIReference{ID: p.serviceID, Type: "service_reference"}, + Body: &pagerduty.APIDetails{ + Type: "incident_body", + Details: details, + }, + Urgency: mapSeverityToUrgency(result.Severity), + } + + resp, err := p.client.CreateIncident("policy-reporter", &incident) + if err != nil { + zap.L().Error("failed to create PagerDuty incident", + zap.String("policy", result.Policy), + zap.Error(err), + ) + return + } + + // Store the incident ID for later resolution + p.incidents.Store(key, resp.Id) + + zap.L().Info("PagerDuty incident created", + zap.String("policy", result.Policy), + zap.String("severity", string(result.Severity)), + zap.String("incidentId", resp.Id), + ) +} + +func (p *client) resolveIncident(incidentID string) { + incident := pagerduty.ManageIncidentsOptions{ + ID: incidentID, + Incidents: []pagerduty.ManageIncident{ + { + Status: "resolved", + Resolution: "Policy violation has been resolved", + }, + }, + } + + if err := p.client.ManageIncidents("policy-reporter", &incident); err != nil { + zap.L().Error("failed to resolve PagerDuty incident", + zap.String("incidentId", incidentID), + zap.Error(err), + ) + return + } + + zap.L().Info("PagerDuty incident resolved", + zap.String("incidentId", incidentID), + ) +} + +func (p *client) Type() target.ClientType { + return target.SingleSend +} + +func mapSeverityToUrgency(severity v1alpha2.PolicySeverity) string { + switch severity { + case v1alpha2.SeverityCritical, v1alpha2.SeverityHigh: + return "high" + default: + return "low" + } +} + +// SetClient allows replacing the PagerDuty client for testing +func (p *client) SetClient(c interface{}) { + if pdClient, ok := c.(interface { + CreateIncident(string, *pagerduty.CreateIncidentOptions) (*pagerduty.Incident, error) + ManageIncidents(string, *pagerduty.ManageIncidentsOptions) error + }); ok { + p.client = pdClient + } +} + +// NewClient creates a new PagerDuty client +func NewClient(options Options) target.Client { + return &client{ + target.NewBaseClient(options.ClientOptions), + pagerduty.NewClient(options.APIToken), + options.ServiceID, + options.CustomFields, + sync.Map{}, + } +} \ No newline at end of file diff --git a/pkg/target/pagerduty/pagerduty_test.go b/pkg/target/pagerduty/pagerduty_test.go new file mode 100644 index 00000000..7d2193a0 --- /dev/null +++ b/pkg/target/pagerduty/pagerduty_test.go @@ -0,0 +1,167 @@ +package pagerduty_test + +import ( + "testing" + + "github.com/PagerDuty/go-pagerduty" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/kyverno/policy-reporter/pkg/crd/api/policyreport/v1alpha2" + "github.com/kyverno/policy-reporter/pkg/target" + "github.com/kyverno/policy-reporter/pkg/target/pagerduty" +) + +type mockPagerDutyClient struct { + createCalls int + resolveCalls int + lastIncidentID string + lastIncident *pagerduty.CreateIncidentOptions + lastResolve *pagerduty.ManageIncidentsOptions + shouldError bool +} + +func (m *mockPagerDutyClient) CreateIncident(from string, incident *pagerduty.CreateIncidentOptions) (*pagerduty.Incident, error) { + m.createCalls++ + m.lastIncident = incident + return &pagerduty.Incident{Id: "test-incident-id"}, nil +} + +func (m *mockPagerDutyClient) ManageIncidents(from string, incident *pagerduty.ManageIncidentsOptions) error { + m.resolveCalls++ + m.lastResolve = incident + m.lastIncidentID = incident.ID + return nil +} + +func createTestResult(status v1alpha2.PolicyResult) v1alpha2.PolicyReportResult { + return v1alpha2.PolicyReportResult{ + Policy: "test-policy", + Rule: "test-rule", + Message: "test message", + Result: status, + Severity: v1alpha2.SeverityHigh, + Resources: []*corev1.ObjectReference{ + { + Kind: "Pod", + Name: "test-pod", + Namespace: "test-ns", + UID: types.UID("test-uid"), + }, + }, + Properties: map[string]string{ + "test-prop": "test-value", + }, + } +} + +func TestPagerDutyTarget(t *testing.T) { + t.Run("Create incident for failing policy", func(t *testing.T) { + mockClient := &mockPagerDutyClient{} + client := pagerduty.NewClient(pagerduty.Options{ + ClientOptions: target.ClientOptions{ + Name: "test-pagerduty", + }, + APIToken: "test-token", + ServiceID: "test-service", + CustomFields: map[string]string{"cluster": "test-cluster"}, + }) + // Replace internal PD client with mock + client.(*pagerduty.Client).SetClient(mockClient) + + result := createTestResult(v1alpha2.StatusFail) + client.Send(result) + + assert.Equal(t, 1, mockClient.createCalls) + assert.Equal(t, 0, mockClient.resolveCalls) + assert.Equal(t, "Policy Violation: test-policy", mockClient.lastIncident.Title) + assert.Equal(t, "high", mockClient.lastIncident.Urgency) + }) + + t.Run("Do not create duplicate incidents", func(t *testing.T) { + mockClient := &mockPagerDutyClient{} + client := pagerduty.NewClient(pagerduty.Options{ + ClientOptions: target.ClientOptions{ + Name: "test-pagerduty", + }, + APIToken: "test-token", + ServiceID: "test-service", + }) + client.(*pagerduty.Client).SetClient(mockClient) + + result := createTestResult(v1alpha2.StatusFail) + + // Send same failing result twice + client.Send(result) + client.Send(result) + + assert.Equal(t, 1, mockClient.createCalls) + }) + + t.Run("Resolve incident when policy passes", func(t *testing.T) { + mockClient := &mockPagerDutyClient{} + client := pagerduty.NewClient(pagerduty.Options{ + ClientOptions: target.ClientOptions{ + Name: "test-pagerduty", + }, + APIToken: "test-token", + ServiceID: "test-service", + }) + client.(*pagerduty.Client).SetClient(mockClient) + + // First send failing result + failResult := createTestResult(v1alpha2.StatusFail) + client.Send(failResult) + + // Then send passing result for same policy + passResult := createTestResult(v1alpha2.StatusPass) + client.Send(passResult) + + assert.Equal(t, 1, mockClient.createCalls) + assert.Equal(t, 1, mockClient.resolveCalls) + assert.Equal(t, "test-incident-id", mockClient.lastIncidentID) + }) + + t.Run("Ignore non-fail results", func(t *testing.T) { + mockClient := &mockPagerDutyClient{} + client := pagerduty.NewClient(pagerduty.Options{ + ClientOptions: target.ClientOptions{ + Name: "test-pagerduty", + }, + APIToken: "test-token", + ServiceID: "test-service", + }) + client.(*pagerduty.Client).SetClient(mockClient) + + result := createTestResult(v1alpha2.StatusWarn) + client.Send(result) + + assert.Equal(t, 0, mockClient.createCalls) + assert.Equal(t, 0, mockClient.resolveCalls) + }) + + t.Run("Map severity to urgency", func(t *testing.T) { + mockClient := &mockPagerDutyClient{} + client := pagerduty.NewClient(pagerduty.Options{ + ClientOptions: target.ClientOptions{ + Name: "test-pagerduty", + }, + APIToken: "test-token", + ServiceID: "test-service", + }) + client.(*pagerduty.Client).SetClient(mockClient) + + result := createTestResult(v1alpha2.StatusFail) + + // Test high severity + result.Severity = v1alpha2.SeverityHigh + client.Send(result) + assert.Equal(t, "high", mockClient.lastIncident.Urgency) + + // Test low severity + result.Severity = v1alpha2.SeverityLow + client.Send(result) + assert.Equal(t, "low", mockClient.lastIncident.Urgency) + }) +} \ No newline at end of file