fix more tests

go.mod

@@ -421,4 +421,4 @@ replace (
 // Exclude this version because the version has an invalid checksum.
 exclude github.com/docker/distribution v2.8.0+incompatible
 
-replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20231220154111-efc1fba83b4b
+replace github.com/elastic/elastic-agent-libs => github.com/belimawr/elastic-agent-libs v0.2.9-0.20231221105324-aedb70a4f832

go.sum

@@ -368,8 +368,8 @@ github.com/awslabs/goformation/v4 v4.1.0 h1:JRxIW0IjhYpYDrIZOTJGMu2azXKI+OK5dP56
 github.com/awslabs/goformation/v4 v4.1.0/go.mod h1:MBDN7u1lMNDoehbFuO4uPvgwPeolTMA2TzX1yO6KlxI=
 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 h1:lxW5Q6K2IisyF5tlr6Ts0W4POGWQZco05MJjFmoeIHs=
 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5/go.mod h1:0Qr1uMHFmHsIYMcG4T7BJ9yrJtWadhOmpABCX69dwuc=
-github.com/belimawr/elastic-agent-libs v0.2.9-0.20231220154111-efc1fba83b4b h1:GEwwH9rTwJzcHAcdCFkfta0AHbADcTNpxZhL51ASLpo=
-github.com/belimawr/elastic-agent-libs v0.2.9-0.20231220154111-efc1fba83b4b/go.mod h1:EbRwBMsWoU4IHGKJlTrxbxC03hkihS9W4h+UgraLdDM=
+github.com/belimawr/elastic-agent-libs v0.2.9-0.20231221105324-aedb70a4f832 h1:hCPNCDrtpZg8GekH7RptPcJ9C/Dgr2ebku2lETqFFw0=
+github.com/belimawr/elastic-agent-libs v0.2.9-0.20231221105324-aedb70a4f832/go.mod h1:EbRwBMsWoU4IHGKJlTrxbxC03hkihS9W4h+UgraLdDM=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/immutable v0.2.1/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI=
 github.com/benbjohnson/tmpl v1.0.0/go.mod h1:igT620JFIi44B6awvU9IsDhR77IXWtFigTLil/RPdps=

libbeat/cmd/instance/beat_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/outputs"
 	"github.com/elastic/beats/v7/libbeat/publisher/queue/memqueue"
 	"github.com/elastic/elastic-agent-libs/config"
+	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/go-ucfg/yaml"
 
 	"github.com/gofrs/uuid"
@@ -247,7 +248,7 @@ elasticsearch:
 
 		update := &reload.ConfigWithMeta{Config: c}
 		m := &outputReloaderMock{}
-		reloader := b.makeOutputReloader(m)
+		reloader := b.makeOutputReloader(m, logp.Config{})
 
 		require.False(t, b.Config.Output.IsSet(), "the output should not be set yet")
 		require.True(t, b.isConnectionToOlderVersionAllowed(), "allow_older_versions flag should be true from 8.11")
@@ -266,7 +267,8 @@ type outputReloaderMock struct {
 
 func (r *outputReloaderMock) Reload(
 	cfg *reload.ConfigWithMeta,
-	factory func(o outputs.Observer, cfg config.Namespace) (outputs.Group, error),
+	eventsLoggerCfg logp.Config,
+	factory func(o outputs.Observer, cfg config.Namespace, eventsLoggerCfg logp.Config) (outputs.Group, error),
 ) error {
 	r.cfg = cfg
 	return nil

libbeat/outputs/elasticsearch/client_integration_test.go

@@ -422,7 +422,15 @@ func connectTestEs(t *testing.T, cfg interface{}, stats outputs.Observer) (outpu
 	info := beat.Info{Beat: "libbeat"}
 	// disable ILM if using specified index name
 	im, _ := idxmgmt.DefaultSupport(nil, info, conf.MustNewConfigFrom(map[string]interface{}{"setup.ilm.enabled": "false"}))
-	output, err := makeES(im, info, stats, config, logp.Config{})
+
+	// Creates the events logger configuration for testing
+	// It used the default one but logs to stderr instead of a file
+	eventsLoggerCfg := logp.DefaultConfig(logp.DefaultEnvironment)
+	eventsLoggerCfg.Level = logp.DebugLevel
+	eventsLoggerCfg.ToStderr = true
+	eventsLoggerCfg.ToFiles = false
+
+	output, err := makeES(im, info, stats, config, eventsLoggerCfg)
 	if err != nil {
 		t.Fatal(err)
 	}

libbeat/outputs/elasticsearch/client_test.go

@@ -90,6 +90,8 @@ func (bm *batchMock) RetryEvents(events []publisher.Event) {
 func TestPublish(t *testing.T) {
 	makePublishTestClient := func(t *testing.T, url string) *Client {
 		client, err := NewClient(
+			logp.L(),
+			logp.L(),
 			ClientSettings{
 				Observer:           outputs.NewNilObserver(),
 				ConnectionSettings: eslegclient.ConnectionSettings{URL: url},
@@ -248,6 +250,8 @@ func TestPublish(t *testing.T) {
 
 func TestCollectPublishFailsNone(t *testing.T) {
 	client, err := NewClient(
+		logp.L(),
+		logp.L(),
 		ClientSettings{
 			Observer:           outputs.NewNilObserver(),
 			NonIndexableAction: "drop",
@@ -272,6 +276,8 @@ func TestCollectPublishFailsNone(t *testing.T) {
 
 func TestCollectPublishFailMiddle(t *testing.T) {
 	client, err := NewClient(
+		logp.L(),
+		logp.L(),
 		ClientSettings{
 			Observer:           outputs.NewNilObserver(),
 			NonIndexableAction: "drop",
@@ -302,6 +308,8 @@ func TestCollectPublishFailMiddle(t *testing.T) {
 
 func TestCollectPublishFailDeadLetterQueue(t *testing.T) {
 	client, err := NewClient(
+		logp.L(),
+		logp.L(),
 		ClientSettings{
 			Observer:           outputs.NewNilObserver(),
 			NonIndexableAction: "dead_letter_index",
@@ -361,6 +369,8 @@ func TestCollectPublishFailDeadLetterQueue(t *testing.T) {
 
 func TestCollectPublishFailDrop(t *testing.T) {
 	client, err := NewClient(
+		logp.L(),
+		logp.L(),
 		ClientSettings{
 			Observer:           outputs.NewNilObserver(),
 			NonIndexableAction: "drop",
@@ -405,6 +415,8 @@ func TestCollectPublishFailDrop(t *testing.T) {
 
 func TestCollectPublishFailAll(t *testing.T) {
 	client, err := NewClient(
+		logp.L(),
+		logp.L(),
 		ClientSettings{
 			Observer:           outputs.NewNilObserver(),
 			NonIndexableAction: "drop",
@@ -434,6 +446,8 @@ func TestCollectPipelinePublishFail(t *testing.T) {
 	logp.TestingSetup(logp.WithSelectors("elasticsearch"))
 
 	client, err := NewClient(
+		logp.L(),
+		logp.L(),
 		ClientSettings{
 			Observer:           outputs.NewNilObserver(),
 			NonIndexableAction: "drop",
@@ -481,6 +495,8 @@ func TestCollectPipelinePublishFail(t *testing.T) {
 
 func BenchmarkCollectPublishFailsNone(b *testing.B) {
 	client, err := NewClient(
+		logp.L(),
+		logp.L(),
 		ClientSettings{
 			Observer:           outputs.NewNilObserver(),
 			NonIndexableAction: "drop",
@@ -510,6 +526,8 @@ func BenchmarkCollectPublishFailsNone(b *testing.B) {
 
 func BenchmarkCollectPublishFailMiddle(b *testing.B) {
 	client, err := NewClient(
+		logp.L(),
+		logp.L(),
 		ClientSettings{
 			Observer:           outputs.NewNilObserver(),
 			NonIndexableAction: "drop",
@@ -540,6 +558,8 @@ func BenchmarkCollectPublishFailMiddle(b *testing.B) {
 
 func BenchmarkCollectPublishFailAll(b *testing.B) {
 	client, err := NewClient(
+		logp.L(),
+		logp.L(),
 		ClientSettings{
 			Observer:           outputs.NewNilObserver(),
 			NonIndexableAction: "drop",
@@ -589,17 +609,20 @@ func TestClientWithHeaders(t *testing.T) {
 	}))
 	defer ts.Close()
 
-	client, err := NewClient(ClientSettings{
-		Observer: outputs.NewNilObserver(),
-		ConnectionSettings: eslegclient.ConnectionSettings{
-			URL: ts.URL,
-			Headers: map[string]string{
-				"host":   "myhost.local",
-				"X-Test": "testing value",
+	client, err := NewClient(
+		logp.L(),
+		logp.L(),
+		ClientSettings{
+			Observer: outputs.NewNilObserver(),
+			ConnectionSettings: eslegclient.ConnectionSettings{
+				URL: ts.URL,
+				Headers: map[string]string{
+					"host":   "myhost.local",
+					"X-Test": "testing value",
+				},
 			},
-		},
-		Index: outil.MakeSelector(outil.ConstSelectorExpr("test", outil.SelectorLowerCase)),
-	}, nil)
+			Index: outil.MakeSelector(outil.ConstSelectorExpr("test", outil.SelectorLowerCase)),
+		}, nil)
 	assert.NoError(t, err)
 
 	// simple ping
@@ -667,6 +690,8 @@ func TestBulkEncodeEvents(t *testing.T) {
 			}
 
 			client, err := NewClient(
+				logp.L(),
+				logp.L(),
 				ClientSettings{
 					Observer: outputs.NewNilObserver(),
 					Index:    index,
@@ -743,6 +768,8 @@ func TestBulkEncodeEventsWithOpType(t *testing.T) {
 	}
 
 	client, _ := NewClient(
+		logp.L(),
+		logp.L(),
 		ClientSettings{
 			Observer: outputs.NewNilObserver(),
 			Index:    index,
@@ -786,13 +813,16 @@ func TestClientWithAPIKey(t *testing.T) {
 	}))
 	defer ts.Close()
 
-	client, err := NewClient(ClientSettings{
-		Observer: outputs.NewNilObserver(),
-		ConnectionSettings: eslegclient.ConnectionSettings{
-			URL:    ts.URL,
-			APIKey: "hyokHG4BfWk5viKZ172X:o45JUkyuS--yiSAuuxl8Uw",
-		},
-	}, nil)
+	client, err := NewClient(
+		logp.L(),
+		logp.L(),
+		ClientSettings{
+			Observer: outputs.NewNilObserver(),
+			ConnectionSettings: eslegclient.ConnectionSettings{
+				URL:    ts.URL,
+				APIKey: "hyokHG4BfWk5viKZ172X:o45JUkyuS--yiSAuuxl8Uw",
+			},
+		}, nil)
 	assert.NoError(t, err)
 
 	// This connection will fail since the server doesn't return a valid
@@ -806,6 +836,8 @@ func TestClientWithAPIKey(t *testing.T) {
 func TestPublishEventsWithBulkFiltering(t *testing.T) {
 	makePublishTestClient := func(t *testing.T, url string, configParams map[string]string) *Client {
 		client, err := NewClient(
+			logp.L(),
+			logp.L(),
 			ClientSettings{
 				Observer: outputs.NewNilObserver(),
 				ConnectionSettings: eslegclient.ConnectionSettings{

libbeat/outputs/elasticsearch/elasticsearch_test.go

@@ -122,11 +122,12 @@ func TestPipelineSelection(t *testing.T) {
 			selector, err := buildPipelineSelector(config.MustNewConfigFrom(test.cfg))
 
 			client, err := NewClient(
+				logp.L(),
+				logp.L(),
 				ClientSettings{
 					Pipeline: &selector,
 				},
 				nil,
-				logp.Config{},
 			)
 			assert.NoError(t, err)
 

libbeat/publisher/pipeline/stress/out.go

@@ -26,6 +26,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/outputs"
 	"github.com/elastic/beats/v7/libbeat/publisher"
 	conf "github.com/elastic/elastic-agent-libs/config"
+	"github.com/elastic/elastic-agent-libs/logp"
 )
 
 type testOutput struct {
@@ -55,7 +56,7 @@ func init() {
 	outputs.RegisterType("test", makeTestOutput)
 }
 
-func makeTestOutput(_ outputs.IndexManager, beat beat.Info, observer outputs.Observer, cfg *conf.C) (outputs.Group, error) {
+func makeTestOutput(_ outputs.IndexManager, beat beat.Info, observer outputs.Observer, cfg *conf.C, eventsLoggerCfg logp.Config) (outputs.Group, error) {
 	config := defaultTestOutputConfig
 	if err := cfg.Unpack(&config); err != nil {
 		return outputs.Fail(err)