From 4a6887a15e95761dc71ce46ec939aada8b2582ad Mon Sep 17 00:00:00 2001 From: Ankur Date: Wed, 19 Feb 2025 15:47:22 +0000 Subject: [PATCH] Add `page.on('request')` (#4290) * Add request to the PageOnEvent type This is where the request will be set for the page.on handler to read. * Add a new page on request event * Add a onRequest method This method is to be called when a new request is about to be sent from chrome to the WuT. It takes the request and send it to the page.on handler where the user test script can read the request data from. * Add call to onRequest from network manager * Update the page on mapping with the request event * Refactor metricInterceptor to eventInterceptor * Fix request.Timing This is a temporary fix. When working with page.on('request') the timing values haven't yet been received, which are part of the response object. This will need to be fixed later when we can wait for a response while working with page.on('request'). * Add a test for page.on('request') * Remove call to Body.Close() from server * Refactor test to work with tb.url * Refactor test to reduce boilerplate code The issue was that HeadersArray was out of order. If they're put in order then the comparison can be made. * Refactor test to work with slices.IndexFunc --- .../k6/browser/browser/page_mapping.go | 4 + .../k6/browser/browser/request_mapping.go | 6 + internal/js/modules/k6/browser/common/http.go | 4 + .../k6/browser/common/network_manager.go | 35 +-- .../k6/browser/common/network_manager_test.go | 8 +- internal/js/modules/k6/browser/common/page.go | 33 +++ .../js/modules/k6/browser/tests/page_test.go | 248 ++++++++++++++++++ 7 files changed, 319 insertions(+), 19 deletions(-) diff --git a/internal/js/modules/k6/browser/browser/page_mapping.go b/internal/js/modules/k6/browser/browser/page_mapping.go index b4b2c11e01a..a67f8ffe164 100644 --- a/internal/js/modules/k6/browser/browser/page_mapping.go +++ b/internal/js/modules/k6/browser/browser/page_mapping.go @@ -585,6 +585,10 @@ func mapPageOn(vu moduleVU, p *common.Page) func(common.PageOnEventName, sobek.C init: prepK6BrowserRegExChecker(rt), wait: true, }, + common.EventPageRequestCalled: { + mapp: mapRequestEvent, + wait: false, + }, } pageOnEvent, ok := pageOnEvents[eventName] if !ok { diff --git a/internal/js/modules/k6/browser/browser/request_mapping.go b/internal/js/modules/k6/browser/browser/request_mapping.go index 3a1c3dd8a0b..6ed55a22913 100644 --- a/internal/js/modules/k6/browser/browser/request_mapping.go +++ b/internal/js/modules/k6/browser/browser/request_mapping.go @@ -7,6 +7,12 @@ import ( "go.k6.io/k6/internal/js/modules/k6/browser/k6ext" ) +func mapRequestEvent(vu moduleVU, event common.PageOnEvent) mapping { + r := event.Request + + return mapRequest(vu, r) +} + // mapRequest to the JS module. func mapRequest(vu moduleVU, r *common.Request) mapping { rt := vu.Runtime() diff --git a/internal/js/modules/k6/browser/common/http.go b/internal/js/modules/k6/browser/common/http.go index 77b1a3e91bf..592daaf5420 100644 --- a/internal/js/modules/k6/browser/common/http.go +++ b/internal/js/modules/k6/browser/common/http.go @@ -361,6 +361,10 @@ type resourceTiming struct { // Timing returns the request timing information. func (r *Request) Timing() *resourceTiming { + if r.response == nil { + return nil + } + timing := r.response.timing return &resourceTiming{ diff --git a/internal/js/modules/k6/browser/common/network_manager.go b/internal/js/modules/k6/browser/common/network_manager.go index 15de532f123..4e713ff14db 100644 --- a/internal/js/modules/k6/browser/common/network_manager.go +++ b/internal/js/modules/k6/browser/common/network_manager.go @@ -43,24 +43,25 @@ func (c Credentials) IsEmpty() bool { return c == (Credentials{}) } -type metricInterceptor interface { +type eventInterceptor interface { urlTagName(urlTag string, method string) (string, bool) + onRequest(request *Request) } // NetworkManager manages all frames in HTML document. type NetworkManager struct { BaseEventEmitter - ctx context.Context - logger *log.Logger - session session - parent *NetworkManager - frameManager *FrameManager - credentials Credentials - resolver k6netext.Resolver - vu k6modules.VU - customMetrics *k6ext.CustomMetrics - mi metricInterceptor + ctx context.Context + logger *log.Logger + session session + parent *NetworkManager + frameManager *FrameManager + credentials Credentials + resolver k6netext.Resolver + vu k6modules.VU + customMetrics *k6ext.CustomMetrics + eventInterceptor eventInterceptor // TODO: manage inflight requests separately (move them between the two maps // as they transition from inflight -> completed) @@ -84,7 +85,7 @@ func NewNetworkManager( s session, fm *FrameManager, parent *NetworkManager, - mi metricInterceptor, + ei eventInterceptor, ) (*NetworkManager, error) { vu := k6ext.GetVU(ctx) state := vu.State() @@ -110,7 +111,7 @@ func NewNetworkManager( attemptedAuth: make(map[fetch.RequestID]bool), extraHTTPHeaders: make(map[string]string), networkProfile: NewNetworkProfile(), - mi: mi, + eventInterceptor: ei, } m.initEvents() if err := m.initDomains(); err != nil { @@ -178,7 +179,7 @@ func (m *NetworkManager) emitRequestMetrics(req *Request) { tags = tags.With("method", req.method) } if state.Options.SystemTags.Has(k6metrics.TagURL) { - tags = handleURLTag(m.mi, req.URL(), req.method, tags) + tags = handleURLTag(m.eventInterceptor, req.URL(), req.method, tags) } tags = tags.With("resource_type", req.ResourceType()) @@ -232,7 +233,7 @@ func (m *NetworkManager) emitResponseMetrics(resp *Response, req *Request) { tags = tags.With("method", req.method) } if state.Options.SystemTags.Has(k6metrics.TagURL) { - tags = handleURLTag(m.mi, url, req.method, tags) + tags = handleURLTag(m.eventInterceptor, url, req.method, tags) } if state.Options.SystemTags.Has(k6metrics.TagIP) { tags = tags.With("ip", ipAddress) @@ -280,7 +281,7 @@ func (m *NetworkManager) emitResponseMetrics(resp *Response, req *Request) { // handleURLTag will check if the url tag needs to be grouped by testing // against user supplied regex. If there's a match a user supplied name will // be used instead of the url for the url tag, otherwise the url will be used. -func handleURLTag(mi metricInterceptor, url string, method string, tags *k6metrics.TagSet) *k6metrics.TagSet { +func handleURLTag(mi eventInterceptor, url string, method string, tags *k6metrics.TagSet) *k6metrics.TagSet { if newTagName, urlMatched := mi.urlTagName(url, method); urlMatched { tags = tags.With("url", newTagName) tags = tags.With("name", newTagName) @@ -511,6 +512,8 @@ func (m *NetworkManager) onRequest(event *network.EventRequestWillBeSent, interc m.reqsMu.Unlock() m.emitRequestMetrics(req) m.frameManager.requestStarted(req) + + m.eventInterceptor.onRequest(req) } func (m *NetworkManager) onRequestPaused(event *fetch.EventRequestPaused) { diff --git a/internal/js/modules/k6/browser/common/network_manager_test.go b/internal/js/modules/k6/browser/common/network_manager_test.go index bf91603519f..7f0eef1bd53 100644 --- a/internal/js/modules/k6/browser/common/network_manager_test.go +++ b/internal/js/modules/k6/browser/common/network_manager_test.go @@ -209,12 +209,14 @@ func TestOnRequestPausedBlockedIPs(t *testing.T) { } } -type MetricInterceptorMock struct{} +type EventInterceptorMock struct{} -func (m *MetricInterceptorMock) urlTagName(_ string, _ string) (string, bool) { +func (m *EventInterceptorMock) urlTagName(_ string, _ string) (string, bool) { return "", false } +func (m *EventInterceptorMock) onRequest(request *Request) {} + func TestNetworkManagerEmitRequestResponseMetricsTimingSkew(t *testing.T) { t.Parallel() @@ -277,7 +279,7 @@ func TestNetworkManagerEmitRequestResponseMetricsTimingSkew(t *testing.T) { var ( vu = k6test.NewVU(t) - nm = &NetworkManager{ctx: vu.Context(), vu: vu, customMetrics: k6m, mi: &MetricInterceptorMock{}} + nm = &NetworkManager{ctx: vu.Context(), vu: vu, customMetrics: k6m, eventInterceptor: &EventInterceptorMock{}} ) vu.ActivateVU() diff --git a/internal/js/modules/k6/browser/common/page.go b/internal/js/modules/k6/browser/common/page.go index 322a901652f..9fed912a952 100644 --- a/internal/js/modules/k6/browser/common/page.go +++ b/internal/js/modules/k6/browser/common/page.go @@ -42,6 +42,9 @@ const ( // EventPageMetricCalled represents the page.on('metric') event. EventPageMetricCalled PageOnEventName = "metric" + + // EventPageRequestCalled represents the page.on('request') event. + EventPageRequestCalled PageOnEventName = "request" ) // MediaType represents the type of media to emulate. @@ -485,6 +488,32 @@ func (p *Page) urlTagName(url string, method string) (string, bool) { return newTagName, urlMatched } +func (p *Page) onRequest(request *Request) { + if !hasPageOnHandler(p, EventPageRequestCalled) { + return + } + + p.eventHandlersMu.RLock() + defer p.eventHandlersMu.RUnlock() + for _, h := range p.eventHandlers[EventPageRequestCalled] { + err := func() error { + // Handlers can register other handlers, so we need to + // unlock the mutex before calling the next handler. + p.eventHandlersMu.RUnlock() + defer p.eventHandlersMu.RLock() + + // Call and wait for the handler to complete. + return h(PageOnEvent{ + Request: request, + }) + }() + if err != nil { + p.logger.Warnf("onRequest", "handler returned an error: %v", err) + return + } + } +} + func (p *Page) onConsoleAPICalled(event *runtime.EventConsoleAPICalled) { if !hasPageOnHandler(p, EventPageConsoleAPICalled) { return @@ -1169,6 +1198,10 @@ type PageOnEvent struct { // Metric is the metric event event. Metric *MetricEvent + + // Request is the read only request that is about to be sent from the + // browser to the WuT. + Request *Request } // On subscribes to a page event for which the given handler will be executed diff --git a/internal/js/modules/k6/browser/tests/page_test.go b/internal/js/modules/k6/browser/tests/page_test.go index 0ad65e34c2c..c2c9f8ce99d 100644 --- a/internal/js/modules/k6/browser/tests/page_test.go +++ b/internal/js/modules/k6/browser/tests/page_test.go @@ -9,7 +9,9 @@ import ( "io" "net/http" "runtime" + "slices" "strconv" + "strings" "sync/atomic" "testing" "time" @@ -2200,3 +2202,249 @@ func TestPageOnMetric(t *testing.T) { }) } } + +func TestPageOnRequest(t *testing.T) { + t.Parallel() + + // Start and setup a webserver to test the page.on('request') handler. + tb := newTestBrowser(t, withHTTPServer()) + defer tb.Browser.Close() + + tb.withHandler("/home", func(w http.ResponseWriter, r *http.Request) { + _, err := fmt.Fprintf(w, ` + + + + + + + +`) + require.NoError(t, err) + }) + tb.withHandler("/api", func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + + var data struct { + Name string `json:"name"` + } + err = json.Unmarshal(body, &data) + require.NoError(t, err) + + _, err = fmt.Fprintf(w, `{"message": "Hello %s!"}`, data.Name) + require.NoError(t, err) + }) + tb.withHandler("/style.css", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/css") + _, err := fmt.Fprintf(w, `body { background-color: #f0f0f0; }`) + require.NoError(t, err) + }) + + // Start and setup a k6 iteration to test the page.on('request') handler. + vu, _, _, cleanUp := startIteration(t) + defer cleanUp() + + // Some of the business logic is in the mapping layer unfortunately. + // To test everything is wried up correctly, we're required to work + // with RunPromise. + // + // The code below is the JavaScript code that is executed in the k6 iteration. + // It will wait for all requests to be captured in returnValue, before returning. + gv, err := vu.RunAsync(t, ` + const context = await browser.newContext({locale: 'en-US', userAgent: 'some-user-agent'}); + const page = await context.newPage(); + + var returnValue = []; + page.on('request', async (request) => { + returnValue.push({ + allHeaders: await request.allHeaders(), + frameUrl: request.frame().url(), + acceptLanguageHeader: await request.headerValue('Accept-Language'), + headers: request.headers(), + headersArray: await request.headersArray(), + isNavigationRequest: request.isNavigationRequest(), + method: request.method(), + postData: request.postData(), + postDataBuffer: request.postDataBuffer() ? String.fromCharCode.apply(null, new Uint8Array(request.postDataBuffer())) : null, + resourceType: request.resourceType(), + // Ignoring response for now since it is not reliable as we don't explicitly wait for the request to finish. + // response: await request.response(), + size: request.size(), + // Ignoring timing for now since it is not reliable as we don't explicitly wait for the request to finish. + // timing: request.timing(), + url: request.url() + }); + }); + + await page.goto('%s', {waitUntil: 'networkidle'}); + + await page.close(); + + return JSON.stringify(returnValue, null, 2); + `, tb.url("/home")) + assert.NoError(t, err) + + got := k6test.ToPromise(t, gv) + + // Convert the result to a string and then to a slice of requests. + var requests []request + err = json.Unmarshal([]byte(got.Result().String()), &requests) + require.NoError(t, err) + + expected := []request{ + { + AllHeaders: map[string]string{ + "accept-language": "en-US", + "upgrade-insecure-requests": "1", + "user-agent": "some-user-agent", + }, + FrameURL: "about:blank", + AcceptLanguageHeader: "en-US", + Headers: map[string]string{ + "Accept-Language": "en-US", + "Upgrade-Insecure-Requests": "1", + "User-Agent": "some-user-agent", + }, + HeadersArray: []map[string]string{ + {"name": "Upgrade-Insecure-Requests", "value": "1"}, + {"name": "User-Agent", "value": "some-user-agent"}, + {"name": "Accept-Language", "value": "en-US"}, + }, + IsNavigationRequest: true, + Method: "GET", + PostData: "", + PostDataBuffer: "", + ResourceType: "Document", + Size: map[string]int{ + "body": 0, + "headers": 103, + }, + URL: tb.url("/home"), + }, + { + AllHeaders: map[string]string{ + "accept-language": "en-US", + "referer": tb.url("/home"), + "user-agent": "some-user-agent", + }, + FrameURL: tb.url("/home"), + AcceptLanguageHeader: "en-US", + Headers: map[string]string{ + "Accept-Language": "en-US", + "Referer": tb.url("/home"), + "User-Agent": "some-user-agent", + }, + HeadersArray: []map[string]string{ + {"name": "User-Agent", "value": "some-user-agent"}, + {"name": "Accept-Language", "value": "en-US"}, + {"name": "Referer", "value": tb.url("/home")}, + }, + IsNavigationRequest: false, + Method: "GET", + PostData: "", + PostDataBuffer: "", + ResourceType: "Stylesheet", + Size: map[string]int{ + "body": 0, + "headers": 116, + }, + URL: tb.url("/style.css"), + }, + { + AllHeaders: map[string]string{ + "accept-language": "en-US", + "content-type": "application/json", + "referer": tb.url("/home"), + "user-agent": "some-user-agent", + }, + FrameURL: tb.url("/home"), + AcceptLanguageHeader: "en-US", + Headers: map[string]string{ + "Accept-Language": "en-US", + "Content-Type": "application/json", + "Referer": tb.url("/home"), + "User-Agent": "some-user-agent", + }, + HeadersArray: []map[string]string{ + {"name": "Referer", "value": tb.url("/home")}, + {"name": "User-Agent", "value": "some-user-agent"}, + {"name": "Accept-Language", "value": "en-US"}, + {"name": "Content-Type", "value": "application/json"}, + }, + IsNavigationRequest: false, + Method: "POST", + PostData: `{"name":"tester"}`, + PostDataBuffer: `{"name":"tester"}`, + ResourceType: "Fetch", + Size: map[string]int{ + "body": 17, + "headers": 143, + }, + URL: tb.url("/api"), + }, + { + AllHeaders: map[string]string{ + "accept-language": "en-US", + "referer": tb.url("/home"), + "user-agent": "some-user-agent", + }, + FrameURL: tb.url("/home"), + AcceptLanguageHeader: "en-US", + Headers: map[string]string{ + "Accept-Language": "en-US", + "Referer": tb.url("/home"), + "User-Agent": "some-user-agent", + }, + HeadersArray: []map[string]string{ + {"name": "Accept-Language", "value": "en-US"}, + {"name": "Referer", "value": tb.url("/home")}, + {"name": "User-Agent", "value": "some-user-agent"}, + }, + IsNavigationRequest: false, + Method: "GET", + PostData: "", + PostDataBuffer: "", + ResourceType: "Other", + Size: map[string]int{ + "body": 0, + "headers": 118, + }, + URL: tb.url("/favicon.ico"), + }, + } + + // Compare each request one by one for better test failure visibility + for _, req := range requests { + i := slices.IndexFunc(expected, func(r request) bool { return req.URL == r.URL }) + assert.NotEqual(t, -1, i, "failed to find expected request with URL %s", req.URL) + + sortByName := func(m1, m2 map[string]string) int { + return strings.Compare(m1["name"], m2["name"]) + } + slices.SortFunc(req.HeadersArray, sortByName) + slices.SortFunc(expected[i].HeadersArray, sortByName) + assert.Equal(t, expected[i], req) + } +} + +type request struct { + AllHeaders map[string]string `json:"allHeaders"` + FrameURL string `json:"frameUrl"` + AcceptLanguageHeader string `json:"acceptLanguageHeader"` + Headers map[string]string `json:"headers"` + HeadersArray []map[string]string `json:"headersArray"` + IsNavigationRequest bool `json:"isNavigationRequest"` + Method string `json:"method"` + PostData string `json:"postData"` + PostDataBuffer string `json:"postDataBuffer"` + ResourceType string `json:"resourceType"` + Size map[string]int `json:"size"` + URL string `json:"url"` +}