-
Notifications
You must be signed in to change notification settings - Fork 60
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* feat: close watcher gracefully in error responder * feat: upstream info filter * feat: add trace log * feat: tracing metric * fix: typo Reqeust -> Request
- Loading branch information
Showing
25 changed files
with
778 additions
and
112 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| package filters | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
| "io" | ||
| "net/http" | ||
|
|
||
| utilnet "k8s.io/apimachinery/pkg/util/net" | ||
| "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" | ||
| apirequest "k8s.io/apiserver/pkg/endpoints/request" | ||
| "k8s.io/klog" | ||
|
|
||
| "github.com/kubewharf/kubegateway/pkg/clusters/features" | ||
| "github.com/kubewharf/kubegateway/pkg/gateway/endpoints/request" | ||
| "github.com/kubewharf/kubegateway/pkg/gateway/metrics" | ||
| "github.com/kubewharf/kubegateway/pkg/util/tracing" | ||
| ) | ||
|
|
||
| // WithTraceLog is a filter that record trace log. | ||
| func WithTraceLog(handler http.Handler, enableTracing bool, longRunningRequestCheck apirequest.LongRunningRequestCheck) http.Handler { | ||
| if !enableTracing { | ||
| return handler | ||
| } | ||
|
|
||
| klog.V(2).Infof("Enable proxy tracing, ShortRequestLogThreshold=%v, ListRequestLogThreshold=%v", request.ShortRequestLogThreshold, request.ListRequestLogThreshold) | ||
|
|
||
| return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { | ||
| ctx := req.Context() | ||
|
|
||
| requestInfo, ok := apirequest.RequestInfoFrom(ctx) | ||
| if !ok { | ||
| // if this happens, the handler chain isn't setup correctly because there is no request info | ||
| responsewriters.InternalError(w, req, errors.New("no RequestInfo found in the context")) | ||
| return | ||
| } | ||
|
|
||
| // Skip tracing long-running requests. | ||
| if longRunningRequestCheck(req, requestInfo) { | ||
| handler.ServeHTTP(w, req) | ||
| return | ||
| } | ||
|
|
||
| extraInfo, ok := request.ExtraRequestInfoFrom(ctx) | ||
| if !ok { | ||
| responsewriters.InternalError(w, req, fmt.Errorf("failed to get extra request info from context")) | ||
| return | ||
| } | ||
|
|
||
| cluster := extraInfo.UpstreamCluster | ||
| if cluster == nil || !cluster.FeatureEnabled(features.Tracing) { | ||
| handler.ServeHTTP(w, req) | ||
| return | ||
| } | ||
|
|
||
| tr := tracing.New(fmt.Sprintf("Trace for %v %v", req.Method, req.RequestURI)) | ||
| ctx = tracing.WithRequestTraceInfo(ctx, tr) | ||
|
|
||
| req = req.WithContext(ctx) | ||
|
|
||
| defer func() { | ||
| tr.End() | ||
|
|
||
| metrics.RecordProxyTraceLatency(tr.StageLatency(), extraInfo.Hostname, requestInfo) | ||
|
|
||
| threshold := request.LogThreshold(requestInfo.Verb) | ||
| if req.Header.Get("x-debug-trace-log") == "1" || tr.IfLong(threshold) { | ||
| tr.WithAttributes(traceFields(req, requestInfo)...) | ||
| tr.Log() | ||
| } | ||
| }() | ||
|
|
||
| rd := &traceReader{ | ||
| ReadCloser: req.Body, | ||
| trace: tr, | ||
| } | ||
| req.Body = rd | ||
|
|
||
| handler.ServeHTTP(w, req) | ||
| }) | ||
| } | ||
|
|
||
| func traceFields(req *http.Request, requestInfo *apirequest.RequestInfo) []tracing.KeyValue { | ||
| sourceIPs := utilnet.SourceIPs(req) | ||
| return []tracing.KeyValue{ | ||
| tracing.StringKeyValue("verb", requestInfo.Verb), | ||
| tracing.StringKeyValue("resource", requestInfo.Resource), | ||
| tracing.StringKeyValue("name", requestInfo.Name), | ||
| tracing.StringKeyValue("host", req.Host), | ||
| tracing.StringKeyValue("user-agent", req.Header.Get("User-Agent")), | ||
| tracing.StringKeyValue("srcIP", fmt.Sprintf("%v", sourceIPs)), | ||
| } | ||
| } | ||
|
|
||
| var _ io.ReadCloser = &traceReader{} | ||
|
|
||
| type traceReader struct { | ||
| io.ReadCloser | ||
| trace *tracing.RequestTraceInfo | ||
| } | ||
|
|
||
| // Write implements io.ReadCloser | ||
| func (r *traceReader) Read(p []byte) (int, error) { | ||
| n, err := r.ReadCloser.Read(p) | ||
| if err == io.EOF { | ||
| r.trace.Step(tracing.StepReadRequest) | ||
| } | ||
| return n, err | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| // Copyright 2022 ByteDance and its affiliates. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package filters | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "github.com/kubewharf/kubegateway/pkg/clusters" | ||
| "github.com/kubewharf/kubegateway/pkg/clusters/features" | ||
| "github.com/kubewharf/kubegateway/pkg/gateway/endpoints/response" | ||
| "k8s.io/apimachinery/pkg/api/errors" | ||
| "k8s.io/apimachinery/pkg/runtime" | ||
| "net" | ||
| "net/http" | ||
|
|
||
| "github.com/kubewharf/kubegateway/pkg/gateway/endpoints/request" | ||
| ) | ||
|
|
||
| // WithUpstreamInfo attaches upstream cluster info to ExtraRequestInfo | ||
| func WithUpstreamInfo(handler http.Handler, clusterManager clusters.Manager, s runtime.NegotiatedSerializer) http.Handler { | ||
| return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { | ||
| ctx := req.Context() | ||
|
|
||
| info, ok := request.ExtraRequestInfoFrom(ctx) | ||
| if !ok { | ||
| handler.ServeHTTP(w, req) | ||
| return | ||
| } | ||
|
|
||
| if ip := net.ParseIP(info.Hostname); ip == nil { | ||
| info.IsProxyRequest = true | ||
| cluster, ok := clusterManager.Get(info.Hostname) | ||
| if !ok { | ||
| response.TerminateWithError(s, | ||
| errors.NewServiceUnavailable(fmt.Sprintf("the request cluster(%s) is not being proxied", info.Hostname)), | ||
| response.TerminationReasonClusterNotBeingProxied, w, req) | ||
| return | ||
| } | ||
| info.UpstreamCluster = cluster | ||
|
|
||
| if cluster.FeatureEnabled(features.CloseConnectionWhenIdle) { | ||
| // Send a GOAWAY and tear down the TCP connection when idle. | ||
| w.Header().Set("Connection", "close") | ||
| } | ||
|
|
||
| if cluster.FeatureEnabled(features.DenyAllRequests) { | ||
| response.TerminateWithError(s, errors.NewServiceUnavailable(fmt.Sprintf("request for %v denied by featureGate(DenyAllRequests)", info.Hostname)), | ||
| response.TerminationReasonCircuitBreaker, w, req) | ||
| return | ||
| } | ||
| } | ||
|
|
||
| req = req.WithContext(request.WithExtraRequestInfo(ctx, info)) | ||
| handler.ServeHTTP(w, req) | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| package request | ||
|
|
||
| import ( | ||
| "k8s.io/klog" | ||
| "os" | ||
| "strconv" | ||
| "strings" | ||
| "time" | ||
| ) | ||
|
|
||
| var ( | ||
| ShortRequestLogThreshold = time.Second * 5 | ||
| ListRequestLogThreshold = time.Second * 30 | ||
| ) | ||
|
|
||
| func init() { | ||
| if val := os.Getenv("SHORT_REQUEST_LOG_THRESHOLD_SECONDS"); len(val) > 0 { | ||
| i, err := strconv.Atoi(val) | ||
| if err != nil { | ||
| klog.Warningf("Illegal REQUEST_TRACE_LOG_THRESHOLD_SECONDS: %v", val) | ||
| } else { | ||
| ShortRequestLogThreshold = time.Second * time.Duration(i) | ||
| } | ||
| } | ||
|
|
||
| if val := os.Getenv("LIST_REQUEST_LOG_THRESHOLD_SECONDS"); len(val) > 0 { | ||
| i, err := strconv.Atoi(val) | ||
| if err != nil { | ||
| klog.Warningf("Illegal LIST_REQUEST_TRACE_LOG_THRESHOLD_SECONDS: %v", val) | ||
| } else { | ||
| ListRequestLogThreshold = time.Second * time.Duration(i) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func LogThreshold(verb string) time.Duration { | ||
| threshold := ShortRequestLogThreshold | ||
| if strings.Contains(strings.ToLower(verb), "list") { | ||
| threshold = ListRequestLogThreshold | ||
| } | ||
| return threshold | ||
| } |
Oops, something went wrong.