diff --git a/Makefile b/Makefile index 8f9d3a2f57..abe40a9f9a 100644 --- a/Makefile +++ b/Makefile @@ -20,8 +20,9 @@ PROJECT := aks-engine VERSION ?= $(shell git rev-parse HEAD) VERSION_SHORT ?= $(shell git rev-parse --short HEAD) GITTAG := $(shell git describe --exact-match --tags $(shell git log -n1 --pretty='%h') 2> /dev/null) -GOBIN ?= $(shell $(GO) env GOPATH)/bin -TOOLSBIN := $(CURDIR)/hack/tools/bin +GOBIN ?= $(shell $(GO) env GOPATH)/bin +TOOLSBIN := $(CURDIR)/hack/tools/bin +AIKey ?= c92d8284-b550-4b06-b7ba-e80fd7178faa ifeq ($(GITTAG),) GITTAG := $(VERSION_SHORT) endif @@ -35,7 +36,7 @@ DEV_CMD_RUN := docker run $(DEV_ENV_OPTS) ifdef DEBUG LDFLAGS := -X main.version=$(VERSION) else -LDFLAGS := -s -X main.version=$(VERSION) +LDFLAGS := -s -X main.version=$(VERSION) -X github.com/Azure/$(PROJECT)/pkg/telemetry.AKSEngineAppInsightsKey=$(AIKey) endif BINARY_DEST_DIR ?= bin @@ -160,7 +161,7 @@ ginkgoBuild: generate make -C ./test/e2e ginkgo-build test: generate ginkgoBuild - ginkgo -mod=vendor -skipPackage test/e2e -failFast -r -v -tags=fast . + ginkgo -mod=vendor -skipPackage test/e2e -failFast -r -v -tags=fast -ldflags '$(LDFLAGS)' . .PHONY: test-style test-style: validate-go validate-shell validate-copyright-headers @@ -199,7 +200,7 @@ ci: bootstrap test-style build test lint .PHONY: coverage coverage: - @scripts/ginkgo.coverage.sh --codecov + LDFLAGS="$(LDFLAGS)" ./scripts/ginkgo.coverage.sh --codecov include versioning.mk include test.mk diff --git a/docs/topics/telemetry.md b/docs/topics/telemetry.md index 5c17309296..888db71d0b 100644 --- a/docs/topics/telemetry.md +++ b/docs/topics/telemetry.md @@ -1,9 +1,10 @@ -# [WIP] +# Telemetry -## Motivation +## Custom Script Extensions -The aks-engine team is working towards collecting telemetry from custom script extensions (CSEs) used to configure Kubernetes nodes. -This data will be used to collect aggregate, non-identifiable data to help us answer gather the following information: +The AKS Engine team has instrumented custom script extensions (CSEs) so collection of telemetry may +be enabled. This instrumentation is used to collect aggregate, non-identifiable data to help us +answer gather the following information: - Timing metrics for various operations performed during CSE execution - Metrics on failures encountered during CSE execution @@ -12,15 +13,23 @@ This data will be used to collect aggregate, non-identifiable data to help us an - Node pool counts and VM Sku data - Which versions of VHDs are still in use -This data will be used to monitor the health of cluster deployments (including AKS clusters) that are deployed with aks-engine as well as to help us prioritize future investments/feature work in the tool. +This data will be used to monitor the health of cluster deployments (including AKS clusters) that +are deployed with AKS Engine as well as to help us prioritize future investments/feature work in +the tool. ## Configuration Collection of all telemetry is currently **disabled** by default. -Telemetry can be enabled by setting the `EnableTelemetry` feature flag to `true` in the apimodel.json. +Telemetry can be enabled by setting the `enableTelemetry` feature flag to `true` in the +apimodel.json. When `enableTelemetry` is set to `true`, telemetry will be sent to the AKS Engine +Application Insights cluster. -Telemetry can be routed to different application insights instance be specifying `telemetryProfile.applicationInsightsKey` to the instrumentation key of your application insights instance. +Telemetry can be routed to an additional Application Insights instance by specifying +`telemetryProfile.applicationInsightsKey` with the value of the instrumentation key of your +Application Insights instance. + +That means you can see and use the same data that is being sent to the AKS Engine team. ``` javascript { @@ -35,3 +44,21 @@ Telemetry can be routed to different application insights instance be specifying } } ``` + +### Windows Specific Differences (wip) + +Currently, Windows CSE will only log telemetry to the AKS Engine Application Insights instance, even +if `applicationInsightsKey` is set. In a future change set, this feature will be enabled. + +### Collection Settings and Results + +AKS Engine has the concept of a system defined AKS Engine Application Insights key as well as a user +defined key. The table below describes where telemetry is logged based on the configuration +settings. + +| enableTelemetry | applicationInsightsKey set | result | +| ------------- | ------------- | ------------- | +| true | true | both user and AKS Engine telemetry is tracked | +| false | true | no telemetry is tracked | +| true | false | only AKS Engine telemetry is tracked | +| false | false | no telemetry is tracked | \ No newline at end of file diff --git a/go.mod b/go.mod index 391a285bfd..3b15fae628 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,7 @@ require ( gopkg.in/go-playground/validator.v9 v9.25.0 gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.41.0 + gopkg.in/yaml.v2 v2.2.2 // indirect k8s.io/api v0.0.0-20190222213804-5cb15d344471 k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 k8s.io/client-go v10.0.0+incompatible diff --git a/go.sum b/go.sum index 06a1642e00..cd15cfa81c 100644 --- a/go.sum +++ b/go.sum @@ -177,6 +177,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= k8s.io/api v0.0.0-20190222213804-5cb15d344471 h1:MzQGt8qWQCR+39kbYRd0uQqsvSidpYqJLFeWiJ9l4OE= k8s.io/api v0.0.0-20190222213804-5cb15d344471/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 h1:UYfHH+KEF88OTg+GojQUwFTNxbxwmoktLwutUzR0GPg= diff --git a/parts/k8s/cloud-init/artifacts/cse_main.sh b/parts/k8s/cloud-init/artifacts/cse_main.sh index 216f875e07..9c1f63602c 100755 --- a/parts/k8s/cloud-init/artifacts/cse_main.sh +++ b/parts/k8s/cloud-init/artifacts/cse_main.sh @@ -20,9 +20,9 @@ source {{GetCSEHelpersScriptFilepath}} wait_for_file 3600 1 {{GetCSEInstallScriptFilepath}} || exit $ERR_FILE_WATCH_TIMEOUT source {{GetCSEInstallScriptFilepath}} -ensureAPMZ "v0.4.0" +ensureAPMZ "v0.5.1" {{- if HasTelemetryEnabled }} -eval "$(apmz bash -n "cse" -t "{{GetLinuxDefaultTelemetryTags}}" --api-key "{{GetApplicationInsightsTelemetryKey}}")" +eval "$(apmz bash -n "cse" -t "{{GetLinuxDefaultTelemetryTags}}" --api-keys "{{GetApplicationInsightsTelemetryKeys}}")" {{else}} eval "$(apmz bash -d)" {{end}} diff --git a/pkg/api/const.go b/pkg/api/const.go index 508ebc585d..85fd44d97c 100644 --- a/pkg/api/const.go +++ b/pkg/api/const.go @@ -513,7 +513,3 @@ const TLSStrongCipherSuitesAPIServer = "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS // TLSStrongCipherSuitesKubelet is a kube-bench-recommended allowed cipher suites for kubelet const TLSStrongCipherSuitesKubelet = "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256" - -// Default instrmentation key used for routing Application Insights data -// NOTE! this is in a test sub and needs to be switched to a production sub before collecting user data! -const DefaultApplicationInsightsKey = "c92d8284-b550-4b06-b7ba-e80fd7178faa" diff --git a/pkg/api/defaults.go b/pkg/api/defaults.go index dcc2840bcc..72c9c7c1c6 100644 --- a/pkg/api/defaults.go +++ b/pkg/api/defaults.go @@ -16,11 +16,12 @@ import ( "github.com/Azure/go-autorest/autorest/to" - "github.com/Azure/aks-engine/pkg/api/common" - "github.com/Azure/aks-engine/pkg/helpers" "github.com/blang/semver" "github.com/pkg/errors" log "github.com/sirupsen/logrus" + + "github.com/Azure/aks-engine/pkg/api/common" + "github.com/Azure/aks-engine/pkg/helpers" ) // DistroValues is a list of currently supported distros @@ -774,7 +775,7 @@ func (p *Properties) setTelemetryProfileDefaults() { } if len(p.TelemetryProfile.ApplicationInsightsKey) == 0 { - p.TelemetryProfile.ApplicationInsightsKey = DefaultApplicationInsightsKey + p.TelemetryProfile.ApplicationInsightsKey = "" } } diff --git a/pkg/api/defaults_test.go b/pkg/api/defaults_test.go index 630d71e071..96a870c680 100644 --- a/pkg/api/defaults_test.go +++ b/pkg/api/defaults_test.go @@ -13,13 +13,14 @@ import ( "strings" "testing" - "github.com/Azure/aks-engine/pkg/api/common" - "github.com/Azure/aks-engine/pkg/helpers" "github.com/Azure/go-autorest/autorest/azure" "github.com/Azure/go-autorest/autorest/to" "github.com/google/go-cmp/cmp" "github.com/jarcoal/httpmock" "github.com/pkg/errors" + + "github.com/Azure/aks-engine/pkg/api/common" + "github.com/Azure/aks-engine/pkg/helpers" ) func TestCertsAlreadyPresent(t *testing.T) { @@ -3893,14 +3894,14 @@ func TestSetTelemetryProfileDefaults(t *testing.T) { name: "default", telemetryProfile: nil, expected: &TelemetryProfile{ - ApplicationInsightsKey: DefaultApplicationInsightsKey, + ApplicationInsightsKey: "", }, }, { name: "key not set", telemetryProfile: &TelemetryProfile{}, expected: &TelemetryProfile{ - ApplicationInsightsKey: DefaultApplicationInsightsKey, + ApplicationInsightsKey: "", }, }, { diff --git a/pkg/engine/armvariables.go b/pkg/engine/armvariables.go index b94995d1bc..61fb1051a0 100644 --- a/pkg/engine/armvariables.go +++ b/pkg/engine/armvariables.go @@ -12,6 +12,8 @@ import ( "github.com/Azure/aks-engine/pkg/api" "github.com/Azure/aks-engine/pkg/api/common" "github.com/Azure/aks-engine/pkg/helpers" + "github.com/Azure/aks-engine/pkg/telemetry" + "github.com/Azure/go-autorest/autorest/to" ) @@ -593,7 +595,7 @@ func getTelemetryVars(cs *api.ContainerService) map[string]interface{} { applicationInsightsKey := "" if cs.Properties.TelemetryProfile != nil { - applicationInsightsKey = cs.Properties.TelemetryProfile.ApplicationInsightsKey + applicationInsightsKey = telemetry.AKSEngineAppInsightsKey } telemetryVars := map[string]interface{}{ diff --git a/pkg/engine/template_generator.go b/pkg/engine/template_generator.go index a3ee5da4d2..afab64f5da 100644 --- a/pkg/engine/template_generator.go +++ b/pkg/engine/template_generator.go @@ -20,6 +20,8 @@ import ( "github.com/Azure/aks-engine/pkg/api/common" "github.com/Azure/aks-engine/pkg/helpers" "github.com/Azure/aks-engine/pkg/i18n" + "github.com/Azure/aks-engine/pkg/telemetry" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -785,11 +787,25 @@ func getContainerServiceFuncMap(cs *api.ContainerService) template.FuncMap { "HasTelemetryEnabled": func() bool { return cs.Properties.FeatureFlags != nil && cs.Properties.FeatureFlags.EnableTelemetry }, - "GetApplicationInsightsTelemetryKey": func() string { - if cs.Properties.TelemetryProfile == nil { - return "" + "GetApplicationInsightsTelemetryKeys": func() string { + userSuppliedAIKey := "" + if cs.Properties.TelemetryProfile != nil { + userSuppliedAIKey = cs.Properties.TelemetryProfile.ApplicationInsightsKey + } + + possibleKeys := []string{ + telemetry.AKSEngineAppInsightsKey, + userSuppliedAIKey, } - return cs.Properties.TelemetryProfile.ApplicationInsightsKey + + var keys []string + for _, key := range possibleKeys { + if key != "" { + keys = append(keys, key) + } + } + + return strings.Join(keys, ",") }, "GetLinuxDefaultTelemetryTags": func() string { tags := map[string]string{ diff --git a/pkg/engine/template_generator_test.go b/pkg/engine/template_generator_test.go index e0145b41bf..1331b7e346 100644 --- a/pkg/engine/template_generator_test.go +++ b/pkg/engine/template_generator_test.go @@ -5,11 +5,14 @@ package engine import ( "encoding/json" + "fmt" "reflect" "testing" "text/template" "github.com/Azure/aks-engine/pkg/api" + "github.com/Azure/aks-engine/pkg/telemetry" + "github.com/Azure/go-autorest/autorest/azure" "github.com/pkg/errors" ) @@ -823,28 +826,6 @@ func TestTemplateGenerator_FunctionMap(t *testing.T) { }, ExpectedResult: "foo/k8s/core/pause:1.2.0", }, - { - Name: "HasTelemetryEnabled", - FuncName: "HasTelemetryEnabled", - MutateFunc: func(cs api.ContainerService) api.ContainerService { - cs.Properties.FeatureFlags = &api.FeatureFlags{ - EnableTelemetry: true, - } - return cs - }, - ExpectedResult: true, - }, - { - Name: "GetApplicationInsightsTelemetryKey", - FuncName: "GetApplicationInsightsTelemetryKey", - MutateFunc: func(cs api.ContainerService) api.ContainerService { - cs.Properties.TelemetryProfile = &api.TelemetryProfile{ - ApplicationInsightsKey: "my_telemetry_key", - } - return cs - }, - ExpectedResult: "my_telemetry_key", - }, { Name: "HasCiliumNetworkPolicy - cilium", FuncName: "HasCiliumNetworkPolicy", @@ -909,13 +890,35 @@ func TestTemplateGenerator_FunctionMap(t *testing.T) { ExpectedResult: false, }, { - Name: "GetEmptyApplicationInsightsTelemetryKey", - FuncName: "GetApplicationInsightsTelemetryKey", + Name: "HasTelemetryEnabled", + FuncName: "HasTelemetryEnabled", + MutateFunc: func(cs api.ContainerService) api.ContainerService { + cs.Properties.FeatureFlags = &api.FeatureFlags{ + EnableTelemetry: true, + } + return cs + }, + ExpectedResult: true, + }, + { + Name: "GetEmptyApplicationInsightsTelemetryKeys", + FuncName: "GetApplicationInsightsTelemetryKeys", MutateFunc: func(cs api.ContainerService) api.ContainerService { cs.Properties.TelemetryProfile = nil return cs }, - ExpectedResult: "", + ExpectedResult: telemetry.AKSEngineAppInsightsKey, + }, + { + Name: "GetApplicationInsightsTelemetryKeysWithUserSuppliedKey", + FuncName: "GetApplicationInsightsTelemetryKeys", + MutateFunc: func(cs api.ContainerService) api.ContainerService { + cs.Properties.TelemetryProfile = &api.TelemetryProfile{ + ApplicationInsightsKey: "my_telemetry_key", + } + return cs + }, + ExpectedResult: fmt.Sprintf("%s,%s", telemetry.AKSEngineAppInsightsKey, "my_telemetry_key"), }, { Name: "GetLinuxDefaultTelemetryTags", diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go index de8af7145d..d6be76f0c1 100644 --- a/pkg/engine/templates_generated.go +++ b/pkg/engine/templates_generated.go @@ -37654,9 +37654,9 @@ source {{GetCSEHelpersScriptFilepath}} wait_for_file 3600 1 {{GetCSEInstallScriptFilepath}} || exit $ERR_FILE_WATCH_TIMEOUT source {{GetCSEInstallScriptFilepath}} -ensureAPMZ "v0.4.0" +ensureAPMZ "v0.5.1" {{- if HasTelemetryEnabled }} -eval "$(apmz bash -n "cse" -t "{{GetLinuxDefaultTelemetryTags}}" --api-key "{{GetApplicationInsightsTelemetryKey}}")" +eval "$(apmz bash -n "cse" -t "{{GetLinuxDefaultTelemetryTags}}" --api-keys "{{GetApplicationInsightsTelemetryKeys}}")" {{else}} eval "$(apmz bash -d)" {{end}} diff --git a/pkg/telemetry/telemetry.go b/pkg/telemetry/telemetry.go new file mode 100644 index 0000000000..c98f91ed67 --- /dev/null +++ b/pkg/telemetry/telemetry.go @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +package telemetry + +var ( + // AKSEngineAppInsightsKey is the Application Insights Key used by AKS Engine when telemetry is enabled + AKSEngineAppInsightsKey string +) diff --git a/scripts/ginkgo.coverage.sh b/scripts/ginkgo.coverage.sh index 99f6b641ba..e6b5d0ca60 100755 --- a/scripts/ginkgo.coverage.sh +++ b/scripts/ginkgo.coverage.sh @@ -25,7 +25,7 @@ hash goveralls 2>/dev/null || go get github.com/mattn/goveralls hash godir 2>/dev/null || go get github.com/Masterminds/godir generate_cover_data() { - ginkgo -mod=vendor -skipPackage test/e2e -failFast -cover -r -v -tags=fast . + ginkgo -mod=vendor -skipPackage test/e2e -failFast -cover -r -v -tags=fast -ldflags "${LDFLAGS}" . echo "" > ${coveragetxt} find . -type f -name "*.coverprofile" | while read -r file; do cat "$file" >> ${coveragetxt} && mv "$file" "${coverdir}"; done echo "mode: $covermode" >"$profile" diff --git a/test/e2e/Makefile b/test/e2e/Makefile index 5b37ca9514..1178957240 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -1,6 +1,8 @@ GO ?= go -BINDIR := bin -GINKGO := ginkgo +BINDIR := bin +GINKGO := ginkgo +AIKey := c92d8284-b550-4b06-b7ba-e80fd7178faa +LDFLAGS := -X github.com/Azure/$(PROJECT)/pkg/telemetry.AKSEngineAppInsightsKey=$(AIKey) all: build @@ -9,13 +11,13 @@ build: clean runner-build ginkgo-build .PHONY: runner-build runner-build: - $(GO) build -tags=test -o ./$(BINDIR)/e2e-runner . + $(GO) build -tags=test -o ./$(BINDIR)/e2e-runner -ldflags '$(LDFLAGS)' . .PHONY: ginkgo-build ginkgo-build: - $(GINKGO) build -tags=test ./kubernetes + $(GINKGO) build -tags=test -ldflags '$(LDFLAGS)' ./kubernetes .PHONY: clean clean: rm -f ./kubernetes/kubernetes.test - rm -f ./$(BINDIR)/e2e-runner + rm -f ./$(BINDIR)/e2e-runner \ No newline at end of file diff --git a/test/e2e/go.sum b/test/e2e/go.sum index 29b078f70f..c8c4662f2b 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -1,7 +1,4 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/Azure/azure-sdk-for-go v34.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v36.0.0+incompatible h1:XIaBmA4pgKqQ7jInQPaNJQ4pOHrdJjw9gYXhbyiChaU= -github.com/Azure/azure-sdk-for-go v36.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v36.2.0+incompatible h1:09cv2WoH0g6jl6m2iT+R9qcIPZKhXEL0sbmLhxP895s= github.com/Azure/azure-sdk-for-go v36.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= @@ -160,6 +157,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= k8s.io/api v0.0.0-20190222213804-5cb15d344471/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 h1:UYfHH+KEF88OTg+GojQUwFTNxbxwmoktLwutUzR0GPg= k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=