feat: multi AI telemetry keys (Azure#2606)

Makefile

@@ -20,8 +20,9 @@ PROJECT         := aks-engine
 VERSION         ?= $(shell git rev-parse HEAD)
 VERSION_SHORT   ?= $(shell git rev-parse --short HEAD)
 GITTAG          := $(shell git describe --exact-match --tags $(shell git log -n1 --pretty='%h') 2> /dev/null)
-GOBIN			?= $(shell $(GO) env GOPATH)/bin
-TOOLSBIN		:= $(CURDIR)/hack/tools/bin
+GOBIN           ?= $(shell $(GO) env GOPATH)/bin
+TOOLSBIN        := $(CURDIR)/hack/tools/bin
+AIKey           ?= c92d8284-b550-4b06-b7ba-e80fd7178faa
 ifeq ($(GITTAG),)
 GITTAG := $(VERSION_SHORT)
 endif
@@ -35,7 +36,7 @@ DEV_CMD_RUN := docker run $(DEV_ENV_OPTS)
 ifdef DEBUG
 LDFLAGS := -X main.version=$(VERSION)
 else
-LDFLAGS := -s -X main.version=$(VERSION)
+LDFLAGS := -s -X main.version=$(VERSION) -X github.com/Azure/$(PROJECT)/pkg/telemetry.AKSEngineAppInsightsKey=$(AIKey)
 endif
 BINARY_DEST_DIR ?= bin
 
@@ -160,7 +161,7 @@ ginkgoBuild: generate
 	make -C ./test/e2e ginkgo-build
 
 test: generate ginkgoBuild
-	ginkgo -mod=vendor -skipPackage test/e2e -failFast -r -v -tags=fast .
+	ginkgo -mod=vendor -skipPackage test/e2e -failFast -r -v -tags=fast -ldflags '$(LDFLAGS)' .
 
 .PHONY: test-style
 test-style: validate-go validate-shell validate-copyright-headers
@@ -199,7 +200,7 @@ ci: bootstrap test-style build test lint
 
 .PHONY: coverage
 coverage:
-	@scripts/ginkgo.coverage.sh --codecov
+	LDFLAGS="$(LDFLAGS)" ./scripts/ginkgo.coverage.sh --codecov
 
 include versioning.mk
 include test.mk

docs/topics/telemetry.md

@@ -1,9 +1,10 @@
-# [WIP]
+# Telemetry
 
-## Motivation
+## Custom Script Extensions
 
-The aks-engine team is working towards collecting telemetry from custom script extensions (CSEs) used to configure Kubernetes nodes.
-This data will be used to collect aggregate, non-identifiable data to help us answer gather the following information:
+The AKS Engine team has instrumented custom script extensions (CSEs) so collection of telemetry may 
+be enabled. This instrumentation is used to collect aggregate, non-identifiable data to help us 
+answer gather the following information:
 
 - Timing metrics for various operations performed during CSE execution
 - Metrics on failures encountered during CSE execution
@@ -12,15 +13,23 @@ This data will be used to collect aggregate, non-identifiable data to help us an
   - Node pool counts and VM Sku data
   - Which versions of VHDs are still in use
 
-This data will be used to monitor the health of cluster deployments (including AKS clusters) that are deployed with aks-engine as well as to help us prioritize future investments/feature work in the tool.
+This data will be used to monitor the health of cluster deployments (including AKS clusters) that 
+are deployed with AKS Engine as well as to help us prioritize future investments/feature work in 
+the tool.
 
 ## Configuration
 
 Collection of all telemetry is currently **disabled** by default.
 
-Telemetry can be enabled by setting the `EnableTelemetry` feature flag to `true` in the apimodel.json.
+Telemetry can be enabled by setting the `enableTelemetry` feature flag to `true` in the 
+apimodel.json. When `enableTelemetry` is set to `true`, telemetry will be sent to the AKS Engine 
+Application Insights cluster.
 
-Telemetry can be routed to different application insights instance be specifying `telemetryProfile.applicationInsightsKey` to the instrumentation key of your application insights instance.
+Telemetry can be routed to an additional Application Insights instance by specifying 
+`telemetryProfile.applicationInsightsKey` with the value of the instrumentation key of your 
+Application Insights instance. 
+
+That means you can see and use the same data that is being sent to the AKS Engine team.
 
 ``` javascript
 {
@@ -35,3 +44,21 @@ Telemetry can be routed to different application insights instance be specifying
   }
 }
 ```
+
+### Windows Specific Differences (wip)
+
+Currently, Windows CSE will only log telemetry to the AKS Engine Application Insights instance, even
+if `applicationInsightsKey` is set. In a future change set, this feature will be enabled.
+
+### Collection Settings and Results
+
+AKS Engine has the concept of a system defined AKS Engine Application Insights key as well as a user
+defined key. The table below describes where telemetry is logged based on the configuration
+settings.
+
+| enableTelemetry  | applicationInsightsKey set | result |
+| ------------- | ------------- | ------------- |
+| true | true | both user and AKS Engine telemetry is tracked |
+| false  | true  | no telemetry is tracked |
+| true | false | only AKS Engine telemetry is tracked |
+| false | false | no telemetry is tracked |

go.mod

@@ -57,6 +57,7 @@ require (
 	gopkg.in/go-playground/validator.v9 v9.25.0
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.41.0
+	gopkg.in/yaml.v2 v2.2.2 // indirect
 	k8s.io/api v0.0.0-20190222213804-5cb15d344471
 	k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628
 	k8s.io/client-go v10.0.0+incompatible

go.sum

@@ -177,6 +177,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 k8s.io/api v0.0.0-20190222213804-5cb15d344471 h1:MzQGt8qWQCR+39kbYRd0uQqsvSidpYqJLFeWiJ9l4OE=
 k8s.io/api v0.0.0-20190222213804-5cb15d344471/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
 k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 h1:UYfHH+KEF88OTg+GojQUwFTNxbxwmoktLwutUzR0GPg=

parts/k8s/cloud-init/artifacts/cse_main.sh

@@ -20,9 +20,9 @@ source {{GetCSEHelpersScriptFilepath}}
 wait_for_file 3600 1 {{GetCSEInstallScriptFilepath}} || exit $ERR_FILE_WATCH_TIMEOUT
 source {{GetCSEInstallScriptFilepath}}
 
-ensureAPMZ "v0.4.0"
+ensureAPMZ "v0.5.1"
 {{- if HasTelemetryEnabled }}
-eval "$(apmz bash -n "cse" -t "{{GetLinuxDefaultTelemetryTags}}" --api-key "{{GetApplicationInsightsTelemetryKey}}")"
+eval "$(apmz bash -n "cse" -t "{{GetLinuxDefaultTelemetryTags}}" --api-keys "{{GetApplicationInsightsTelemetryKeys}}")"
 {{else}}
 eval "$(apmz bash -d)"
 {{end}}

pkg/api/const.go

@@ -513,7 +513,3 @@ const TLSStrongCipherSuitesAPIServer = "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS
 
 // TLSStrongCipherSuitesKubelet is a kube-bench-recommended allowed cipher suites for kubelet
 const TLSStrongCipherSuitesKubelet = "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256"
-
-// Default instrmentation key used for routing Application Insights data
-// NOTE! this is in a test sub and needs to be switched to a production sub before collecting user data!
-const DefaultApplicationInsightsKey = "c92d8284-b550-4b06-b7ba-e80fd7178faa"

pkg/api/defaults.go

@@ -16,11 +16,12 @@ import (
 
 	"github.com/Azure/go-autorest/autorest/to"
 
-	"github.com/Azure/aks-engine/pkg/api/common"
-	"github.com/Azure/aks-engine/pkg/helpers"
 	"github.com/blang/semver"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/Azure/aks-engine/pkg/api/common"
+	"github.com/Azure/aks-engine/pkg/helpers"
 )
 
 // DistroValues is a list of currently supported distros
@@ -774,7 +775,7 @@ func (p *Properties) setTelemetryProfileDefaults() {
 	}
 
 	if len(p.TelemetryProfile.ApplicationInsightsKey) == 0 {
-		p.TelemetryProfile.ApplicationInsightsKey = DefaultApplicationInsightsKey
+		p.TelemetryProfile.ApplicationInsightsKey = ""
 	}
 }
 

pkg/api/defaults_test.go

@@ -13,13 +13,14 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/Azure/aks-engine/pkg/api/common"
-	"github.com/Azure/aks-engine/pkg/helpers"
 	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Azure/go-autorest/autorest/to"
 	"github.com/google/go-cmp/cmp"
 	"github.com/jarcoal/httpmock"
 	"github.com/pkg/errors"
+
+	"github.com/Azure/aks-engine/pkg/api/common"
+	"github.com/Azure/aks-engine/pkg/helpers"
 )
 
 func TestCertsAlreadyPresent(t *testing.T) {
@@ -3893,14 +3894,14 @@ func TestSetTelemetryProfileDefaults(t *testing.T) {
 			name:             "default",
 			telemetryProfile: nil,
 			expected: &TelemetryProfile{
-				ApplicationInsightsKey: DefaultApplicationInsightsKey,
+				ApplicationInsightsKey: "",
 			},
 		},
 		{
 			name:             "key not set",
 			telemetryProfile: &TelemetryProfile{},
 			expected: &TelemetryProfile{
-				ApplicationInsightsKey: DefaultApplicationInsightsKey,
+				ApplicationInsightsKey: "",
 			},
 		},
 		{

pkg/engine/armvariables.go

@@ -12,6 +12,8 @@ import (
 	"github.com/Azure/aks-engine/pkg/api"
 	"github.com/Azure/aks-engine/pkg/api/common"
 	"github.com/Azure/aks-engine/pkg/helpers"
+	"github.com/Azure/aks-engine/pkg/telemetry"
+
 	"github.com/Azure/go-autorest/autorest/to"
 )
 
@@ -593,7 +595,7 @@ func getTelemetryVars(cs *api.ContainerService) map[string]interface{} {
 
 	applicationInsightsKey := ""
 	if cs.Properties.TelemetryProfile != nil {
-		applicationInsightsKey = cs.Properties.TelemetryProfile.ApplicationInsightsKey
+		applicationInsightsKey = telemetry.AKSEngineAppInsightsKey
 	}
 
 	telemetryVars := map[string]interface{}{

pkg/engine/template_generator.go

@@ -20,6 +20,8 @@ import (
 	"github.com/Azure/aks-engine/pkg/api/common"
 	"github.com/Azure/aks-engine/pkg/helpers"
 	"github.com/Azure/aks-engine/pkg/i18n"
+	"github.com/Azure/aks-engine/pkg/telemetry"
+
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 )
@@ -785,11 +787,25 @@ func getContainerServiceFuncMap(cs *api.ContainerService) template.FuncMap {
 		"HasTelemetryEnabled": func() bool {
 			return cs.Properties.FeatureFlags != nil && cs.Properties.FeatureFlags.EnableTelemetry
 		},
-		"GetApplicationInsightsTelemetryKey": func() string {
-			if cs.Properties.TelemetryProfile == nil {
-				return ""
+		"GetApplicationInsightsTelemetryKeys": func() string {
+			userSuppliedAIKey := ""
+			if cs.Properties.TelemetryProfile != nil {
+				userSuppliedAIKey = cs.Properties.TelemetryProfile.ApplicationInsightsKey
+			}
+
+			possibleKeys := []string{
+				telemetry.AKSEngineAppInsightsKey,
+				userSuppliedAIKey,
 			}
-			return cs.Properties.TelemetryProfile.ApplicationInsightsKey
+
+			var keys []string
+			for _, key := range possibleKeys {
+				if key != "" {
+					keys = append(keys, key)
+				}
+			}
+
+			return strings.Join(keys, ",")
 		},
 		"GetLinuxDefaultTelemetryTags": func() string {
 			tags := map[string]string{

pkg/engine/template_generator_test.go

@@ -5,11 +5,14 @@ package engine
 
 import (
 	"encoding/json"
+	"fmt"
 	"reflect"
 	"testing"
 	"text/template"
 
 	"github.com/Azure/aks-engine/pkg/api"
+	"github.com/Azure/aks-engine/pkg/telemetry"
+
 	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/pkg/errors"
 )
@@ -823,28 +826,6 @@ func TestTemplateGenerator_FunctionMap(t *testing.T) {
 			},
 			ExpectedResult: "foo/k8s/core/pause:1.2.0",
 		},
-		{
-			Name:     "HasTelemetryEnabled",
-			FuncName: "HasTelemetryEnabled",
-			MutateFunc: func(cs api.ContainerService) api.ContainerService {
-				cs.Properties.FeatureFlags = &api.FeatureFlags{
-					EnableTelemetry: true,
-				}
-				return cs
-			},
-			ExpectedResult: true,
-		},
-		{
-			Name:     "GetApplicationInsightsTelemetryKey",
-			FuncName: "GetApplicationInsightsTelemetryKey",
-			MutateFunc: func(cs api.ContainerService) api.ContainerService {
-				cs.Properties.TelemetryProfile = &api.TelemetryProfile{
-					ApplicationInsightsKey: "my_telemetry_key",
-				}
-				return cs
-			},
-			ExpectedResult: "my_telemetry_key",
-		},
 		{
 			Name:     "HasCiliumNetworkPolicy - cilium",
 			FuncName: "HasCiliumNetworkPolicy",
@@ -909,13 +890,35 @@ func TestTemplateGenerator_FunctionMap(t *testing.T) {
 			ExpectedResult: false,
 		},
 		{
-			Name:     "GetEmptyApplicationInsightsTelemetryKey",
-			FuncName: "GetApplicationInsightsTelemetryKey",
+			Name:     "HasTelemetryEnabled",
+			FuncName: "HasTelemetryEnabled",
+			MutateFunc: func(cs api.ContainerService) api.ContainerService {
+				cs.Properties.FeatureFlags = &api.FeatureFlags{
+					EnableTelemetry: true,
+				}
+				return cs
+			},
+			ExpectedResult: true,
+		},
+		{
+			Name:     "GetEmptyApplicationInsightsTelemetryKeys",
+			FuncName: "GetApplicationInsightsTelemetryKeys",
 			MutateFunc: func(cs api.ContainerService) api.ContainerService {
 				cs.Properties.TelemetryProfile = nil
 				return cs
 			},
-			ExpectedResult: "",
+			ExpectedResult: telemetry.AKSEngineAppInsightsKey,
+		},
+		{
+			Name:     "GetApplicationInsightsTelemetryKeysWithUserSuppliedKey",
+			FuncName: "GetApplicationInsightsTelemetryKeys",
+			MutateFunc: func(cs api.ContainerService) api.ContainerService {
+				cs.Properties.TelemetryProfile = &api.TelemetryProfile{
+					ApplicationInsightsKey: "my_telemetry_key",
+				}
+				return cs
+			},
+			ExpectedResult: fmt.Sprintf("%s,%s", telemetry.AKSEngineAppInsightsKey, "my_telemetry_key"),
 		},
 		{
 			Name:     "GetLinuxDefaultTelemetryTags",

pkg/telemetry/telemetry.go

@@ -0,0 +1,9 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+package telemetry
+
+var (
+	// AKSEngineAppInsightsKey is the Application Insights Key used by AKS Engine when telemetry is enabled
+	AKSEngineAppInsightsKey string
+)

scripts/ginkgo.coverage.sh

@@ -25,7 +25,7 @@ hash goveralls 2>/dev/null || go get github.com/mattn/goveralls
 hash godir 2>/dev/null || go get github.com/Masterminds/godir
 
 generate_cover_data() {
-  ginkgo -mod=vendor -skipPackage test/e2e -failFast -cover -r -v -tags=fast .
+  ginkgo -mod=vendor -skipPackage test/e2e -failFast -cover -r -v -tags=fast -ldflags "${LDFLAGS}" .
   echo "" > ${coveragetxt}
   find . -type f -name "*.coverprofile" | while read -r file;  do cat "$file" >> ${coveragetxt} && mv "$file" "${coverdir}"; done
   echo "mode: $covermode" >"$profile"

test/e2e/Makefile

@@ -1,6 +1,8 @@
 GO              ?= go
-BINDIR			:= bin
-GINKGO			:= ginkgo
+BINDIR          := bin
+GINKGO          := ginkgo
+AIKey           := c92d8284-b550-4b06-b7ba-e80fd7178faa
+LDFLAGS         := -X github.com/Azure/$(PROJECT)/pkg/telemetry.AKSEngineAppInsightsKey=$(AIKey)
 
 all: build
 
@@ -9,13 +11,13 @@ build: clean runner-build ginkgo-build
 
 .PHONY: runner-build
 runner-build:
-	$(GO) build -tags=test -o ./$(BINDIR)/e2e-runner .
+	$(GO) build -tags=test -o ./$(BINDIR)/e2e-runner -ldflags '$(LDFLAGS)' .
 
 .PHONY: ginkgo-build
 ginkgo-build:
-	$(GINKGO) build -tags=test ./kubernetes
+	$(GINKGO) build -tags=test -ldflags '$(LDFLAGS)' ./kubernetes
 
 .PHONY: clean
 clean:
 	rm -f ./kubernetes/kubernetes.test
-	rm -f ./$(BINDIR)/e2e-runner
+	rm -f ./$(BINDIR)/e2e-runner