Remove dependency on gcr.io/kubebuilder/kube-rbac-proxy (#680)

* Follow https://book.kubebuilder.io/migration/manually_migration_guide_gov3_to_gov4 upgrade instructions

* Fix credentials

* Enable certmanager kustomize component

* Update makefile

* Add missing flags to main.go file

* Fix rolebinding

* RBAC

* Fix main imports

* Remove unwanted CA config

* Fix ca cert-manager injection

* fix package comment

* Fix controllers path

* Run make generate

* fix manager build command

* Ensure right labels

* Drop extra line

Dockerfile

@@ -34,13 +34,15 @@ ARG ARCH
 ARG ldflags
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} GO111MODULE=on \
   go build -a -trimpath -ldflags "${ldflags} -extldflags '-static'" \
-  -o manager main.go
+  -o manager cmd/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
-FROM gcr.io/distroless/static:latest
+FROM gcr.io/distroless/static:nonroot
 WORKDIR /
 COPY --from=builder /workspace/manager .
-USER nobody
+# Use uid of nonroot user (65532) because kubernetes expects numeric user when applying pod security policies
+# See: https://github.com/kubernetes-sigs/cluster-api/pull/4064/files
+USER 65532
 
 ENTRYPOINT ["/manager"]

Makefile

@@ -169,7 +169,7 @@ $(ARTIFACTS):
 
 .PHONY: test
 test: generate ## Run tests
-	source ./scripts/fetch_ext_bins.sh; fetch_tools; setup_envs; go test -v -covermode=atomic -coverprofile=coverage.tmp.out ./api/... ./controllers/... ./cloud/...
+	source ./scripts/fetch_ext_bins.sh; fetch_tools; setup_envs; go test -v -covermode=atomic -coverprofile=coverage.tmp.out ./api/... ./internal/controller/... ./cloud/...
 	@cat coverage.tmp.out | grep -v "generated" > coverage.out
 	@rm coverage.tmp.out
 
@@ -202,7 +202,7 @@ binaries: manager ## Builds and installs all binaries
 
 .PHONY: manager
 manager: ## Build manager binary.
-	go build -trimpath -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/manager .
+	go build -trimpath -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/manager ./cmd
 
 ## --------------------------------------
 ## Tooling Binaries
@@ -269,7 +269,7 @@ generate: ## Generate code
 .PHONY: generate-go
 generate-go: $(CONTROLLER_GEN) $(CONVERSION_GEN) $(MOCKGEN) ## Runs Go related generate targets
 	$(CONTROLLER_GEN) \
-		paths=./ \
+		paths=./cmd \
 		paths=./... \
 		paths=./$(EXP_DIR)/api/... \
 		object:headerFile=./hack/boilerplate/boilerplate.generatego.txt
@@ -278,16 +278,16 @@ generate-go: $(CONTROLLER_GEN) $(CONVERSION_GEN) $(MOCKGEN) ## Runs Go related g
 .PHONY: generate-manifests
 generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc.
 	$(CONTROLLER_GEN) \
-		paths=./ \
+		paths=./cmd \
 		paths=./api/... \
 		crd:crdVersions=v1 \
 		rbac:roleName=manager-role \
 		output:crd:dir=$(CRD_ROOT) \
 		output:webhook:dir=$(WEBHOOK_ROOT) \
 		webhook
 	$(CONTROLLER_GEN) \
-		paths=./ \
-		paths=./controllers/... \
+		paths=./cmd \
+		paths=./internal/controller/... \
 		output:rbac:dir=$(RBAC_ROOT) \
 		rbac:roleName=manager-role
 

PROJECT

@@ -1,5 +1,7 @@
 version: "3"
 domain: cluster.x-k8s.io
+layout:
+- go.kubebuilder.io/v4
 repo: sigs.k8s.io/cluster-api-provider-digitalocean
 resources:
 # v1alpha4 types

main.go → cmd/main.go

@@ -18,35 +18,37 @@ limitations under the License.
 package main
 
 import (
+	"crypto/tls"
 	"flag"
 	"net/http"
 	"os"
+	"path/filepath"
 	"time"
 
 	_ "net/http/pprof"
 
 	// +kubebuilder:scaffold:imports
 
 	"github.com/spf13/pflag"
-	"k8s.io/klog/v2"
-
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	cgrecord "k8s.io/client-go/tools/record"
+	"k8s.io/klog/v2"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/util/flags"
+	"sigs.k8s.io/cluster-api/util/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/certwatcher"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
-	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
-	"sigs.k8s.io/cluster-api/util/flags"
-	"sigs.k8s.io/cluster-api/util/record"
-
 	infrav1alpha4 "sigs.k8s.io/cluster-api-provider-digitalocean/api/v1alpha4"
 	infrav1beta1 "sigs.k8s.io/cluster-api-provider-digitalocean/api/v1beta1"
 	infrawebhooks "sigs.k8s.io/cluster-api-provider-digitalocean/api/webhooks"
-	"sigs.k8s.io/cluster-api-provider-digitalocean/controllers"
+	capdocontroller "sigs.k8s.io/cluster-api-provider-digitalocean/internal/controller"
 	dnsutil "sigs.k8s.io/cluster-api-provider-digitalocean/util/dns"
 	dnsresolver "sigs.k8s.io/cluster-api-provider-digitalocean/util/dns/resolver"
 	"sigs.k8s.io/cluster-api-provider-digitalocean/util/reconciler"
@@ -79,11 +81,20 @@ var (
 	healthAddr                  string
 	watchNamespace              string
 	profilerAddress             string
-	webhookCertDir              string
 	webhookPort                 int
 	enableLeaderElection        bool
 	restConfigQPS               float32
 	restConfigBurst             int
+	metricsAddr                 string
+	probeAddr                   string
+	webhookCertPath             string
+	metricsCertPath             string
+	metricsCertName             string
+	metricsCertKey              string
+	secureMetrics               bool
+	webhookCertName             string
+	webhookCertKey              string
+	tlsOpts                     []func(*tls.Config)
 )
 
 func initFlags(fs *pflag.FlagSet) {
@@ -100,7 +111,15 @@ func initFlags(fs *pflag.FlagSet) {
 	fs.DurationVar(&reconcileTimeout, "reconcile-timeout", reconciler.DefaultLoopTimeout, "The maximum duration a reconcile loop can run (e.g. 90m)")
 	fs.Float32Var(&restConfigQPS, "kube-api-qps", 20, "Maximum queries per second from the controller client to the Kubernetes API server.")
 	fs.IntVar(&restConfigBurst, "kube-api-burst", 30, "Maximum number of queries that should be allowed in one burst from the controller client to the Kubernetes API server.")
-	fs.StringVar(&webhookCertDir, "webhook-cert-dir", "/tmp/k8s-webhook-server/serving-certs", "Webhook Server Certificate Directory, is the directory that contains the server key and certificate")
+	fs.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
+	fs.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
+	fs.StringVar(&webhookCertPath, "webhook-cert-path", "", "The directory that contains the webhook certificate.")
+	fs.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.")
+	fs.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.")
+	fs.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
+	fs.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+	fs.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.")
+	fs.StringVar(&webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.")
 	flags.AddDiagnosticsOptions(fs, &diagnosticsOptions)
 }
 
@@ -155,6 +174,77 @@ func main() {
 	restConfig.QPS = restConfigQPS
 	restConfig.Burst = restConfigBurst
 
+	// Create watchers for metrics and webhooks certificates
+	var metricsCertWatcher, webhookCertWatcher *certwatcher.CertWatcher
+
+	// Initial webhook TLS options
+	webhookTLSOpts := tlsOpts
+
+	if len(webhookCertPath) > 0 {
+		setupLog.Info("Initializing webhook certificate watcher using provided certificates",
+			"webhook-cert-path", webhookCertPath, "webhook-cert-name", webhookCertName, "webhook-cert-key", webhookCertKey)
+
+		var err error
+		webhookCertWatcher, err = certwatcher.New(
+			filepath.Join(webhookCertPath, webhookCertName),
+			filepath.Join(webhookCertPath, webhookCertKey),
+		)
+		if err != nil {
+			setupLog.Error(err, "Failed to initialize webhook certificate watcher")
+			os.Exit(1)
+		}
+
+		webhookTLSOpts = append(webhookTLSOpts, func(config *tls.Config) {
+			config.GetCertificate = webhookCertWatcher.GetCertificate
+		})
+	}
+
+	webhookServer := webhook.NewServer(webhook.Options{
+		TLSOpts: webhookTLSOpts,
+		Port:    9443,
+	})
+
+	// Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server.
+	// More info:
+	// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/server
+	// - https://book.kubebuilder.io/reference/metrics.html
+	metricsServerOptions := diagnosticsOpts
+	if metricsAddr != "" {
+		metricsServerOptions.BindAddress = metricsAddr
+	}
+
+	if secureMetrics {
+		// FilterProvider is used to protect the metrics endpoint with authn/authz.
+		// These configurations ensure that only authorized users and service accounts
+		// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
+		// https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/filters#WithAuthenticationAndAuthorization
+		metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
+	}
+
+	if len(metricsCertPath) > 0 {
+		setupLog.Info("Initializing metrics certificate watcher using provided certificates",
+			"metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey)
+
+		var err error
+		metricsCertWatcher, err = certwatcher.New(
+			filepath.Join(metricsCertPath, metricsCertName),
+			filepath.Join(metricsCertPath, metricsCertKey),
+		)
+		if err != nil {
+			setupLog.Error(err, "to initialize metrics certificate watcher", "error", err)
+			os.Exit(1)
+		}
+
+		metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+			config.GetCertificate = metricsCertWatcher.GetCertificate
+		})
+	}
+
+	healthProbeBindAddress := healthAddr
+	if probeAddr != "" {
+		healthProbeBindAddress = probeAddr
+	}
+
 	mgr, err := ctrl.NewManager(restConfig, ctrl.Options{
 		Scheme:                     scheme,
 		Metrics:                    diagnosticsOpts,
@@ -169,11 +259,8 @@ func main() {
 			DefaultNamespaces: watchNamespaces,
 			SyncPeriod:        &syncPeriod,
 		},
-		WebhookServer: webhook.NewServer(webhook.Options{
-			Port:    webhookPort,
-			CertDir: webhookCertDir,
-		}),
-		HealthProbeBindAddress: healthAddr,
+		WebhookServer:          webhookServer,
+		HealthProbeBindAddress: healthProbeBindAddress,
 		EventBroadcaster:       broadcaster,
 	})
 	if err != nil {
@@ -192,15 +279,15 @@ func main() {
 
 	dnsutil.InitFromDNSResolver(dnsresolver)
 
-	if err = (&controllers.DOClusterReconciler{
+	if err = (&capdocontroller.DOClusterReconciler{
 		Client:           mgr.GetClient(),
 		Recorder:         mgr.GetEventRecorderFor("docluster-controller"),
 		ReconcileTimeout: reconcileTimeout,
 	}).SetupWithManager(ctx, mgr, controller.Options{}); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "DOCluster")
 		os.Exit(1)
 	}
-	if err = (&controllers.DOMachineReconciler{
+	if err = (&capdocontroller.DOMachineReconciler{
 		Client:           mgr.GetClient(),
 		Recorder:         mgr.GetEventRecorderFor("domachine-controller"),
 		ReconcileTimeout: reconcileTimeout,
@@ -241,6 +328,22 @@ func main() {
 		os.Exit(1)
 	}
 
+	if metricsCertWatcher != nil {
+		setupLog.Info("Adding metrics certificate watcher to manager")
+		if err := mgr.Add(metricsCertWatcher); err != nil {
+			setupLog.Error(err, "unable to add metrics certificate watcher to manager")
+			os.Exit(1)
+		}
+	}
+
+	if webhookCertWatcher != nil {
+		setupLog.Info("Adding webhook certificate watcher to manager")
+		if err := mgr.Add(webhookCertWatcher); err != nil {
+			setupLog.Error(err, "unable to add webhook certificate watcher to manager")
+			os.Exit(1)
+		}
+	}
+
 	setupLog.Info("starting manager", "version", version.Get().String(), "extended_info", version.Get())
 	if err := mgr.Start(ctx); err != nil {
 		setupLog.Error(err, "problem running manager")

config/certmanager/certificate-metrics.yaml

@@ -0,0 +1,17 @@
+# The following manifests contain a self-signed issuer CR and a metrics certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: metrics-certs  # this name should match the one appeared in kustomizeconfig.yaml
+  namespace: system
+spec:
+  dnsNames:
+    # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+    # replacements in the config/default/kustomization.yaml file.
+    - SERVICE_NAME.SERVICE_NAMESPACE.svc
+    - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: metrics-server-cert

config/certmanager/certificate-webhook.yaml

@@ -0,0 +1,17 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: serving-cert  # this name should match the one appeared in kustomizeconfig.yaml
+  namespace: system
+spec:
+  # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+  # replacements in the config/default/kustomization.yaml file.
+  dnsNames:
+    - SERVICE_NAME.SERVICE_NAMESPACE.svc
+    - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: webhook-server-cert

config/certmanager/issuer.yaml

@@ -0,0 +1,10 @@
+# The following manifest contains a self-signed issuer CR.
+# More information can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: selfsigned-issuer
+  namespace: system
+spec:
+  selfSigned: {}

config/certmanager/kustomization.yaml

@@ -1,7 +1,7 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
 resources:
-- certificate.yaml
+- issuer.yaml
+- certificate-webhook.yaml
+- certificate-metrics.yaml
 
 configurations:
 - kustomizeconfig.yaml

config/certmanager/kustomizeconfig.yaml

@@ -1,19 +1,8 @@
-# This configuration is for teaching kustomize how to update name ref and var substitution 
+# This configuration is for teaching kustomize how to update name ref substitution
 nameReference:
 - kind: Issuer
   group: cert-manager.io
   fieldSpecs:
-  - kind: Certificate
-    group: cert-manager.io
-    path: spec/issuerRef/name
-
-varReference:
-- kind: Certificate
-  group: cert-manager.io
-  path: spec/commonName
-- kind: Certificate
-  group: cert-manager.io
-  path: spec/dnsNames
-- kind: Certificate
-  group: cert-manager.io
-  path: spec/secretName
+    - kind: Certificate
+      group: cert-manager.io
+      path: spec/issuerRef/name