Support for Version Management on Imported RKE2/K3s Clusters (#669)

docs.md

@@ -89,6 +89,14 @@ If yes, the webhook redacts the role, so that it only grants a deletion permissi
 
 ## Cluster
 
+
+### Mutation Checks
+
+##### Feature: version management on imported RKE2/K3s cluster 
+
+- When a cluster is created or updated, add the `rancher.io/imported-cluster-version-management: system-default` annotation if the annotation is missing or its value is an empty string.
+
+
 ### Validation Checks
 
 #### Annotations validation
@@ -99,6 +107,14 @@ When a cluster is updated `field.cattle.io/creator-principal-name` and `field.ca
 
 If `field.cattle.io/no-creator-rbac` annotation is set, `field.cattle.io/creatorId` cannot be set.
 
+
+##### Feature: version management on imported RKE2/K3s cluster
+
+ - When a cluster is created or updated, the `rancher.io/imported-cluster-version-management` annotation must be set with a valid value (true, false, or system-default). 
+ - If the cluster represents other types of clusters and the annotation is present, the webhook will permit the request with a warning that the annotation is intended for imported RKE2/k3s clusters and will not take effect on this cluster.
+ - If version management is determined to be disabled, and the `.spec.rke2Config` or `.spec.k3sConfig` field exists in the new cluster object with a value different from the old one, the webhook will permit the update with a warning indicating that these changes will not take effect until version management is enabled for the cluster.
+ - If version management is determined to be disabled, and the `.spec.rke2Config` or `.spec.k3sConfig` field is missing, the webhook will permit the request to allow users to remove the unused fields via API or Terraform.
+
 ## ClusterProxyConfig
 
 ### Validation Checks

go.mod

@@ -18,9 +18,11 @@ replace (
 	k8s.io/component-helpers => k8s.io/component-helpers v0.32.1
 	k8s.io/controller-manager => k8s.io/controller-manager v0.32.1
 	k8s.io/cri-api => k8s.io/cri-api v0.32.1
+	k8s.io/cri-client => k8s.io/cri-client v0.32.1
 	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.32.1
 	k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.32.1
 	k8s.io/endpointslice => k8s.io/endpointslice v0.32.1
+	k8s.io/externaljwt => k8s.io/externaljwt v0.32.1
 	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.32.1
 	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.32.1
 	k8s.io/kube-proxy => k8s.io/kube-proxy v0.32.1

pkg/resources/management.cattle.io/v3/cluster/Cluster.md

@@ -1,3 +1,11 @@
+
+## Mutation Checks
+
+#### Feature: version management on imported RKE2/K3s cluster 
+
+- When a cluster is created or updated, add the `rancher.io/imported-cluster-version-management: system-default` annotation if the annotation is missing or its value is an empty string.
+
+
 ## Validation Checks
 
 ### Annotations validation
@@ -7,3 +15,11 @@ When a cluster is created and `field.cattle.io/creator-principal-name` annotatio
 When a cluster is updated `field.cattle.io/creator-principal-name` and `field.cattle.io/creatorId` annotations must stay the same or removed.
 
 If `field.cattle.io/no-creator-rbac` annotation is set, `field.cattle.io/creatorId` cannot be set.
+
+
+#### Feature: version management on imported RKE2/K3s cluster
+
+ - When a cluster is created or updated, the `rancher.io/imported-cluster-version-management` annotation must be set with a valid value (true, false, or system-default). 
+ - If the cluster represents other types of clusters and the annotation is present, the webhook will permit the request with a warning that the annotation is intended for imported RKE2/k3s clusters and will not take effect on this cluster.
+ - If version management is determined to be disabled, and the `.spec.rke2Config` or `.spec.k3sConfig` field exists in the new cluster object with a value different from the old one, the webhook will permit the update with a warning indicating that these changes will not take effect until version management is enabled for the cluster.
+ - If version management is determined to be disabled, and the `.spec.rke2Config` or `.spec.k3sConfig` field is missing, the webhook will permit the request to allow users to remove the unused fields via API or Terraform.

pkg/resources/management.cattle.io/v3/cluster/mutator.go

@@ -64,9 +64,34 @@ func (m *ManagementClusterMutator) Admit(request *admission.Request) (*admission
 	if err != nil {
 		return nil, fmt.Errorf("unable to re-marshal new cluster: %w", err)
 	}
+
+	err = m.mutatePSACT(oldCluster, newCluster, request.Operation)
+	if err != nil {
+		return nil, fmt.Errorf("failed to mutate PSACT: %w", err)
+	}
+
+	m.mutateVersionManagement(newCluster, request.Operation)
+
+	response := &admissionv1.AdmissionResponse{}
+	// we use the re-marshalled new cluster to make sure that the patch doesn't drop "unknown" fields which were
+	// in the json, but not in the cluster struct. This can occur due to out of date RKE versions
+	if err := patch.CreatePatch(newClusterRaw, newCluster, response); err != nil {
+		return nil, fmt.Errorf("failed to create patch: %w", err)
+	}
+	response.Allowed = true
+	return response, nil
+}
+
+// mutatePSACT updates the newCluster's Pod Security Admission (PSA) configuration based on changes to
+// the cluster's `DefaultPodSecurityAdmissionConfigurationTemplateName`.
+// It applies or removes the PSA plugin configuration depending on the operation and the current cluster state.
+func (m *ManagementClusterMutator) mutatePSACT(oldCluster, newCluster *apisv3.Cluster, operation admissionv1.Operation) error {
 	// no need to mutate the local cluster, or imported cluster which represents a KEv2 cluster (GKE/EKS/AKS) or v1 Provisioning Cluster
 	if newCluster.Name == "local" || newCluster.Spec.RancherKubernetesEngineConfig == nil {
-		return admission.ResponseAllowed(), nil
+		return nil
+	}
+	if operation != admissionv1.Update && operation != admissionv1.Create {
+		return nil
 	}
 	newTemplateName := newCluster.Spec.DefaultPodSecurityAdmissionConfigurationTemplateName
 	oldTemplateName := oldCluster.Spec.DefaultPodSecurityAdmissionConfigurationTemplateName
@@ -75,13 +100,11 @@ func (m *ManagementClusterMutator) Admit(request *admission.Request) (*admission
 	if newTemplateName != "" {
 		err := m.setPSAConfig(newCluster)
 		if err != nil && !apierrors.IsNotFound(err) {
-			return nil, fmt.Errorf("failed to set PSAconfig: %w", err)
+			return fmt.Errorf("failed to set PSAconfig: %w", err)
 		}
 	} else {
-		switch request.Operation {
-		case admissionv1.Create:
-			return admission.ResponseAllowed(), nil
-		case admissionv1.Update:
+		if operation == admissionv1.Update {
+			// The case of dropping the PSACT in the UPDATE operation:
 			// It is a valid use case where user switches from using PSACT to putting a PluginConfig for PSA under kube-api.AdmissionConfiguration,
 			// but it is not a valid use case where the PluginConfig for PSA has the same content as the one in the previous-set PSACT,
 			// so we need to drop it in this case.
@@ -97,15 +120,7 @@ func (m *ManagementClusterMutator) Admit(request *admission.Request) (*admission
 			}
 		}
 	}
-
-	response := &admissionv1.AdmissionResponse{}
-	// we use the re-marshalled new cluster to make sure that the patch doesn't drop "unknown" fields which were
-	// in the json, but not in the cluster struct. This can occur due to out of date RKE versions
-	if err := patch.CreatePatch(newClusterRaw, newCluster, response); err != nil {
-		return response, fmt.Errorf("failed to create patch: %w", err)
-	}
-	response.Allowed = true
-	return response, nil
+	return nil
 }
 
 // setPSAConfig makes sure that the PodSecurity config under the admission_configuration section matches the
@@ -135,3 +150,22 @@ func (m *ManagementClusterMutator) setPSAConfig(cluster *apisv3.Cluster) error {
 	cluster.Spec.RancherKubernetesEngineConfig.Services.KubeAPI.AdmissionConfiguration = admissionConfig
 	return nil
 }
+
+// mutateVersionManagement set the annotation for version management if it is missing or has empty value on an imported RKE2/K3s cluster
+func (m *ManagementClusterMutator) mutateVersionManagement(cluster *apisv3.Cluster, operation admissionv1.Operation) {
+	if operation != admissionv1.Update && operation != admissionv1.Create {
+		return
+	}
+	if cluster.Status.Driver != apisv3.ClusterDriverRke2 && cluster.Status.Driver != apisv3.ClusterDriverK3s {
+		return
+	}
+
+	val, ok := cluster.Annotations[VersionManagementAnno]
+	if !ok || val == "" {
+		if cluster.Annotations == nil {
+			cluster.Annotations = make(map[string]string)
+		}
+		cluster.Annotations[VersionManagementAnno] = "system-default"
+	}
+	return
+}

pkg/resources/management.cattle.io/v3/cluster/mutator_test.go

@@ -9,6 +9,7 @@ import (
 	data2 "github.com/rancher/wrangler/v3/pkg/data"
 	"github.com/stretchr/testify/assert"
 	admissionv1 "k8s.io/api/admission/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -44,3 +45,63 @@ func TestAdmitPreserveUnknownFields(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Nil(t, response.Patch)
 }
+
+func TestMutateVersionManagement(t *testing.T) {
+	tests := []struct {
+		name      string
+		cluster   *v3.Cluster
+		operation admissionv1.Operation
+		expect    bool
+	}{
+		{
+			name:      "invalid operation",
+			cluster:   &v3.Cluster{},
+			operation: admissionv1.Delete,
+			expect:    false,
+		},
+		{
+			name: "invalid cluster",
+			cluster: &v3.Cluster{
+				Status: v3.ClusterStatus{
+					Driver: "imported",
+				},
+			},
+			operation: admissionv1.Update,
+			expect:    false,
+		},
+		{
+			name: "missing annotation",
+			cluster: &v3.Cluster{
+				Status: v3.ClusterStatus{
+					Driver: "rke2",
+				},
+			},
+			operation: admissionv1.Create,
+			expect:    true,
+		},
+		{
+			name: "empty value",
+			cluster: &v3.Cluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						VersionManagementAnno: "",
+					},
+				},
+				Status: v3.ClusterStatus{
+					Driver: "k3s",
+				},
+			},
+			operation: admissionv1.Update,
+			expect:    true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			m := &ManagementClusterMutator{}
+			m.mutateVersionManagement(tt.cluster, tt.operation)
+			if tt.expect {
+				assert.Equal(t, tt.cluster.Annotations[VersionManagementAnno], "system-default")
+			}
+		})
+	}
+}