custom karmada component flags

Signed-off-by: zhzhuang-zju <[email protected]>

operator/pkg/apis/operator/v1alpha1/helper.go

@@ -66,3 +66,21 @@ func KarmadaFailed(karmada *Karmada, conditionType ConditionType, message string
 
 	apimeta.SetStatusCondition(&karmada.Status.Conditions, newCondition)
 }
+
+// GetCustomFlagNames returns a map of flag names to true for quick name existence checks.
+func GetCustomFlagNames(extraFlags []Flag) map[string]bool {
+	names := make(map[string]bool)
+	for i := range extraFlags {
+		names[extraFlags[i].Name] = true
+	}
+	return names
+}
+
+// GenerateFlags converts a slice of Flag structs into a slice of formatted flag strings.
+func GenerateFlags(extraFlags []Flag) []string {
+	var flags []string
+	for i := range extraFlags {
+		flags = append(flags, fmt.Sprintf("--%s=%s", extraFlags[i].Name, extraFlags[i].Value))
+	}
+	return flags
+}

operator/pkg/apis/operator/v1alpha1/type.go

@@ -103,6 +103,7 @@ type KarmadaSpec struct {
 	// - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models
 	// More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates.
 	FeatureGates map[string]bool `json:"featureGates,omitempty"`
 
 	// CRDTarball specifies the source from which the Karmada CRD tarball should be downloaded, along with the download policy to use.
@@ -325,6 +326,7 @@ type KarmadaAPIServer struct {
 	// https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/
 	// for details.
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags.
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 
 	// ExtraVolumes specifies a list of extra volumes for the API server's pod
@@ -352,6 +354,7 @@ type KarmadaAPIServer struct {
 	// FeatureGates enabled by the user.
 	// More info: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates.
 	FeatureGates map[string]bool `json:"featureGates,omitempty"`
 }
 
@@ -374,6 +377,7 @@ type KarmadaAggregatedAPIServer struct {
 	// https://karmada.io/docs/reference/components/karmada-aggregated-apiserver
 	// for details.
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags.
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 
 	// CertSANs sets extra Subject Alternative Names for the API Server signing cert.
@@ -384,6 +388,7 @@ type KarmadaAggregatedAPIServer struct {
 	// - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models
 	// More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates.
 	FeatureGates map[string]bool `json:"featureGates,omitempty"`
 }
 
@@ -445,11 +450,13 @@ type KubeControllerManager struct {
 	// https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/
 	// for details.
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags.
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 
 	// FeatureGates enabled by the user.
 	// More info: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates.
 	FeatureGates map[string]bool `json:"featureGates,omitempty"`
 }
 
@@ -487,6 +494,7 @@ type KarmadaControllerManager struct {
 	// https://karmada.io/docs/reference/components/karmada-controller-manager
 	// for details.
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags.
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 
 	// FeatureGates enabled by the user.
@@ -496,6 +504,7 @@ type KarmadaControllerManager struct {
 	// - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models
 	// More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates.
 	FeatureGates map[string]bool `json:"featureGates,omitempty"`
 }
 
@@ -518,12 +527,14 @@ type KarmadaScheduler struct {
 	// https://karmada.io/docs/reference/components/karmada-scheduler
 	// for details.
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags.
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 
 	// FeatureGates enabled by the user.
 	// - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models
 	// More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates.
 	FeatureGates map[string]bool `json:"featureGates,omitempty"`
 }
 
@@ -546,6 +557,7 @@ type KarmadaDescheduler struct {
 	// https://karmada.io/docs/reference/components/karmada-descheduler
 	// for details.
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags.
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 }
 
@@ -568,6 +580,7 @@ type KarmadaSearch struct {
 	// https://karmada.io/docs/reference/components/karmada-search
 	// for details.
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags.
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 }
 
@@ -590,6 +603,7 @@ type KarmadaMetricsAdapter struct {
 	// https://karmada.io/docs/reference/components/karmada-metrics-adapter
 	// for details.
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags.
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 }
 
@@ -612,6 +626,7 @@ type KarmadaWebhook struct {
 	// https://karmada.io/docs/reference/components/karmada-webhook
 	// for details.
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags.
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 }
 
@@ -648,6 +663,29 @@ type CommonSettings struct {
 	// More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
 	// +optional
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
+
+	// ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in
+	// this list is the flag name as it appears on the command line except without leading
+	// dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed.
+	//
+	// Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy
+	// state. Before you do it, please confirm that you understand the risks of this configuration.
+	//
+	// For supported flags for the components, please see
+	// https://karmada.io/docs/reference/components/karmada-aggregated-apiserver
+	// for details.
+	// +optional
+	ExtraFlags []Flag `json:"extraFlags,omitempty"`
+}
+
+// Flag represents a flag with a name and a value.
+type Flag struct {
+	// Name the name of the flag.
+	// +required
+	Name string `json:"name,omitempty"`
+	// Value the value of the flag.
+	// +required
+	Value string `json:"value,omitempty"`
 }
 
 // Image allows to customize the image used for components.

operator/pkg/controlplane/apiserver/apiserver.go

@@ -80,7 +80,7 @@ func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.K
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
 		WithExtraArgs(cfg.ExtraArgs).WithExtraVolumeMounts(cfg.ExtraVolumeMounts).
-		WithExtraVolumes(cfg.ExtraVolumes).WithResources(cfg.Resources).ForDeployment(apiserverDeployment)
+		WithExtraVolumes(cfg.ExtraVolumes).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForDeployment(apiserverDeployment)
 
 	if err := apiclient.CreateOrUpdateDeployment(client, apiserverDeployment); err != nil {
 		return fmt.Errorf("error when creating deployment for %s, err: %w", apiserverDeployment.Name, err)
@@ -143,7 +143,7 @@ func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operator
 	}
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
-		WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).ForDeployment(aggregatedAPIServerDeployment)
+		WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForDeployment(aggregatedAPIServerDeployment)
 
 	if err := apiclient.CreateOrUpdateDeployment(client, aggregatedAPIServerDeployment); err != nil {
 		return fmt.Errorf("error when creating deployment for %s, err: %w", aggregatedAPIServerDeployment.Name, err)

operator/pkg/controlplane/controlplane.go

@@ -106,7 +106,7 @@ func getKubeControllerManagerManifest(name, namespace string, cfg *operatorv1alp
 	}
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).
-		WithLabels(cfg.Labels).WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).ForDeployment(kcm)
+		WithLabels(cfg.Labels).WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForDeployment(kcm)
 	return kcm, nil
 }
 
@@ -134,7 +134,7 @@ func getKarmadaControllerManagerManifest(name, namespace string, featureGates ma
 	}
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
-		WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).ForDeployment(kcm)
+		WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForDeployment(kcm)
 	return kcm, nil
 }
 
@@ -163,7 +163,7 @@ func getKarmadaSchedulerManifest(name, namespace string, featureGates map[string
 	}
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
-		WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).ForDeployment(scheduler)
+		WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForDeployment(scheduler)
 	return scheduler, nil
 }
 
@@ -192,7 +192,7 @@ func getKarmadaDeschedulerManifest(name, namespace string, featureGates map[stri
 	}
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
-		WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).ForDeployment(descheduler)
+		WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForDeployment(descheduler)
 
 	return descheduler, nil
 }

operator/pkg/controlplane/etcd/etcd.go

@@ -91,7 +91,7 @@ func installKarmadaEtcd(client clientset.Interface, name, namespace string, cfg
 	}
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
-		WithVolumeData(cfg.VolumeData).WithResources(cfg.Resources).ForStatefulSet(etcdStatefulSet)
+		WithVolumeData(cfg.VolumeData).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForStatefulSet(etcdStatefulSet)
 
 	if err := apiclient.CreateOrUpdateStatefulSet(client, etcdStatefulSet); err != nil {
 		return fmt.Errorf("error when creating Etcd statefulset, err: %w", err)

operator/pkg/controlplane/metricsadapter/metricsadapter.go

@@ -63,7 +63,7 @@ func installKarmadaMetricAdapter(client clientset.Interface, cfg *operatorv1alph
 		return fmt.Errorf("err when decoding KarmadaMetricAdapter Deployment: %w", err)
 	}
 
-	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).WithResources(cfg.Resources).ForDeployment(metricAdapter)
+	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForDeployment(metricAdapter)
 
 	if err := apiclient.CreateOrUpdateDeployment(client, metricAdapter); err != nil {
 		return fmt.Errorf("error when creating deployment for %s, err: %w", metricAdapter.Name, err)

operator/pkg/controlplane/search/search.go

@@ -70,7 +70,7 @@ func installKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.Karm
 	}
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
-		WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).ForDeployment(searchDeployment)
+		WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForDeployment(searchDeployment)
 
 	if err := apiclient.CreateOrUpdateDeployment(client, searchDeployment); err != nil {
 		return fmt.Errorf("error when creating deployment for %s, err: %w", searchDeployment.Name, err)

operator/pkg/controlplane/webhook/webhook.go

@@ -64,7 +64,7 @@ func installKarmadaWebhook(client clientset.Interface, cfg *operatorv1alpha1.Kar
 	}
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
-		WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).ForDeployment(webhookDeployment)
+		WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForDeployment(webhookDeployment)
 
 	if err := apiclient.CreateOrUpdateDeployment(client, webhookDeployment); err != nil {
 		return fmt.Errorf("error when creating deployment for %s, err: %w", webhookDeployment.Name, err)

operator/pkg/util/patcher/pather.go

@@ -43,6 +43,7 @@ type Patcher struct {
 	featureGates      map[string]bool
 	volume            *operatorv1alpha1.VolumeData
 	resources         corev1.ResourceRequirements
+	extraFlags        []operatorv1alpha1.Flag
 }
 
 // NewPatcher returns a patcher.
@@ -98,6 +99,12 @@ func (p *Patcher) WithResources(resources corev1.ResourceRequirements) *Patcher
 	return p
 }
 
+// WithExtraFlags sets extraFlags to the patcher
+func (p *Patcher) WithExtraFlags(extraFlags []operatorv1alpha1.Flag) *Patcher {
+	p.extraFlags = extraFlags
+	return p
+}
+
 // ForDeployment patches the deployment manifest.
 func (p *Patcher) ForDeployment(deployment *appsv1.Deployment) {
 	deployment.Labels = labels.Merge(deployment.Labels, p.labels)
@@ -110,6 +117,30 @@ func (p *Patcher) ForDeployment(deployment *appsv1.Deployment) {
 		// It's considered the first container is the karmada component by default.
 		deployment.Spec.Template.Spec.Containers[0].Resources = p.resources
 	}
+	// Add extra volumes and volume mounts
+	// First container in the pod is expected to contain the Karmada component
+	deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, p.extraVolumes...)
+	deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[0].VolumeMounts, p.extraVolumeMounts...)
+
+	if len(p.extraFlags) != 0 {
+		// It's considered the first container is the karmada component by default.
+		baseArguments := deployment.Spec.Template.Spec.Containers[0].Command
+		argsMap := parseArgumentListToMap(baseArguments)
+
+		var command []string
+		overrideFlags := operatorv1alpha1.GetCustomFlagNames(p.extraFlags)
+		for name, value := range argsMap {
+			if _, ok := overrideFlags[name]; !ok {
+				command = append(command, fmt.Sprintf("--%s=%s", name, value))
+			}
+		}
+		command = append(command, operatorv1alpha1.GenerateFlags(p.extraFlags)...)
+		sort.Strings(command)
+		deployment.Spec.Template.Spec.Containers[0].Command = append([]string{baseArguments[0]}, command...)
+		return
+	}
+
+	// TODO: when the API ExtraArgs and FeatureGates are removed, delete the following logic.
 	if len(p.extraArgs) != 0 || len(p.featureGates) != 0 {
 		// It's considered the first container is the karmada component by default.
 		baseArguments := deployment.Spec.Template.Spec.Containers[0].Command
@@ -136,10 +167,6 @@ func (p *Patcher) ForDeployment(deployment *appsv1.Deployment) {
 		command = append(command, buildArgumentListFromMap(argsMap, overrideArgs)...)
 		deployment.Spec.Template.Spec.Containers[0].Command = command
 	}
-	// Add extra volumes and volume mounts
-	// First container in the pod is expected to contain the Karmada component
-	deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, p.extraVolumes...)
-	deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[0].VolumeMounts, p.extraVolumeMounts...)
 }
 
 // ForStatefulSet patches the statefulset manifest.
@@ -158,11 +185,24 @@ func (p *Patcher) ForStatefulSet(sts *appsv1.StatefulSet) {
 		// It's considered the first container is the karmada component by default.
 		sts.Spec.Template.Spec.Containers[0].Resources = p.resources
 	}
-	if len(p.extraArgs) != 0 {
+
+	if len(p.extraFlags) != 0 {
 		// It's considered the first container is the karmada component by default.
 		baseArguments := sts.Spec.Template.Spec.Containers[0].Command
 		argsMap := parseArgumentListToMap(baseArguments)
-		sts.Spec.Template.Spec.Containers[0].Command = buildArgumentListFromMap(argsMap, p.extraArgs)
+
+		// the first argument is most often the binary name
+		var command []string
+		overrideFlags := operatorv1alpha1.GetCustomFlagNames(p.extraFlags)
+		for name, value := range argsMap {
+			if _, ok := overrideFlags[name]; !ok {
+				command = append(command, fmt.Sprintf("--%s=%s", name, value))
+			}
+		}
+		command = append(command, operatorv1alpha1.GenerateFlags(p.extraFlags)...)
+		sort.Strings(command)
+		sts.Spec.Template.Spec.Containers[0].Command = append([]string{baseArguments[0]}, command...)
+		return
 	}
 }