custom karmada component flags

Signed-off-by: zhzhuang-zju <[email protected]>

operator/pkg/apis/operator/v1alpha1/helper.go

@@ -66,3 +66,21 @@ func KarmadaFailed(karmada *Karmada, conditionType ConditionType, message string
 
 	apimeta.SetStatusCondition(&karmada.Status.Conditions, newCondition)
 }
+
+// GetCommandArgNames returns a map of command argument names to true for quick name existence checks.
+func GetCommandArgNames(commandArgs []CommandArg) map[string]bool {
+	names := make(map[string]bool)
+	for i := range commandArgs {
+		names[commandArgs[i].Name] = true
+	}
+	return names
+}
+
+// ConvertCommandArgsIntoFlags converts a slice of CommandArg structs into a slice of formatted flag strings.
+func ConvertCommandArgsIntoFlags(commandArgs []CommandArg) []string {
+	var flags []string
+	for i := range commandArgs {
+		flags = append(flags, fmt.Sprintf("--%s=%s", commandArgs[i].Name, commandArgs[i].Value))
+	}
+	return flags
+}

operator/pkg/apis/operator/v1alpha1/type.go

@@ -103,6 +103,7 @@ type KarmadaSpec struct {
 	// - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models
 	// More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraCommandArgs for setting feature gates.
 	FeatureGates map[string]bool `json:"featureGates,omitempty"`
 
 	// CRDTarball specifies the source from which the Karmada CRD tarball should be downloaded, along with the download policy to use.
@@ -325,6 +326,7 @@ type KarmadaAPIServer struct {
 	// https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/
 	// for details.
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraCommandArgs for setting extra flags.
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 
 	// ExtraVolumes specifies a list of extra volumes for the API server's pod
@@ -352,6 +354,7 @@ type KarmadaAPIServer struct {
 	// FeatureGates enabled by the user.
 	// More info: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraCommandArgs for setting feature gates.
 	FeatureGates map[string]bool `json:"featureGates,omitempty"`
 }
 
@@ -374,6 +377,7 @@ type KarmadaAggregatedAPIServer struct {
 	// https://karmada.io/docs/reference/components/karmada-aggregated-apiserver
 	// for details.
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraCommandArgs for setting extra flags.
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 
 	// CertSANs sets extra Subject Alternative Names for the API Server signing cert.
@@ -445,11 +449,13 @@ type KubeControllerManager struct {
 	// https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/
 	// for details.
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraCommandArgs for setting extra flags.
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 
 	// FeatureGates enabled by the user.
 	// More info: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraCommandArgs for setting feature gates.
 	FeatureGates map[string]bool `json:"featureGates,omitempty"`
 }
 
@@ -487,6 +493,7 @@ type KarmadaControllerManager struct {
 	// https://karmada.io/docs/reference/components/karmada-controller-manager
 	// for details.
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraCommandArgs for setting extra flags.
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 
 	// FeatureGates enabled by the user.
@@ -496,6 +503,7 @@ type KarmadaControllerManager struct {
 	// - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models
 	// More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraCommandArgs for setting feature gates.
 	FeatureGates map[string]bool `json:"featureGates,omitempty"`
 }
 
@@ -518,12 +526,14 @@ type KarmadaScheduler struct {
 	// https://karmada.io/docs/reference/components/karmada-scheduler
 	// for details.
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraCommandArgs for setting extra flags.
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 
 	// FeatureGates enabled by the user.
 	// - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models
 	// More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraCommandArgs for setting feature gates.
 	FeatureGates map[string]bool `json:"featureGates,omitempty"`
 }
 
@@ -546,6 +556,7 @@ type KarmadaDescheduler struct {
 	// https://karmada.io/docs/reference/components/karmada-descheduler
 	// for details.
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraCommandArgs for setting extra flags.
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 }
 
@@ -568,6 +579,7 @@ type KarmadaSearch struct {
 	// https://karmada.io/docs/reference/components/karmada-search
 	// for details.
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraCommandArgs for setting extra flags.
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 }
 
@@ -590,6 +602,7 @@ type KarmadaMetricsAdapter struct {
 	// https://karmada.io/docs/reference/components/karmada-metrics-adapter
 	// for details.
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraCommandArgs for setting extra flags.
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 }
 
@@ -612,6 +625,7 @@ type KarmadaWebhook struct {
 	// https://karmada.io/docs/reference/components/karmada-webhook
 	// for details.
 	// +optional
+	// Deprecated: This field is deprecated and will be removed in a future version. Use ExtraCommandArgs for setting extra flags.
 	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 }
 
@@ -654,6 +668,29 @@ type CommonSettings struct {
 	// +kubebuilder:default="system-node-critical"
 	// +optional
 	PriorityClassName string `json:"priorityClassName,omitempty"`
+
+	// ExtraCommandArgs is an extra set of flags to pass to the karmada component. A command argument name in
+	// this list is the flag name as it appears on the command line except without leading
+	// dash(es). Extra command arguments will override existing default flags. Duplicate extra command arguments are allowed.
+	//
+	// Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy
+	// state. Before you do it, please confirm that you understand the risks of this configuration.
+	//
+	// For supported flags for the components, please see
+	// https://karmada.io/docs/reference/components/{{ component_name }}
+	// for details.
+	// +optional
+	ExtraCommandArgs []CommandArg `json:"extraCommandArgs,omitempty"`
+}
+
+// CommandArg represents a command-line parameters with a name and a value.
+type CommandArg struct {
+	// Name the name of the command-line parameter.
+	// +required
+	Name string `json:"name,omitempty"`
+	// Value the value of the command-line parameter.
+	// +required
+	Value string `json:"value,omitempty"`
 }
 
 // Image allows to customize the image used for components.

operator/pkg/controlplane/apiserver/apiserver.go

@@ -81,6 +81,7 @@ func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.K
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
 		WithPriorityClassName(cfg.CommonSettings.PriorityClassName).
 		WithExtraArgs(cfg.ExtraArgs).WithExtraVolumeMounts(cfg.ExtraVolumeMounts).
+		WithExtraCommandArgs(cfg.ExtraCommandArgs).
 		WithExtraVolumes(cfg.ExtraVolumes).WithResources(cfg.Resources).ForDeployment(apiserverDeployment)
 
 	if err := apiclient.CreateOrUpdateDeployment(client, apiserverDeployment); err != nil {
@@ -145,6 +146,7 @@ func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operator
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
 		WithPriorityClassName(cfg.CommonSettings.PriorityClassName).
+		WithExtraCommandArgs(cfg.ExtraCommandArgs).
 		WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).ForDeployment(aggregatedAPIServerDeployment)
 
 	if err := apiclient.CreateOrUpdateDeployment(client, aggregatedAPIServerDeployment); err != nil {

operator/pkg/controlplane/controlplane.go

@@ -107,6 +107,7 @@ func getKubeControllerManagerManifest(name, namespace string, cfg *operatorv1alp
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).
 		WithPriorityClassName(cfg.CommonSettings.PriorityClassName).
+		WithExtraCommandArgs(cfg.ExtraCommandArgs).
 		WithLabels(cfg.Labels).WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).ForDeployment(kcm)
 	return kcm, nil
 }
@@ -136,6 +137,7 @@ func getKarmadaControllerManagerManifest(name, namespace string, featureGates ma
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
 		WithPriorityClassName(cfg.CommonSettings.PriorityClassName).
+		WithExtraCommandArgs(cfg.ExtraCommandArgs).
 		WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).ForDeployment(kcm)
 	return kcm, nil
 }
@@ -166,6 +168,7 @@ func getKarmadaSchedulerManifest(name, namespace string, featureGates map[string
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
 		WithPriorityClassName(cfg.CommonSettings.PriorityClassName).
+		WithExtraCommandArgs(cfg.ExtraCommandArgs).
 		WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).ForDeployment(scheduler)
 	return scheduler, nil
 }
@@ -196,6 +199,7 @@ func getKarmadaDeschedulerManifest(name, namespace string, featureGates map[stri
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
 		WithPriorityClassName(cfg.CommonSettings.PriorityClassName).
+		WithExtraCommandArgs(cfg.ExtraCommandArgs).
 		WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).ForDeployment(descheduler)
 
 	return descheduler, nil

operator/pkg/controlplane/etcd/etcd.go

@@ -92,6 +92,7 @@ func installKarmadaEtcd(client clientset.Interface, name, namespace string, cfg
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
 		WithPriorityClassName(cfg.CommonSettings.PriorityClassName).
+		WithExtraCommandArgs(cfg.ExtraCommandArgs).
 		WithVolumeData(cfg.VolumeData).WithResources(cfg.Resources).ForStatefulSet(etcdStatefulSet)
 
 	if err := apiclient.CreateOrUpdateStatefulSet(client, etcdStatefulSet); err != nil {

operator/pkg/controlplane/metricsadapter/metricsadapter.go

@@ -65,6 +65,7 @@ func installKarmadaMetricAdapter(client clientset.Interface, cfg *operatorv1alph
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).WithPriorityClassName(cfg.CommonSettings.PriorityClassName).
 		WithResources(cfg.Resources).ForDeployment(metricAdapter)
+	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).WithResources(cfg.Resources).WithExtraCommandArgs(cfg.ExtraCommandArgs).ForDeployment(metricAdapter)
 
 	if err := apiclient.CreateOrUpdateDeployment(client, metricAdapter); err != nil {
 		return fmt.Errorf("error when creating deployment for %s, err: %w", metricAdapter.Name, err)

operator/pkg/controlplane/search/search.go

@@ -71,6 +71,7 @@ func installKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.Karm
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
 		WithPriorityClassName(cfg.CommonSettings.PriorityClassName).
+		WithExtraCommandArgs(cfg.ExtraCommandArgs).
 		WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).ForDeployment(searchDeployment)
 
 	if err := apiclient.CreateOrUpdateDeployment(client, searchDeployment); err != nil {

operator/pkg/controlplane/webhook/webhook.go

@@ -65,6 +65,7 @@ func installKarmadaWebhook(client clientset.Interface, cfg *operatorv1alpha1.Kar
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
 		WithPriorityClassName(cfg.CommonSettings.PriorityClassName).
+		WithExtraCommandArgs(cfg.ExtraCommandArgs).
 		WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).ForDeployment(webhookDeployment)
 
 	if err := apiclient.CreateOrUpdateDeployment(client, webhookDeployment); err != nil {

operator/pkg/util/patcher/pather.go

@@ -44,6 +44,7 @@ type Patcher struct {
 	featureGates      map[string]bool
 	volume            *operatorv1alpha1.VolumeData
 	resources         corev1.ResourceRequirements
+	extraCommandArgs  []operatorv1alpha1.CommandArg
 }
 
 // NewPatcher returns a patcher.
@@ -105,6 +106,12 @@ func (p *Patcher) WithResources(resources corev1.ResourceRequirements) *Patcher
 	return p
 }
 
+// WithExtraCommandArgs sets extraCommandArgs to the patcher.
+func (p *Patcher) WithExtraCommandArgs(extraCommandArgs []operatorv1alpha1.CommandArg) *Patcher {
+	p.extraCommandArgs = extraCommandArgs
+	return p
+}
+
 // ForDeployment patches the deployment manifest.
 func (p *Patcher) ForDeployment(deployment *appsv1.Deployment) {
 	deployment.Labels = labels.Merge(deployment.Labels, p.labels)
@@ -118,6 +125,30 @@ func (p *Patcher) ForDeployment(deployment *appsv1.Deployment) {
 		// It's considered the first container is the karmada component by default.
 		deployment.Spec.Template.Spec.Containers[0].Resources = p.resources
 	}
+	// Add extra volumes and volume mounts
+	// First container in the pod is expected to contain the Karmada component
+	deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, p.extraVolumes...)
+	deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[0].VolumeMounts, p.extraVolumeMounts...)
+
+	if len(p.extraCommandArgs) != 0 {
+		// It's considered the first container is the karmada component by default.
+		baseArguments := deployment.Spec.Template.Spec.Containers[0].Command
+		argsMap := parseArgumentListToMap(baseArguments)
+
+		var command []string
+		extraCommandArgNames := operatorv1alpha1.GetCommandArgNames(p.extraCommandArgs)
+		for name, value := range argsMap {
+			if _, ok := extraCommandArgNames[name]; !ok {
+				command = append(command, fmt.Sprintf("--%s=%s", name, value))
+			}
+		}
+		command = append(command, operatorv1alpha1.ConvertCommandArgsIntoFlags(p.extraCommandArgs)...)
+		sort.Strings(command)
+		deployment.Spec.Template.Spec.Containers[0].Command = append([]string{baseArguments[0]}, command...)
+		return
+	}
+
+	// TODO: when the API ExtraArgs and FeatureGates are removed, delete the following logic.
 	if len(p.extraArgs) != 0 || len(p.featureGates) != 0 {
 		// It's considered the first container is the karmada component by default.
 		baseArguments := deployment.Spec.Template.Spec.Containers[0].Command
@@ -144,10 +175,6 @@ func (p *Patcher) ForDeployment(deployment *appsv1.Deployment) {
 		command = append(command, buildArgumentListFromMap(argsMap, overrideArgs)...)
 		deployment.Spec.Template.Spec.Containers[0].Command = command
 	}
-	// Add extra volumes and volume mounts
-	// First container in the pod is expected to contain the Karmada component
-	deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, p.extraVolumes...)
-	deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[0].VolumeMounts, p.extraVolumeMounts...)
 }
 
 // ForStatefulSet patches the statefulset manifest.
@@ -167,11 +194,24 @@ func (p *Patcher) ForStatefulSet(sts *appsv1.StatefulSet) {
 		// It's considered the first container is the karmada component by default.
 		sts.Spec.Template.Spec.Containers[0].Resources = p.resources
 	}
-	if len(p.extraArgs) != 0 {
+
+	if len(p.extraCommandArgs) != 0 {
 		// It's considered the first container is the karmada component by default.
 		baseArguments := sts.Spec.Template.Spec.Containers[0].Command
 		argsMap := parseArgumentListToMap(baseArguments)
-		sts.Spec.Template.Spec.Containers[0].Command = buildArgumentListFromMap(argsMap, p.extraArgs)
+
+		// the first argument is most often the binary name
+		var command []string
+		extraCommandArgNames := operatorv1alpha1.GetCommandArgNames(p.extraCommandArgs)
+		for name, value := range argsMap {
+			if _, ok := extraCommandArgNames[name]; !ok {
+				command = append(command, fmt.Sprintf("--%s=%s", name, value))
+			}
+		}
+		command = append(command, operatorv1alpha1.ConvertCommandArgsIntoFlags(p.extraCommandArgs)...)
+		sort.Strings(command)
+		sts.Spec.Template.Spec.Containers[0].Command = append([]string{baseArguments[0]}, command...)
+		return
 	}
 }