From ca99d20dd7638b88c8c63a9cebc6e47187c486f8 Mon Sep 17 00:00:00 2001 From: zhzhuang-zju Date: Wed, 15 Jan 2025 16:35:50 +0800 Subject: [PATCH] custom karmada component flags Signed-off-by: zhzhuang-zju --- .../crds/operator.karmada.io_karmadas.yaml | 276 ++++++++++++++++++ .../crds/operator.karmada.io_karmadas.yaml | 276 ++++++++++++++++++ operator/pkg/apis/operator/v1alpha1/helper.go | 18 ++ operator/pkg/apis/operator/v1alpha1/type.go | 38 +++ .../v1alpha1/zz_generated.deepcopy.go | 21 ++ .../pkg/controlplane/apiserver/apiserver.go | 4 +- operator/pkg/controlplane/controlplane.go | 8 +- operator/pkg/controlplane/etcd/etcd.go | 2 +- .../metricsadapter/metricsadapter.go | 2 +- operator/pkg/controlplane/search/search.go | 2 +- operator/pkg/controlplane/webhook/webhook.go | 2 +- operator/pkg/util/patcher/pather.go | 52 +++- 12 files changed, 685 insertions(+), 16 deletions(-) diff --git a/charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml b/charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml index e636ddc1f615..b9ff0e13fd5e 100644 --- a/charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml +++ b/charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml @@ -122,6 +122,33 @@ spec: queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and + a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array imagePullPolicy: description: |- ImagePullPolicy defines the policy for pulling the container image. @@ -560,7 +587,34 @@ spec: For supported flags, please see https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/ for details. + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array extraVolumeMounts: description: |- ExtraVolumeMounts specifies a list of extra volume mounts to be mounted into the API server's container @@ -2439,6 +2493,7 @@ spec: description: |- FeatureGates enabled by the user. More info: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/ + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates. type: object imagePullPolicy: description: |- @@ -2591,7 +2646,34 @@ spec: For supported flags, please see https://karmada.io/docs/reference/components/karmada-aggregated-apiserver for details. + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array featureGates: additionalProperties: type: boolean @@ -2599,6 +2681,7 @@ spec: FeatureGates enabled by the user. - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates. type: object imagePullPolicy: description: |- @@ -2739,7 +2822,34 @@ spec: For supported flags, please see https://karmada.io/docs/reference/components/karmada-controller-manager for details. + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array featureGates: additionalProperties: type: boolean @@ -2750,6 +2860,7 @@ spec: - PropagateDeps: https://karmada.io/docs/userguide/scheduling/propagate-dependencies - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates. type: object imagePullPolicy: description: |- @@ -2874,7 +2985,34 @@ spec: For supported flags, please see https://karmada.io/docs/reference/components/karmada-descheduler for details. + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array imagePullPolicy: description: |- ImagePullPolicy defines the policy for pulling the container image. @@ -2998,7 +3136,34 @@ spec: For supported flags, please see https://karmada.io/docs/reference/components/karmada-metrics-adapter for details. + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array imagePullPolicy: description: |- ImagePullPolicy defines the policy for pulling the container image. @@ -3122,7 +3287,34 @@ spec: For supported flags, please see https://karmada.io/docs/reference/components/karmada-scheduler for details. + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array featureGates: additionalProperties: type: boolean @@ -3130,6 +3322,7 @@ spec: FeatureGates enabled by the user. - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates. type: object imagePullPolicy: description: |- @@ -3254,7 +3447,34 @@ spec: For supported flags, please see https://karmada.io/docs/reference/components/karmada-search for details. + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array imagePullPolicy: description: |- ImagePullPolicy defines the policy for pulling the container image. @@ -3378,7 +3598,34 @@ spec: For supported flags, please see https://karmada.io/docs/reference/components/karmada-webhook for details. + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array imagePullPolicy: description: |- ImagePullPolicy defines the policy for pulling the container image. @@ -3541,13 +3788,41 @@ spec: For supported flags, please see https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/ for details. + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array featureGates: additionalProperties: type: boolean description: |- FeatureGates enabled by the user. More info: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/ + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates. type: object imagePullPolicy: description: |- @@ -3709,6 +3984,7 @@ spec: - PropagateDeps: https://karmada.io/docs/userguide/scheduling/propagate-dependencies - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates. type: object hostCluster: description: |- diff --git a/operator/config/crds/operator.karmada.io_karmadas.yaml b/operator/config/crds/operator.karmada.io_karmadas.yaml index e636ddc1f615..b9ff0e13fd5e 100644 --- a/operator/config/crds/operator.karmada.io_karmadas.yaml +++ b/operator/config/crds/operator.karmada.io_karmadas.yaml @@ -122,6 +122,33 @@ spec: queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and + a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array imagePullPolicy: description: |- ImagePullPolicy defines the policy for pulling the container image. @@ -560,7 +587,34 @@ spec: For supported flags, please see https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/ for details. + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array extraVolumeMounts: description: |- ExtraVolumeMounts specifies a list of extra volume mounts to be mounted into the API server's container @@ -2439,6 +2493,7 @@ spec: description: |- FeatureGates enabled by the user. More info: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/ + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates. type: object imagePullPolicy: description: |- @@ -2591,7 +2646,34 @@ spec: For supported flags, please see https://karmada.io/docs/reference/components/karmada-aggregated-apiserver for details. + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array featureGates: additionalProperties: type: boolean @@ -2599,6 +2681,7 @@ spec: FeatureGates enabled by the user. - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates. type: object imagePullPolicy: description: |- @@ -2739,7 +2822,34 @@ spec: For supported flags, please see https://karmada.io/docs/reference/components/karmada-controller-manager for details. + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array featureGates: additionalProperties: type: boolean @@ -2750,6 +2860,7 @@ spec: - PropagateDeps: https://karmada.io/docs/userguide/scheduling/propagate-dependencies - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates. type: object imagePullPolicy: description: |- @@ -2874,7 +2985,34 @@ spec: For supported flags, please see https://karmada.io/docs/reference/components/karmada-descheduler for details. + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array imagePullPolicy: description: |- ImagePullPolicy defines the policy for pulling the container image. @@ -2998,7 +3136,34 @@ spec: For supported flags, please see https://karmada.io/docs/reference/components/karmada-metrics-adapter for details. + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array imagePullPolicy: description: |- ImagePullPolicy defines the policy for pulling the container image. @@ -3122,7 +3287,34 @@ spec: For supported flags, please see https://karmada.io/docs/reference/components/karmada-scheduler for details. + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array featureGates: additionalProperties: type: boolean @@ -3130,6 +3322,7 @@ spec: FeatureGates enabled by the user. - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates. type: object imagePullPolicy: description: |- @@ -3254,7 +3447,34 @@ spec: For supported flags, please see https://karmada.io/docs/reference/components/karmada-search for details. + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array imagePullPolicy: description: |- ImagePullPolicy defines the policy for pulling the container image. @@ -3378,7 +3598,34 @@ spec: For supported flags, please see https://karmada.io/docs/reference/components/karmada-webhook for details. + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array imagePullPolicy: description: |- ImagePullPolicy defines the policy for pulling the container image. @@ -3541,13 +3788,41 @@ spec: For supported flags, please see https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/ for details. + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. type: object + extraFlags: + description: |- + ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + this list is the flag name as it appears on the command line except without leading + dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + + Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand the risks of this configuration. + + For supported flags for the components, please see + https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + for details. + items: + description: Flag represents a flag with a name and a value. + properties: + name: + description: Name the name of the flag. + type: string + value: + description: Value the value of the flag. + type: string + required: + - name + - value + type: object + type: array featureGates: additionalProperties: type: boolean description: |- FeatureGates enabled by the user. More info: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/ + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates. type: object imagePullPolicy: description: |- @@ -3709,6 +3984,7 @@ spec: - PropagateDeps: https://karmada.io/docs/userguide/scheduling/propagate-dependencies - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go + Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates. type: object hostCluster: description: |- diff --git a/operator/pkg/apis/operator/v1alpha1/helper.go b/operator/pkg/apis/operator/v1alpha1/helper.go index 05b80c0fe98b..8c9b751f5850 100644 --- a/operator/pkg/apis/operator/v1alpha1/helper.go +++ b/operator/pkg/apis/operator/v1alpha1/helper.go @@ -66,3 +66,21 @@ func KarmadaFailed(karmada *Karmada, conditionType ConditionType, message string apimeta.SetStatusCondition(&karmada.Status.Conditions, newCondition) } + +// GetCustomFlagNames returns a map of flag names to true for quick name existence checks. +func GetCustomFlagNames(extraFlags []Flag) map[string]bool { + names := make(map[string]bool) + for i := range extraFlags { + names[extraFlags[i].Name] = true + } + return names +} + +// GenerateFlags converts a slice of Flag structs into a slice of formatted flag strings. +func GenerateFlags(extraFlags []Flag) []string { + var flags []string + for i := range extraFlags { + flags = append(flags, fmt.Sprintf("--%s=%s", extraFlags[i].Name, extraFlags[i].Value)) + } + return flags +} diff --git a/operator/pkg/apis/operator/v1alpha1/type.go b/operator/pkg/apis/operator/v1alpha1/type.go index 69731e83a771..86aef75843dd 100644 --- a/operator/pkg/apis/operator/v1alpha1/type.go +++ b/operator/pkg/apis/operator/v1alpha1/type.go @@ -103,6 +103,7 @@ type KarmadaSpec struct { // - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models // More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go // +optional + // Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates. FeatureGates map[string]bool `json:"featureGates,omitempty"` // CRDTarball specifies the source from which the Karmada CRD tarball should be downloaded, along with the download policy to use. @@ -325,6 +326,7 @@ type KarmadaAPIServer struct { // https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/ // for details. // +optional + // Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. ExtraArgs map[string]string `json:"extraArgs,omitempty"` // ExtraVolumes specifies a list of extra volumes for the API server's pod @@ -352,6 +354,7 @@ type KarmadaAPIServer struct { // FeatureGates enabled by the user. // More info: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/ // +optional + // Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates. FeatureGates map[string]bool `json:"featureGates,omitempty"` } @@ -374,6 +377,7 @@ type KarmadaAggregatedAPIServer struct { // https://karmada.io/docs/reference/components/karmada-aggregated-apiserver // for details. // +optional + // Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. ExtraArgs map[string]string `json:"extraArgs,omitempty"` // CertSANs sets extra Subject Alternative Names for the API Server signing cert. @@ -384,6 +388,7 @@ type KarmadaAggregatedAPIServer struct { // - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models // More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go // +optional + // Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates. FeatureGates map[string]bool `json:"featureGates,omitempty"` } @@ -445,11 +450,13 @@ type KubeControllerManager struct { // https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/ // for details. // +optional + // Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. ExtraArgs map[string]string `json:"extraArgs,omitempty"` // FeatureGates enabled by the user. // More info: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/ // +optional + // Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates. FeatureGates map[string]bool `json:"featureGates,omitempty"` } @@ -487,6 +494,7 @@ type KarmadaControllerManager struct { // https://karmada.io/docs/reference/components/karmada-controller-manager // for details. // +optional + // Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. ExtraArgs map[string]string `json:"extraArgs,omitempty"` // FeatureGates enabled by the user. @@ -496,6 +504,7 @@ type KarmadaControllerManager struct { // - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models // More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go // +optional + // Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates. FeatureGates map[string]bool `json:"featureGates,omitempty"` } @@ -518,12 +527,14 @@ type KarmadaScheduler struct { // https://karmada.io/docs/reference/components/karmada-scheduler // for details. // +optional + // Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. ExtraArgs map[string]string `json:"extraArgs,omitempty"` // FeatureGates enabled by the user. // - CustomizedClusterResourceModeling: https://karmada.io/docs/userguide/scheduling/cluster-resources#start-to-use-cluster-resource-models // More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go // +optional + // Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting feature gates. FeatureGates map[string]bool `json:"featureGates,omitempty"` } @@ -546,6 +557,7 @@ type KarmadaDescheduler struct { // https://karmada.io/docs/reference/components/karmada-descheduler // for details. // +optional + // Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. ExtraArgs map[string]string `json:"extraArgs,omitempty"` } @@ -568,6 +580,7 @@ type KarmadaSearch struct { // https://karmada.io/docs/reference/components/karmada-search // for details. // +optional + // Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. ExtraArgs map[string]string `json:"extraArgs,omitempty"` } @@ -590,6 +603,7 @@ type KarmadaMetricsAdapter struct { // https://karmada.io/docs/reference/components/karmada-metrics-adapter // for details. // +optional + // Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. ExtraArgs map[string]string `json:"extraArgs,omitempty"` } @@ -612,6 +626,7 @@ type KarmadaWebhook struct { // https://karmada.io/docs/reference/components/karmada-webhook // for details. // +optional + // Deprecated: This field is deprecated and will be removed in a future version. Use ExtraFlags for setting extra flags. ExtraArgs map[string]string `json:"extraArgs,omitempty"` } @@ -648,6 +663,29 @@ type CommonSettings struct { // More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ // +optional Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + // ExtraFlags is an extra set of flags to pass to the karmada component. Flag name in + // this list is the flag name as it appears on the command line except without leading + // dash(es). Extra flags will override existing default flags. Duplicate extra flags are allowed. + // + // Note: Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + // state. Before you do it, please confirm that you understand the risks of this configuration. + // + // For supported flags for the components, please see + // https://karmada.io/docs/reference/components/karmada-aggregated-apiserver + // for details. + // +optional + ExtraFlags []Flag `json:"extraFlags,omitempty"` +} + +// Flag represents a flag with a name and a value. +type Flag struct { + // Name the name of the flag. + // +required + Name string `json:"name,omitempty"` + // Value the value of the flag. + // +required + Value string `json:"value,omitempty"` } // Image allows to customize the image used for components. diff --git a/operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go b/operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go index 4bdec62a012f..41fa6fd9237c 100644 --- a/operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go @@ -93,6 +93,11 @@ func (in *CommonSettings) DeepCopyInto(out *CommonSettings) { } } in.Resources.DeepCopyInto(&out.Resources) + if in.ExtraFlags != nil { + in, out := &in.ExtraFlags, &out.ExtraFlags + *out = make([]Flag, len(*in)) + copy(*out, *in) + } return } @@ -190,6 +195,22 @@ func (in *ExternalEtcd) DeepCopy() *ExternalEtcd { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Flag) DeepCopyInto(out *Flag) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Flag. +func (in *Flag) DeepCopy() *Flag { + if in == nil { + return nil + } + out := new(Flag) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPSource) DeepCopyInto(out *HTTPSource) { *out = *in diff --git a/operator/pkg/controlplane/apiserver/apiserver.go b/operator/pkg/controlplane/apiserver/apiserver.go index 4eb765c7b395..c404130a71b9 100644 --- a/operator/pkg/controlplane/apiserver/apiserver.go +++ b/operator/pkg/controlplane/apiserver/apiserver.go @@ -80,7 +80,7 @@ func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.K patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels). WithExtraArgs(cfg.ExtraArgs).WithExtraVolumeMounts(cfg.ExtraVolumeMounts). - WithExtraVolumes(cfg.ExtraVolumes).WithResources(cfg.Resources).ForDeployment(apiserverDeployment) + WithExtraVolumes(cfg.ExtraVolumes).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForDeployment(apiserverDeployment) if err := apiclient.CreateOrUpdateDeployment(client, apiserverDeployment); err != nil { return fmt.Errorf("error when creating deployment for %s, err: %w", apiserverDeployment.Name, err) @@ -143,7 +143,7 @@ func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operator } patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels). - WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).ForDeployment(aggregatedAPIServerDeployment) + WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForDeployment(aggregatedAPIServerDeployment) if err := apiclient.CreateOrUpdateDeployment(client, aggregatedAPIServerDeployment); err != nil { return fmt.Errorf("error when creating deployment for %s, err: %w", aggregatedAPIServerDeployment.Name, err) diff --git a/operator/pkg/controlplane/controlplane.go b/operator/pkg/controlplane/controlplane.go index 693d58bd6770..d541272e4fac 100644 --- a/operator/pkg/controlplane/controlplane.go +++ b/operator/pkg/controlplane/controlplane.go @@ -106,7 +106,7 @@ func getKubeControllerManagerManifest(name, namespace string, cfg *operatorv1alp } patcher.NewPatcher().WithAnnotations(cfg.Annotations). - WithLabels(cfg.Labels).WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).ForDeployment(kcm) + WithLabels(cfg.Labels).WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForDeployment(kcm) return kcm, nil } @@ -134,7 +134,7 @@ func getKarmadaControllerManagerManifest(name, namespace string, featureGates ma } patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels). - WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).ForDeployment(kcm) + WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForDeployment(kcm) return kcm, nil } @@ -163,7 +163,7 @@ func getKarmadaSchedulerManifest(name, namespace string, featureGates map[string } patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels). - WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).ForDeployment(scheduler) + WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForDeployment(scheduler) return scheduler, nil } @@ -192,7 +192,7 @@ func getKarmadaDeschedulerManifest(name, namespace string, featureGates map[stri } patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels). - WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).ForDeployment(descheduler) + WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForDeployment(descheduler) return descheduler, nil } diff --git a/operator/pkg/controlplane/etcd/etcd.go b/operator/pkg/controlplane/etcd/etcd.go index c4dd16b810ac..f66e17ec8d3a 100644 --- a/operator/pkg/controlplane/etcd/etcd.go +++ b/operator/pkg/controlplane/etcd/etcd.go @@ -91,7 +91,7 @@ func installKarmadaEtcd(client clientset.Interface, name, namespace string, cfg } patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels). - WithVolumeData(cfg.VolumeData).WithResources(cfg.Resources).ForStatefulSet(etcdStatefulSet) + WithVolumeData(cfg.VolumeData).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForStatefulSet(etcdStatefulSet) if err := apiclient.CreateOrUpdateStatefulSet(client, etcdStatefulSet); err != nil { return fmt.Errorf("error when creating Etcd statefulset, err: %w", err) diff --git a/operator/pkg/controlplane/metricsadapter/metricsadapter.go b/operator/pkg/controlplane/metricsadapter/metricsadapter.go index 08b6184fc835..3df195532666 100644 --- a/operator/pkg/controlplane/metricsadapter/metricsadapter.go +++ b/operator/pkg/controlplane/metricsadapter/metricsadapter.go @@ -63,7 +63,7 @@ func installKarmadaMetricAdapter(client clientset.Interface, cfg *operatorv1alph return fmt.Errorf("err when decoding KarmadaMetricAdapter Deployment: %w", err) } - patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).WithResources(cfg.Resources).ForDeployment(metricAdapter) + patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForDeployment(metricAdapter) if err := apiclient.CreateOrUpdateDeployment(client, metricAdapter); err != nil { return fmt.Errorf("error when creating deployment for %s, err: %w", metricAdapter.Name, err) diff --git a/operator/pkg/controlplane/search/search.go b/operator/pkg/controlplane/search/search.go index e7251d0c8e4e..e2e03f9f7a52 100644 --- a/operator/pkg/controlplane/search/search.go +++ b/operator/pkg/controlplane/search/search.go @@ -70,7 +70,7 @@ func installKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.Karm } patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels). - WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).ForDeployment(searchDeployment) + WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForDeployment(searchDeployment) if err := apiclient.CreateOrUpdateDeployment(client, searchDeployment); err != nil { return fmt.Errorf("error when creating deployment for %s, err: %w", searchDeployment.Name, err) diff --git a/operator/pkg/controlplane/webhook/webhook.go b/operator/pkg/controlplane/webhook/webhook.go index 66cddfc159e3..07c1ba99754a 100644 --- a/operator/pkg/controlplane/webhook/webhook.go +++ b/operator/pkg/controlplane/webhook/webhook.go @@ -64,7 +64,7 @@ func installKarmadaWebhook(client clientset.Interface, cfg *operatorv1alpha1.Kar } patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels). - WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).ForDeployment(webhookDeployment) + WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).WithExtraFlags(cfg.ExtraFlags).ForDeployment(webhookDeployment) if err := apiclient.CreateOrUpdateDeployment(client, webhookDeployment); err != nil { return fmt.Errorf("error when creating deployment for %s, err: %w", webhookDeployment.Name, err) diff --git a/operator/pkg/util/patcher/pather.go b/operator/pkg/util/patcher/pather.go index 9c883c32aaee..0e76b872d8d9 100644 --- a/operator/pkg/util/patcher/pather.go +++ b/operator/pkg/util/patcher/pather.go @@ -43,6 +43,7 @@ type Patcher struct { featureGates map[string]bool volume *operatorv1alpha1.VolumeData resources corev1.ResourceRequirements + extraFlags []operatorv1alpha1.Flag } // NewPatcher returns a patcher. @@ -98,6 +99,12 @@ func (p *Patcher) WithResources(resources corev1.ResourceRequirements) *Patcher return p } +// WithExtraFlags sets extraFlags to the patcher +func (p *Patcher) WithExtraFlags(extraFlags []operatorv1alpha1.Flag) *Patcher { + p.extraFlags = extraFlags + return p +} + // ForDeployment patches the deployment manifest. func (p *Patcher) ForDeployment(deployment *appsv1.Deployment) { deployment.Labels = labels.Merge(deployment.Labels, p.labels) @@ -110,6 +117,30 @@ func (p *Patcher) ForDeployment(deployment *appsv1.Deployment) { // It's considered the first container is the karmada component by default. deployment.Spec.Template.Spec.Containers[0].Resources = p.resources } + // Add extra volumes and volume mounts + // First container in the pod is expected to contain the Karmada component + deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, p.extraVolumes...) + deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[0].VolumeMounts, p.extraVolumeMounts...) + + if len(p.extraFlags) != 0 { + // It's considered the first container is the karmada component by default. + baseArguments := deployment.Spec.Template.Spec.Containers[0].Command + argsMap := parseArgumentListToMap(baseArguments) + + var command []string + overrideFlags := operatorv1alpha1.GetCustomFlagNames(p.extraFlags) + for name, value := range argsMap { + if _, ok := overrideFlags[name]; !ok { + command = append(command, fmt.Sprintf("--%s=%s", name, value)) + } + } + command = append(command, operatorv1alpha1.GenerateFlags(p.extraFlags)...) + sort.Strings(command) + deployment.Spec.Template.Spec.Containers[0].Command = append([]string{baseArguments[0]}, command...) + return + } + + // TODO: when the API ExtraArgs and FeatureGates are removed, delete the following logic. if len(p.extraArgs) != 0 || len(p.featureGates) != 0 { // It's considered the first container is the karmada component by default. baseArguments := deployment.Spec.Template.Spec.Containers[0].Command @@ -136,10 +167,6 @@ func (p *Patcher) ForDeployment(deployment *appsv1.Deployment) { command = append(command, buildArgumentListFromMap(argsMap, overrideArgs)...) deployment.Spec.Template.Spec.Containers[0].Command = command } - // Add extra volumes and volume mounts - // First container in the pod is expected to contain the Karmada component - deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, p.extraVolumes...) - deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[0].VolumeMounts, p.extraVolumeMounts...) } // ForStatefulSet patches the statefulset manifest. @@ -158,11 +185,24 @@ func (p *Patcher) ForStatefulSet(sts *appsv1.StatefulSet) { // It's considered the first container is the karmada component by default. sts.Spec.Template.Spec.Containers[0].Resources = p.resources } - if len(p.extraArgs) != 0 { + + if len(p.extraFlags) != 0 { // It's considered the first container is the karmada component by default. baseArguments := sts.Spec.Template.Spec.Containers[0].Command argsMap := parseArgumentListToMap(baseArguments) - sts.Spec.Template.Spec.Containers[0].Command = buildArgumentListFromMap(argsMap, p.extraArgs) + + // the first argument is most often the binary name + var command []string + overrideFlags := operatorv1alpha1.GetCustomFlagNames(p.extraFlags) + for name, value := range argsMap { + if _, ok := overrideFlags[name]; !ok { + command = append(command, fmt.Sprintf("--%s=%s", name, value)) + } + } + command = append(command, operatorv1alpha1.GenerateFlags(p.extraFlags)...) + sort.Strings(command) + sts.Spec.Template.Spec.Containers[0].Command = append([]string{baseArguments[0]}, command...) + return } }