diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ada68ff --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +bin/* +Dockerfile.cross + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Go workspace file +go.work + +# Kubernetes Generated files - skip generated files, except for vendored files +!vendor/**/zz_generated.* + +# editor and IDE paraphernalia +.idea +.vscode +*.swp +*.swo +*~ diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 0000000..5909125 --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,71 @@ +# golangci-lint configuration file +# see: https://golangci-lint.run/usage/configuration/ + +# Settings of specific linters +linters-settings: + gocritic: + enabled-checks: + - dupImport + disabled-checks: + - appendAssign + - exitAfterDefer + govet: + enable: + - nilness + gci: + sections: + - standard # Standard section: captures all standard packages. + - default # Default section: contains all imports that could not be matched to another section type. + - prefix(sigs.k8s.io/kueue/cmd/experimental/kjobctl) # Custom section: groups all imports with the specified Prefix. + - blank # Blank section: contains all blank imports. This section is not present unless explicitly enabled. + - dot # Dot section: contains all dot imports. + skip-generated: true # Skip generated files. + perfsprint: + int-conversion: false + errorf: true + sprintf1: false + strconcat: false + revive: + enable-all-rules: false + rules: + - name: empty-lines + - name: var-naming + - name: redundant-import-alias + +# Settings for enabling and disabling linters +linters: + enable: + - copyloopvar + - dupword + - durationcheck + - gci + - ginkgolinter + - gocritic + - govet + - loggercheck + - misspell + - perfsprint + - revive + - unconvert + +# Settings related to issues +issues: + # Which dirs to exclude: issues from them won't be reported + exclude-dirs: + - bin + # Excluding configuration per-path, per-linter, per-text and per-source + exclude-rules: + - linters: + - staticcheck + # TODO(#768): Drop when incrementing the API version. + text: "SA1019: constants.QueueAnnotation is deprecated" + - linters: + - staticcheck + text: "SA1019: j.RayCluster.Status.State is deprecated: the State field is replaced by the Conditions field." + - linters: + - staticcheck + text: "SA1019: rayCluster.Status.State is deprecated: the State field is replaced by the Conditions field." + # Show all issues from a linter + max-issues-per-linter: 0 + # Show all issues with the same text + max-same-issues: 0 diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 0000000..01ec289 --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,7 @@ +# shellcheck configuration file +# see: https://github.com/koalaman/shellcheck/wiki + +# Allow following sourced files that are not specified in the command, +# we need this because we specify one file at a time in order to trivially +# detect which files are failing +external-sources=true diff --git a/LICENSE b/LICENSE index 8dada3e..15b8c14 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright 2024 The Kubernetes Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e15788a --- /dev/null +++ b/Makefile @@ -0,0 +1,265 @@ +# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. +ENVTEST_K8S_VERSION = 1.31.0 + +ifeq ($(shell uname),Darwin) + GOFLAGS ?= -ldflags=-linkmode=internal +endif + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +CGO_ENABLED ?= 0 + +GO_CMD ?= go +GO_FMT ?= gofmt + +PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) +ARTIFACTS ?= $(PROJECT_DIR)/bin +TOOLS_DIR ?= $(PROJECT_DIR)/hack/tools +CLI_PLATFORMS ?= linux/amd64,linux/arm64,darwin/amd64,darwin/arm64 +EXTERNAL_CRDS_DIR ?= $(PROJECT_DIR)/dep-crds + +MANIFEST_DIR := $(PROJECT_DIR)/config/crd +MANIFEST_SOURCES := $(shell find $(MANIFEST_DIR) -name "*.yaml") +EMBEDDED_MANIFEST_DIR := $(PROJECT_DIR)/pkg/cmd/printcrds/embed +EMBEDDED_MANIFEST := $(EMBEDDED_MANIFEST_DIR)/manifest.gz + +# Number of processes to use during integration tests to run specs within a +# suite in parallel. Suites still run sequentially. User may set this value to 1 +# to run without parallelism. +INTEGRATION_NPROCS ?= 4 +# Folder where the integration tests are located. +INTEGRATION_TARGET ?= $(PROJECT_DIR)/test/integration/... + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +# For local testing, we should allow user to use different kind cluster name +# Default will delete default kind cluster +KIND_CLUSTER_NAME ?= kind +E2E_KIND_VERSION ?= kindest/node:v1.31.0 +K8S_VERSION = $(E2E_KIND_VERSION:kindest/node:v%=%) + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk command is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Release +.PHONY: artifacts +artifacts: kustomize ## Generate release artifacts. + if [ -d artifacts ]; then rm -rf artifacts; fi + mkdir -p artifacts + $(KUSTOMIZE) build config/default -o artifacts/manifests.yaml + CGO_ENABLED=$(CGO_ENABLED) GO_CMD="$(GO_CMD)" LD_FLAGS="$(LD_FLAGS)" BUILD_DIR="artifacts" BUILD_NAME=kubectl-kjob PLATFORMS="$(CLI_PLATFORMS)" ./hack/multiplatform-build.sh ./cmd/kjobctl/main.go + +##@ Development + +.PHONY: manifests +manifests: controller-gen ## Generate ClusterRole and CustomResourceDefinition objects. + # We need to allow dangerous types because on RayJobSpec still uses float32 type. + $(CONTROLLER_GEN) crd:allowDangerousTypes=true paths="./..." output:crd:artifacts:config=config/crd/bases + +.PHONY: generate +generate: gomod-download controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + TOOLS_DIR=${TOOLS_DIR} ./hack/update-codegen.sh $(GO_CMD) + +.PHONY: verify +verify: gomod-verify ci-lint shell-lint fmt-verify manifests generate generate-kjobctl-docs embeded-manifest + git --no-pager diff --exit-code config/crd apis client-go docs pkg/cmd/printcrds/embed + +.PHONY: gomod-verify +gomod-verify: + $(GO_CMD) mod tidy + git --no-pager diff --exit-code go.mod go.sum + +.PHONY: ci-lint +ci-lint: golangci-lint + $(GOLANGCI_LINT) run --timeout 15m0s + +.PHONY: shell-lint +shell-lint: ## Run shell linting. + $(PROJECT_DIR)/hack/shellcheck/verify.sh + +.PHONY: fmt-verify +fmt-verify: + @out=`$(GO_FMT) -w -l -d $$(find . -name '*.go')`; \ + if [ -n "$$out" ]; then \ + echo "$$out"; \ + exit 1; \ + fi + +$(EMBEDDED_MANIFEST): kustomize $(MANIFEST_SOURCES) + mkdir -p $(EMBEDDED_MANIFEST_DIR) + # This is embedded in the kjobctl binary. + $(KUSTOMIZE) build $(MANIFEST_DIR) | $(GO_CMD) run $(PROJECT_DIR)/hack/tools/gzip/gzip.go > $(EMBEDDED_MANIFEST) + +embeded-manifest: $(EMBEDDED_MANIFEST) + +.PHONY: gomod-download +gomod-download: + $(GO_CMD) mod download + +.PHONY: vet +vet: ## Run go vet against code. + $(GO_CMD) vet ./... + +.PHONY: test +test: verify vet test-unit test-integration test-e2e ## Run all tests. + +.PHONY: test-unit +test-unit: gomod-download gotestsum embeded-manifest ## Run unit tests. + $(GOTESTSUM) --junitfile $(ARTIFACTS)/junit-unit.xml -- $(GOFLAGS) $(GO_TEST_FLAGS) $(shell $(GO_CMD) list ./... | grep -v '/test/') -coverpkg=./... -coverprofile $(ARTIFACTS)/cover.out + +.PHONY: test-integration +test-integration: gomod-download envtest ginkgo embeded-manifest ray-operator-crd ## Run integration tests. + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" \ + KUEUE_BIN=$(PROJECT_DIR)/bin \ + ENVTEST_K8S_VERSION=$(ENVTEST_K8S_VERSION) \ + $(GINKGO) $(GINKGO_ARGS) -procs=$(INTEGRATION_NPROCS) --race --junit-report=junit-integration.xml --output-dir=$(ARTIFACTS) -v $(INTEGRATION_TARGET) + +CREATE_KIND_CLUSTER ?= true +.PHONY: test-e2e +test-e2e: kind kubectl-kjob + @echo Running e2e for k8s ${K8S_VERSION} + E2E_KIND_VERSION=$(E2E_KIND_VERSION) KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) ARTIFACTS="$(ARTIFACTS)/$@" GINKGO_ARGS="$(GINKGO_ARGS)" ./hack/e2e-test.sh + +.PHONY: lint +lint: golangci-lint ## Run golangci-lint linter + $(GOLANGCI_LINT) run + +.PHONY: lint-fix +lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes + $(GOLANGCI_LINT) run --fix + +##@ Deployment + +ifndef ignore-not-found + ignore-not-found = false +endif + +.PHONY: install +install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | $(KUBECTL) apply --server-side -f - + +.PHONY: uninstall +uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - + + +.PHONY: kubectl-kjob +kubectl-kjob: embeded-manifest ## Build kubectl-kjob binary file to bin directory. + CGO_ENABLED=$(CGO_ENABLED) $(GO_BUILD_ENV) $(GO_CMD) build -ldflags="$(LD_FLAGS)" -o bin/kubectl-kjob cmd/kjobctl/main.go + +.PHONY: kjobctl-docs +kjobctl-docs: ## Build kjobctl-docs binary file to bin directory. + $(GO_BUILD_ENV) $(GO_CMD) build -ldflags="$(LD_FLAGS)" -o $(PROJECT_DIR)/bin/kjobctl-docs ./cmd/kjobctl-docs/main.go + +.PHONY: generate-kjobctl-docs +generate-kjobctl-docs: kjobctl-docs ## Generate kjobctl docs. + rm -Rf $(PROJECT_DIR)/docs/commands/* + $(PROJECT_DIR)/bin/kjobctl-docs \ + $(PROJECT_DIR)/cmd/kjobctl-docs/templates \ + $(PROJECT_DIR)/docs/commands + +##@ Dependencies + +## Location to install dependencies to +LOCALBIN ?= $(shell pwd)/bin +$(LOCALBIN): + mkdir -p $(LOCALBIN) + +## Tool Versions + +# Use go.mod go version as source. +KUSTOMIZE_VERSION ?= $(shell cd $(TOOLS_DIR) && $(GO_CMD) list -m -f '{{.Version}}' sigs.k8s.io/kustomize/kustomize/v5) +CONTROLLER_GEN_VERSION ?= $(shell cd $(TOOLS_DIR) && $(GO_CMD) list -m -f '{{.Version}}' sigs.k8s.io/controller-tools) +ENVTEST_VERSION ?= $(shell cd $(TOOLS_DIR) && $(GO_CMD) list -m -f '{{.Version}}' sigs.k8s.io/controller-runtime/tools/setup-envtest) +GOLANGCI_LINT_VERSION ?= $(shell cd $(TOOLS_DIR) && $(GO_CMD) list -m -f '{{.Version}}' github.com/golangci/golangci-lint) +GOTESTSUM_VERSION ?= $(shell cd $(TOOLS_DIR) && $(GO_CMD) list -m -f '{{.Version}}' gotest.tools/gotestsum) +GINKGO_VERSION ?= $(shell cd $(TOOLS_DIR) && $(GO_CMD) list -m -f '{{.Version}}' github.com/onsi/ginkgo/v2) +KIND_VERSION ?= $(shell cd $(TOOLS_DIR); $(GO_CMD) list -m -f '{{.Version}}' sigs.k8s.io/kind) + +## Tool Binaries +KUBECTL ?= kubectl +KUSTOMIZE ?= $(LOCALBIN)/kustomize-$(KUSTOMIZE_VERSION) +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen-$(CONTROLLER_GEN_VERSION) +ENVTEST ?= $(LOCALBIN)/setup-envtest-$(ENVTEST_VERSION) +GOLANGCI_LINT ?= $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION) +GOTESTSUM ?= $(LOCALBIN)/gotestsum-$(GOTESTSUM_VERSION) +GINKGO ?= $(LOCALBIN)/ginkgo-$(GINKGO_VERSION) + +.PHONY: kustomize +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. +$(KUSTOMIZE): $(LOCALBIN) + $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) + +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. +$(CONTROLLER_GEN): $(LOCALBIN) + $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_GEN_VERSION)) + +.PHONY: envtest +envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. +$(ENVTEST): $(LOCALBIN) + $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) + +.PHONY: golangci-lint +golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. +$(GOLANGCI_LINT): $(LOCALBIN) + $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) + +.PHONY: gotestsum +gotestsum: ## Download gotestsum locally if necessary. + $(call go-install-tool,$(GOTESTSUM),gotest.tools/gotestsum,$(GOTESTSUM_VERSION)) + +.PHONY: ginkgo +ginkgo: ## Download ginkgo locally if necessary. + $(call go-install-tool,$(GINKGO),github.com/onsi/ginkgo/v2/ginkgo,$(GINKGO_VERSION)) + +KIND = $(PROJECT_DIR)/bin/kind +.PHONY: kind +kind: ## Download kind locally if necessary. + @GOBIN=$(PROJECT_DIR)/bin GO111MODULE=on $(GO_CMD) install sigs.k8s.io/kind@$(KIND_VERSION) + +##@ External CRDs + +RAY_ROOT = $(shell $(GO_CMD) list -m -mod=readonly -f "{{.Dir}}" github.com/ray-project/kuberay/ray-operator) +.PHONY: ray-operator-crd +ray-operator-crd: ## Copy the CRDs from the ray-operator to the dep-crds directory. + mkdir -p $(EXTERNAL_CRDS_DIR)/ray-operator/ + cp -f $(RAY_ROOT)/config/crd/bases/* $(EXTERNAL_CRDS_DIR)/ray-operator/ + +# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist +# $1 - target path with name of binary (ideally with version) +# $2 - package url which can be installed +# $3 - specific version of package +define go-install-tool +@[ -f $(1) ] || { \ +set -e; \ +package=$(2)@$(3) ;\ +echo "Downloading $${package}" ;\ +GOBIN=$(LOCALBIN) go install $${package} ;\ +mv "$$(echo "$(1)" | sed "s/-$(3)$$//")" $(1) ;\ +} ;\ +ln -sf $(1) "$$(echo "$(1)" | sed "s/-$(3)$$//")" +endef diff --git a/README.md b/README.md index c9d9061..6dfbfd5 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,91 @@ KJob: Tool for CLI-loving ML researchers +## Description +The `kubectl-kjob` plugin, `kjobctl`, allows you to list, describe and create jobs. + +Read the [overview](docs/_index.md) to learn more. + +## Getting Started + +### Prerequisites +- go version v1.23+ +- kubectl version v1.27+. +- Access to a Kubernetes v1.27+ cluster. + +### To Install + +**Install the CRDs into the cluster:** + +```sh +make install +``` + +**Install `kubectl kjob` plugin:** + +```sh +make kubectl-kjob +sudo cp ./bin/kubectl-kjob /usr/local/bin/kubectl-kjob +``` + +**Additionally, you can create an alias `kjobctl` to allow shorter syntax:** + +```sh +echo 'alias kjobctl="kubectl kjob"' >> ~/.bashrc +# Or if you are using ZSH +echo 'alias kjobctl="kubectl kjob"' >> ~/.zshrc +``` + +**Autocompletion:** + +```bash +echo '[[ $commands[kubectl-kjob] ]] && source <(kubectl-kjob completion bash)' >> ~/.bashrc +# Or if you are using ZSH +echo '[[ $commands[kubectl-kjob] ]] && source <(kubectl-kjob completion zsh)' >> ~/.zshrc + +cat <kubectl_complete-kjob +#!/usr/bin/env sh + +# Call the __complete command passing it all arguments +kubectl kjob __complete "\$@" +EOF + +chmod u+x kubectl_complete-kjob +sudo mv kubectl_complete-kjob /usr/local/bin/kubectl_complete-kjob +``` + +### To Uninstall + +**Delete the APIs(CRDs) from the cluster:** + +```sh +make uninstall +``` + +**Delete `kubectl kjob` plugin:** + +```sh +sudo rm /usr/local/bin/kubectl-kjob +``` + +**NOTE:** Run `make help` for more information on all potential `make` targets + +## License + +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + ## Community, discussion, contribution, and support Learn how to engage with the Kubernetes community on the [community page](http://kubernetes.io/community/). diff --git a/apis/v1alpha1/application_profile_types.go b/apis/v1alpha1/application_profile_types.go new file mode 100644 index 0000000..d8397e9 --- /dev/null +++ b/apis/v1alpha1/application_profile_types.go @@ -0,0 +1,172 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// VolumeBundleReference is the name of the VolumeBundle. +// +// +kubebuilder:validation:MaxLength=253 +// +kubebuilder:validation:Pattern="^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" +type VolumeBundleReference string + +type ApplicationProfileMode string + +const ( + InteractiveMode ApplicationProfileMode = "Interactive" + JobMode ApplicationProfileMode = "Job" + RayJobMode ApplicationProfileMode = "RayJob" + RayClusterMode ApplicationProfileMode = "RayCluster" + SlurmMode ApplicationProfileMode = "Slurm" +) + +// +kubebuilder:validation:Enum=cmd;parallelism;completions;replicas;min-replicas;max-replicas;request;localqueue;raycluster;array;cpus-per-task;error;gpus-per-task;input;job-name;mem-per-cpu;mem-per-gpu;mem-per-task;nodes;ntasks;output;partition +type Flag string + +const ( + CmdFlag Flag = "cmd" + ParallelismFlag Flag = "parallelism" + CompletionsFlag Flag = "completions" + ReplicasFlag Flag = "replicas" + MinReplicasFlag Flag = "min-replicas" + MaxReplicasFlag Flag = "max-replicas" + RequestFlag Flag = "request" + LocalQueueFlag Flag = "localqueue" + RayClusterFlag Flag = "raycluster" + ArrayFlag Flag = "array" + CpusPerTaskFlag Flag = "cpus-per-task" + ErrorFlag Flag = "error" + GpusPerTaskFlag Flag = "gpus-per-task" + InputFlag Flag = "input" + JobNameFlag Flag = "job-name" + MemPerNodeFlag Flag = "mem" + MemPerCPUFlag Flag = "mem-per-cpu" + MemPerGPUFlag Flag = "mem-per-gpu" + MemPerTaskFlag Flag = "mem-per-task" + NodesFlag Flag = "nodes" + NTasksFlag Flag = "ntasks" + OutputFlag Flag = "output" + PartitionFlag Flag = "partition" + PriorityFlag Flag = "priority" + TimeFlag Flag = "time" +) + +// TemplateReference is the name of the template. +// +// +kubebuilder:validation:MaxLength=253 +// +kubebuilder:validation:Pattern="^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" +type TemplateReference string + +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('replicas' in self.requiredFlags) || self.name in ['RayJob', 'RayCluster']", message="replicas flag can be used only on RayJob and RayCluster modes" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('min-replicas' in self.requiredFlags) || self.name in ['RayJob', 'RayCluster']", message="min-replicas flag can be used only on RayJob and RayCluster modes" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('max-replicas' in self.requiredFlags) || self.name in ['RayJob', 'RayCluster']", message="max-replicas flag can be used only on RayJob and RayCluster modes" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('request' in self.requiredFlags) || self.name in ['Job', 'Interactive', 'RayJob']", message="request flag can be used only on Job and Interactive modes" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('cmd' in self.requiredFlags) || self.name in ['Job', 'Interactive', 'RayJob']", message="cmd flag can be used only on Job, Interactive and RayJob modes" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('raycluster' in self.requiredFlags) || self.name == 'RayJob'", message="raycluster flag can be used only on RayJob mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('raycluster' in self.requiredFlags) || !('localqueue' in self.requiredFlags || 'replicas' in self.requiredFlags || 'min-replicas' in self.requiredFlags || 'max-replicas' in self.requiredFlags)", message="if raycluster flag are set none of localqueue, replicas, min-replicas and max-replicas can be" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('array' in self.requiredFlags) || self.name == 'Slurm'", message="array flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('cpus-per-task' in self.requiredFlags) || self.name == 'Slurm'", message="cpus-per-task flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('error' in self.requiredFlags) || self.name == 'Slurm'", message="error flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('gpus-per-task' in self.requiredFlags) || self.name == 'Slurm'", message="gpus-per-task flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('input' in self.requiredFlags) || self.name == 'Slurm'", message="input flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('job-name' in self.requiredFlags) || self.name == 'Slurm'", message="job-name flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('mem' in self.requiredFlags) || self.name == 'Slurm'", message="mem flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('mem-per-cpu' in self.requiredFlags) || self.name == 'Slurm'", message="mem-per-cpu flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('mem-per-gpu' in self.requiredFlags) || self.name == 'Slurm'", message="mem-per-gpu flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('mem-per-task' in self.requiredFlags) || self.name == 'Slurm'", message="mem-per-task flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('nodes' in self.requiredFlags) || self.name == 'Slurm'", message="nodes flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('ntasks' in self.requiredFlags) || self.name == 'Slurm'", message="ntasks flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('output' in self.requiredFlags) || self.name == 'Slurm'", message="output flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('partition' in self.requiredFlags) || self.name == 'Slurm'", message="partition flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || self.name != 'Slurm' || !('parallelism' in self.requiredFlags)", message="parallelism flag can't be used on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || self.name != 'Slurm' || !('completions' in self.requiredFlags)", message="completions flag can't be used on Slurm mode" +type SupportedMode struct { + // name determines which template will be used and which object will eventually be created. + // Possible values are Interactive, Job, RayJob, RayCluster and Slurm. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:Enum=Interactive;Job;RayJob;RayCluster;Slurm + Name ApplicationProfileMode `json:"name"` + + // template is the name of the template. + // Template type depends on ApplicationProfileMode: + // - on Interactive mode it must be v1/PodTemplate + // - on Job mode it must be kjobctl.x-k8s.io/v1alpha1/JobTemplate + // - on RayJob mode it must be kjobctl.x-k8s.io/v1alpha1/RayJobTemplate + // - on RayCluster mode it must be kjobctl.x-k8s.io/v1alpha1/RayClusterTemplate + // - on Slurm mode it must be kjobctl.x-k8s.io/v1alpha1/JobTemplate + // + // +kubebuilder:validation:Required + Template TemplateReference `json:"template"` + + // requiredFlags point which cli flags are required to be passed in order to fill the gaps in the templates. + // Possible values are cmd, parallelism, completions, replicas, min-replicas, max-replicas, request, localqueue, and raycluster. + // replicas, min-replicas, and max-replicas flags used only for RayJob and RayCluster mode. + // The raycluster flag used only for the RayJob mode. + // The request flag used only for Interactive and Job modes. + // The cmd flag used only for Interactive, Job, and RayJob. + // The time and priority flags can be used in all modes. + // If the raycluster flag are set, none of localqueue, replicas, min-replicas, or max-replicas can be set. + // For the Slurm mode, the possible values are: array, cpus-per-task, error, gpus-per-task, input, job-name, mem, mem-per-cpu, + // mem-per-gpu, mem-per-task, nodes, ntasks, output, partition, localqueue. + // + // cmd and requests values are going to be added only to the first primary container. + // + // +optional + // +listType=set + // +kubebuilder:validation:MaxItems=14 + RequiredFlags []Flag `json:"requiredFlags,omitempty"` +} + +// ApplicationProfileSpec defines the desired state of ApplicationProfile +type ApplicationProfileSpec struct { + // +listType=map + // +listMapKey=name + // +kubebuilder:validation:Required + SupportedModes []SupportedMode `json:"supportedModes"` + + // +optional + // +listType=set + VolumeBundles []VolumeBundleReference `json:"volumeBundles,omitempty"` +} + +// +genclient +// +kubebuilder:object:root=true + +// ApplicationProfile is the Schema for the applicationprofiles API +type ApplicationProfile struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ApplicationProfileSpec `json:"spec,omitempty"` +} + +// +kubebuilder:object:root=true + +// ApplicationProfileList contains a list of ApplicationProfile +type ApplicationProfileList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []ApplicationProfile `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ApplicationProfile{}, &ApplicationProfileList{}) +} diff --git a/apis/v1alpha1/doc.go b/apis/v1alpha1/doc.go new file mode 100644 index 0000000..75ca753 --- /dev/null +++ b/apis/v1alpha1/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +kubebuilder:object:generate=true +// +groupName=kjobctl.x-k8s.io + +package v1alpha1 diff --git a/apis/v1alpha1/groupversion_info.go b/apis/v1alpha1/groupversion_info.go new file mode 100644 index 0000000..2b59c4f --- /dev/null +++ b/apis/v1alpha1/groupversion_info.go @@ -0,0 +1,39 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the kjobctl.x-k8s.io v1alpha1 API group +// +kubebuilder:object:generate=true +// +groupName=kjobctl.x-k8s.io +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "kjobctl.x-k8s.io", Version: "v1alpha1"} + + // SchemeGroupVersion is alias to GroupVersion for client-go libraries. + SchemeGroupVersion = GroupVersion + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/apis/v1alpha1/job_template_types.go b/apis/v1alpha1/job_template_types.go new file mode 100644 index 0000000..0340cd1 --- /dev/null +++ b/apis/v1alpha1/job_template_types.go @@ -0,0 +1,63 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + batchv1 "k8s.io/api/batch/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// JobTemplateSpec describes the data a job should have when created from a template +type JobTemplateSpec struct { + // Standard object's metadata. + // + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Specification of the desired behavior of the job. + // + // +kubebuilder:validation:Required + Spec batchv1.JobSpec `json:"spec"` +} + +// +genclient +// +kubebuilder:object:root=true + +// JobTemplate is the Schema for the jobtemplate API +type JobTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Template defines the jobs that will be created from this pod template. + // + // +kubebuilder:validation:Required + Template JobTemplateSpec `json:"template,omitempty"` +} + +// +kubebuilder:object:root=true + +// JobTemplateList contains a list of JobTemplate +type JobTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []JobTemplate `json:"items"` +} + +func init() { + SchemeBuilder.Register(&JobTemplate{}, &JobTemplateList{}) +} diff --git a/apis/v1alpha1/ray_cluster_template_types.go b/apis/v1alpha1/ray_cluster_template_types.go new file mode 100644 index 0000000..c21c47c --- /dev/null +++ b/apis/v1alpha1/ray_cluster_template_types.go @@ -0,0 +1,63 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// RayClusterTemplateSpec describes the data a raycluster should have when created from a template +type RayClusterTemplateSpec struct { + // Standard object's metadata. + // + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Specification of the desired behavior of the raycluster. + // + // +kubebuilder:validation:Required + Spec rayv1.RayClusterSpec `json:"spec"` +} + +// +genclient +// +kubebuilder:object:root=true + +// RayClusterTemplate is the Schema for the rayclustertemplate API +type RayClusterTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Template defines rayclusters that will be created from this raycluster template. + // + // +kubebuilder:validation:Required + Template RayClusterTemplateSpec `json:"template,omitempty"` +} + +// +kubebuilder:object:root=true + +// RayClusterTemplateList contains a list of RayClusterTemplate +type RayClusterTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []RayClusterTemplate `json:"items"` +} + +func init() { + SchemeBuilder.Register(&RayClusterTemplate{}, &RayClusterTemplateList{}) +} diff --git a/apis/v1alpha1/ray_job_template_types.go b/apis/v1alpha1/ray_job_template_types.go new file mode 100644 index 0000000..f31f0d7 --- /dev/null +++ b/apis/v1alpha1/ray_job_template_types.go @@ -0,0 +1,63 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// RayJobTemplateSpec describes the data a rayjob should have when created from a template +type RayJobTemplateSpec struct { + // Standard object's metadata. + // + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Specification of the desired behavior of the rayjob. + // + // +kubebuilder:validation:Required + Spec rayv1.RayJobSpec `json:"spec"` +} + +// +genclient +// +kubebuilder:object:root=true + +// RayJobTemplate is the Schema for the rayjobtemplate API +type RayJobTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Template defines rayjobs that will be created from this rayjob template. + // + // +kubebuilder:validation:Required + Template RayJobTemplateSpec `json:"template,omitempty"` +} + +// +kubebuilder:object:root=true + +// RayJobTemplateList contains a list of RayJobTemplate +type RayJobTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []RayJobTemplate `json:"items"` +} + +func init() { + SchemeBuilder.Register(&RayJobTemplate{}, &RayJobTemplateList{}) +} diff --git a/apis/v1alpha1/volumebundle_types.go b/apis/v1alpha1/volumebundle_types.go new file mode 100644 index 0000000..87c2a5f --- /dev/null +++ b/apis/v1alpha1/volumebundle_types.go @@ -0,0 +1,68 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// VolumeBundleSpec defines the desired state of VolumeBundle +type VolumeBundleSpec struct { + // volumes is a set of volumes that will be added to all pods of the job. + // +listType=map + // +listMapKey=name + // +kubebuilder:validation:MinItems=1 + Volumes []corev1.Volume `json:"volumes"` + + // containerVolumeMounts is a list of locations in each container of a pod where the volumes will be mounted. + // +listType=map + // +listMapKey=mountPath + // +kubebuilder:validation:MinItems=1 + ContainerVolumeMounts []corev1.VolumeMount `json:"containerVolumeMounts"` + + // envVars are environment variables that refer to absolute paths in the container filesystem. + // These key/value pairs will be available in containers as environment variables. + // +optional + // +listType=map + // +listMapKey=name + EnvVars []corev1.EnvVar `json:"envVars,omitempty"` +} + +// +genclient +// +kubebuilder:object:root=true + +// VolumeBundle is the Schema for the volumebundles API +type VolumeBundle struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec VolumeBundleSpec `json:"spec,omitempty"` +} + +// +kubebuilder:object:root=true + +// VolumeBundleList contains a list of VolumeBundle +type VolumeBundleList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []VolumeBundle `json:"items"` +} + +func init() { + SchemeBuilder.Register(&VolumeBundle{}, &VolumeBundleList{}) +} diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000..148dc45 --- /dev/null +++ b/apis/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,450 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/api/core/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ApplicationProfile) DeepCopyInto(out *ApplicationProfile) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ApplicationProfile. +func (in *ApplicationProfile) DeepCopy() *ApplicationProfile { + if in == nil { + return nil + } + out := new(ApplicationProfile) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ApplicationProfile) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ApplicationProfileList) DeepCopyInto(out *ApplicationProfileList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ApplicationProfile, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ApplicationProfileList. +func (in *ApplicationProfileList) DeepCopy() *ApplicationProfileList { + if in == nil { + return nil + } + out := new(ApplicationProfileList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ApplicationProfileList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ApplicationProfileSpec) DeepCopyInto(out *ApplicationProfileSpec) { + *out = *in + if in.SupportedModes != nil { + in, out := &in.SupportedModes, &out.SupportedModes + *out = make([]SupportedMode, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.VolumeBundles != nil { + in, out := &in.VolumeBundles, &out.VolumeBundles + *out = make([]VolumeBundleReference, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ApplicationProfileSpec. +func (in *ApplicationProfileSpec) DeepCopy() *ApplicationProfileSpec { + if in == nil { + return nil + } + out := new(ApplicationProfileSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *JobTemplate) DeepCopyInto(out *JobTemplate) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobTemplate. +func (in *JobTemplate) DeepCopy() *JobTemplate { + if in == nil { + return nil + } + out := new(JobTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *JobTemplate) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *JobTemplateList) DeepCopyInto(out *JobTemplateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]JobTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobTemplateList. +func (in *JobTemplateList) DeepCopy() *JobTemplateList { + if in == nil { + return nil + } + out := new(JobTemplateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *JobTemplateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *JobTemplateSpec) DeepCopyInto(out *JobTemplateSpec) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobTemplateSpec. +func (in *JobTemplateSpec) DeepCopy() *JobTemplateSpec { + if in == nil { + return nil + } + out := new(JobTemplateSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayClusterTemplate) DeepCopyInto(out *RayClusterTemplate) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayClusterTemplate. +func (in *RayClusterTemplate) DeepCopy() *RayClusterTemplate { + if in == nil { + return nil + } + out := new(RayClusterTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RayClusterTemplate) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayClusterTemplateList) DeepCopyInto(out *RayClusterTemplateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]RayClusterTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayClusterTemplateList. +func (in *RayClusterTemplateList) DeepCopy() *RayClusterTemplateList { + if in == nil { + return nil + } + out := new(RayClusterTemplateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RayClusterTemplateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayClusterTemplateSpec) DeepCopyInto(out *RayClusterTemplateSpec) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayClusterTemplateSpec. +func (in *RayClusterTemplateSpec) DeepCopy() *RayClusterTemplateSpec { + if in == nil { + return nil + } + out := new(RayClusterTemplateSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayJobTemplate) DeepCopyInto(out *RayJobTemplate) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayJobTemplate. +func (in *RayJobTemplate) DeepCopy() *RayJobTemplate { + if in == nil { + return nil + } + out := new(RayJobTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RayJobTemplate) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayJobTemplateList) DeepCopyInto(out *RayJobTemplateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]RayJobTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayJobTemplateList. +func (in *RayJobTemplateList) DeepCopy() *RayJobTemplateList { + if in == nil { + return nil + } + out := new(RayJobTemplateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RayJobTemplateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayJobTemplateSpec) DeepCopyInto(out *RayJobTemplateSpec) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayJobTemplateSpec. +func (in *RayJobTemplateSpec) DeepCopy() *RayJobTemplateSpec { + if in == nil { + return nil + } + out := new(RayJobTemplateSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SupportedMode) DeepCopyInto(out *SupportedMode) { + *out = *in + if in.RequiredFlags != nil { + in, out := &in.RequiredFlags, &out.RequiredFlags + *out = make([]Flag, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SupportedMode. +func (in *SupportedMode) DeepCopy() *SupportedMode { + if in == nil { + return nil + } + out := new(SupportedMode) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeBundle) DeepCopyInto(out *VolumeBundle) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeBundle. +func (in *VolumeBundle) DeepCopy() *VolumeBundle { + if in == nil { + return nil + } + out := new(VolumeBundle) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VolumeBundle) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeBundleList) DeepCopyInto(out *VolumeBundleList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VolumeBundle, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeBundleList. +func (in *VolumeBundleList) DeepCopy() *VolumeBundleList { + if in == nil { + return nil + } + out := new(VolumeBundleList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VolumeBundleList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeBundleSpec) DeepCopyInto(out *VolumeBundleSpec) { + *out = *in + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make([]v1.Volume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ContainerVolumeMounts != nil { + in, out := &in.ContainerVolumeMounts, &out.ContainerVolumeMounts + *out = make([]v1.VolumeMount, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.EnvVars != nil { + in, out := &in.EnvVars, &out.EnvVars + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeBundleSpec. +func (in *VolumeBundleSpec) DeepCopy() *VolumeBundleSpec { + if in == nil { + return nil + } + out := new(VolumeBundleSpec) + in.DeepCopyInto(out) + return out +} diff --git a/client-go/clientset/versioned/clientset.go b/client-go/clientset/versioned/clientset.go new file mode 100644 index 0000000..ba83e32 --- /dev/null +++ b/client-go/clientset/versioned/clientset.go @@ -0,0 +1,119 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package versioned + +import ( + "fmt" + "net/http" + + discovery "k8s.io/client-go/discovery" + rest "k8s.io/client-go/rest" + flowcontrol "k8s.io/client-go/util/flowcontrol" + kjobctlv1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/typed/apis/v1alpha1" +) + +type Interface interface { + Discovery() discovery.DiscoveryInterface + KjobctlV1alpha1() kjobctlv1alpha1.KjobctlV1alpha1Interface +} + +// Clientset contains the clients for groups. +type Clientset struct { + *discovery.DiscoveryClient + kjobctlV1alpha1 *kjobctlv1alpha1.KjobctlV1alpha1Client +} + +// KjobctlV1alpha1 retrieves the KjobctlV1alpha1Client +func (c *Clientset) KjobctlV1alpha1() kjobctlv1alpha1.KjobctlV1alpha1Interface { + return c.kjobctlV1alpha1 +} + +// Discovery retrieves the DiscoveryClient +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + if c == nil { + return nil + } + return c.DiscoveryClient +} + +// NewForConfig creates a new Clientset for the given config. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfig will generate a rate-limiter in configShallowCopy. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*Clientset, error) { + configShallowCopy := *c + + if configShallowCopy.UserAgent == "" { + configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent() + } + + // share the transport between all clients + httpClient, err := rest.HTTPClientFor(&configShallowCopy) + if err != nil { + return nil, err + } + + return NewForConfigAndClient(&configShallowCopy, httpClient) +} + +// NewForConfigAndClient creates a new Clientset for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfigAndClient will generate a rate-limiter in configShallowCopy. +func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, error) { + configShallowCopy := *c + if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { + if configShallowCopy.Burst <= 0 { + return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") + } + configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) + } + + var cs Clientset + var err error + cs.kjobctlV1alpha1, err = kjobctlv1alpha1.NewForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + + cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + return &cs, nil +} + +// NewForConfigOrDie creates a new Clientset for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *Clientset { + cs, err := NewForConfig(c) + if err != nil { + panic(err) + } + return cs +} + +// New creates a new Clientset for the given RESTClient. +func New(c rest.Interface) *Clientset { + var cs Clientset + cs.kjobctlV1alpha1 = kjobctlv1alpha1.New(c) + + cs.DiscoveryClient = discovery.NewDiscoveryClient(c) + return &cs +} diff --git a/client-go/clientset/versioned/fake/clientset_generated.go b/client-go/clientset/versioned/fake/clientset_generated.go new file mode 100644 index 0000000..0f9beb6 --- /dev/null +++ b/client-go/clientset/versioned/fake/clientset_generated.go @@ -0,0 +1,88 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/discovery" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/testing" + clientset "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned" + kjobctlv1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/typed/apis/v1alpha1" + fakekjobctlv1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/typed/apis/v1alpha1/fake" +) + +// NewSimpleClientset returns a clientset that will respond with the provided objects. +// It's backed by a very simple object tracker that processes creates, updates and deletions as-is, +// without applying any field management, validations and/or defaults. It shouldn't be considered a replacement +// for a real clientset and is mostly useful in simple unit tests. +// +// DEPRECATED: NewClientset replaces this with support for field management, which significantly improves +// server side apply testing. NewClientset is only available when apply configurations are generated (e.g. +// via --with-applyconfig). +func NewSimpleClientset(objects ...runtime.Object) *Clientset { + o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) + for _, obj := range objects { + if err := o.Add(obj); err != nil { + panic(err) + } + } + + cs := &Clientset{tracker: o} + cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} + cs.AddReactor("*", "*", testing.ObjectReaction(o)) + cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := o.Watch(gvr, ns) + if err != nil { + return false, nil, err + } + return true, watch, nil + }) + + return cs +} + +// Clientset implements clientset.Interface. Meant to be embedded into a +// struct to get a default implementation. This makes faking out just the method +// you want to test easier. +type Clientset struct { + testing.Fake + discovery *fakediscovery.FakeDiscovery + tracker testing.ObjectTracker +} + +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + return c.discovery +} + +func (c *Clientset) Tracker() testing.ObjectTracker { + return c.tracker +} + +var ( + _ clientset.Interface = &Clientset{} + _ testing.FakeClient = &Clientset{} +) + +// KjobctlV1alpha1 retrieves the KjobctlV1alpha1Client +func (c *Clientset) KjobctlV1alpha1() kjobctlv1alpha1.KjobctlV1alpha1Interface { + return &fakekjobctlv1alpha1.FakeKjobctlV1alpha1{Fake: &c.Fake} +} diff --git a/client-go/clientset/versioned/fake/doc.go b/client-go/clientset/versioned/fake/doc.go new file mode 100644 index 0000000..634bd02 --- /dev/null +++ b/client-go/clientset/versioned/fake/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated fake clientset. +package fake diff --git a/client-go/clientset/versioned/fake/register.go b/client-go/clientset/versioned/fake/register.go new file mode 100644 index 0000000..a4e627a --- /dev/null +++ b/client-go/clientset/versioned/fake/register.go @@ -0,0 +1,55 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + kjobctlv1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +var scheme = runtime.NewScheme() +var codecs = serializer.NewCodecFactory(scheme) + +var localSchemeBuilder = runtime.SchemeBuilder{ + kjobctlv1alpha1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(scheme)) +} diff --git a/client-go/clientset/versioned/scheme/doc.go b/client-go/clientset/versioned/scheme/doc.go new file mode 100644 index 0000000..40e42c2 --- /dev/null +++ b/client-go/clientset/versioned/scheme/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package contains the scheme of the automatically generated clientset. +package scheme diff --git a/client-go/clientset/versioned/scheme/register.go b/client-go/clientset/versioned/scheme/register.go new file mode 100644 index 0000000..c335f95 --- /dev/null +++ b/client-go/clientset/versioned/scheme/register.go @@ -0,0 +1,55 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package scheme + +import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + kjobctlv1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +var Scheme = runtime.NewScheme() +var Codecs = serializer.NewCodecFactory(Scheme) +var ParameterCodec = runtime.NewParameterCodec(Scheme) +var localSchemeBuilder = runtime.SchemeBuilder{ + kjobctlv1alpha1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(Scheme)) +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/apis_client.go b/client-go/clientset/versioned/typed/apis/v1alpha1/apis_client.go new file mode 100644 index 0000000..9204ced --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/apis_client.go @@ -0,0 +1,126 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "net/http" + + rest "k8s.io/client-go/rest" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/scheme" +) + +type KjobctlV1alpha1Interface interface { + RESTClient() rest.Interface + ApplicationProfilesGetter + JobTemplatesGetter + RayClusterTemplatesGetter + RayJobTemplatesGetter + VolumeBundlesGetter +} + +// KjobctlV1alpha1Client is used to interact with features provided by the kjobctl.x-k8s.io group. +type KjobctlV1alpha1Client struct { + restClient rest.Interface +} + +func (c *KjobctlV1alpha1Client) ApplicationProfiles(namespace string) ApplicationProfileInterface { + return newApplicationProfiles(c, namespace) +} + +func (c *KjobctlV1alpha1Client) JobTemplates(namespace string) JobTemplateInterface { + return newJobTemplates(c, namespace) +} + +func (c *KjobctlV1alpha1Client) RayClusterTemplates(namespace string) RayClusterTemplateInterface { + return newRayClusterTemplates(c, namespace) +} + +func (c *KjobctlV1alpha1Client) RayJobTemplates(namespace string) RayJobTemplateInterface { + return newRayJobTemplates(c, namespace) +} + +func (c *KjobctlV1alpha1Client) VolumeBundles(namespace string) VolumeBundleInterface { + return newVolumeBundles(c, namespace) +} + +// NewForConfig creates a new KjobctlV1alpha1Client for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*KjobctlV1alpha1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + httpClient, err := rest.HTTPClientFor(&config) + if err != nil { + return nil, err + } + return NewForConfigAndClient(&config, httpClient) +} + +// NewForConfigAndClient creates a new KjobctlV1alpha1Client for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *rest.Config, h *http.Client) (*KjobctlV1alpha1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + client, err := rest.RESTClientForConfigAndClient(&config, h) + if err != nil { + return nil, err + } + return &KjobctlV1alpha1Client{client}, nil +} + +// NewForConfigOrDie creates a new KjobctlV1alpha1Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *KjobctlV1alpha1Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new KjobctlV1alpha1Client for the given RESTClient. +func New(c rest.Interface) *KjobctlV1alpha1Client { + return &KjobctlV1alpha1Client{c} +} + +func setConfigDefaults(config *rest.Config) error { + gv := v1alpha1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/apis" + config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + + return nil +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *KjobctlV1alpha1Client) RESTClient() rest.Interface { + if c == nil { + return nil + } + return c.restClient +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/applicationprofile.go b/client-go/clientset/versioned/typed/apis/v1alpha1/applicationprofile.go new file mode 100644 index 0000000..e57894b --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/applicationprofile.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + scheme "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/scheme" +) + +// ApplicationProfilesGetter has a method to return a ApplicationProfileInterface. +// A group's client should implement this interface. +type ApplicationProfilesGetter interface { + ApplicationProfiles(namespace string) ApplicationProfileInterface +} + +// ApplicationProfileInterface has methods to work with ApplicationProfile resources. +type ApplicationProfileInterface interface { + Create(ctx context.Context, applicationProfile *v1alpha1.ApplicationProfile, opts v1.CreateOptions) (*v1alpha1.ApplicationProfile, error) + Update(ctx context.Context, applicationProfile *v1alpha1.ApplicationProfile, opts v1.UpdateOptions) (*v1alpha1.ApplicationProfile, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.ApplicationProfile, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.ApplicationProfileList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.ApplicationProfile, err error) + ApplicationProfileExpansion +} + +// applicationProfiles implements ApplicationProfileInterface +type applicationProfiles struct { + *gentype.ClientWithList[*v1alpha1.ApplicationProfile, *v1alpha1.ApplicationProfileList] +} + +// newApplicationProfiles returns a ApplicationProfiles +func newApplicationProfiles(c *KjobctlV1alpha1Client, namespace string) *applicationProfiles { + return &applicationProfiles{ + gentype.NewClientWithList[*v1alpha1.ApplicationProfile, *v1alpha1.ApplicationProfileList]( + "applicationprofiles", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *v1alpha1.ApplicationProfile { return &v1alpha1.ApplicationProfile{} }, + func() *v1alpha1.ApplicationProfileList { return &v1alpha1.ApplicationProfileList{} }), + } +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/doc.go b/client-go/clientset/versioned/typed/apis/v1alpha1/doc.go new file mode 100644 index 0000000..28991e2 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated typed clients. +package v1alpha1 diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/fake/doc.go b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/doc.go new file mode 100644 index 0000000..fbfccbb --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_apis_client.go b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_apis_client.go new file mode 100644 index 0000000..d5bd37c --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_apis_client.go @@ -0,0 +1,55 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/typed/apis/v1alpha1" +) + +type FakeKjobctlV1alpha1 struct { + *testing.Fake +} + +func (c *FakeKjobctlV1alpha1) ApplicationProfiles(namespace string) v1alpha1.ApplicationProfileInterface { + return &FakeApplicationProfiles{c, namespace} +} + +func (c *FakeKjobctlV1alpha1) JobTemplates(namespace string) v1alpha1.JobTemplateInterface { + return &FakeJobTemplates{c, namespace} +} + +func (c *FakeKjobctlV1alpha1) RayClusterTemplates(namespace string) v1alpha1.RayClusterTemplateInterface { + return &FakeRayClusterTemplates{c, namespace} +} + +func (c *FakeKjobctlV1alpha1) RayJobTemplates(namespace string) v1alpha1.RayJobTemplateInterface { + return &FakeRayJobTemplates{c, namespace} +} + +func (c *FakeKjobctlV1alpha1) VolumeBundles(namespace string) v1alpha1.VolumeBundleInterface { + return &FakeVolumeBundles{c, namespace} +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeKjobctlV1alpha1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_applicationprofile.go b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_applicationprofile.go new file mode 100644 index 0000000..80963e1 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_applicationprofile.go @@ -0,0 +1,133 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// FakeApplicationProfiles implements ApplicationProfileInterface +type FakeApplicationProfiles struct { + Fake *FakeKjobctlV1alpha1 + ns string +} + +var applicationprofilesResource = v1alpha1.SchemeGroupVersion.WithResource("applicationprofiles") + +var applicationprofilesKind = v1alpha1.SchemeGroupVersion.WithKind("ApplicationProfile") + +// Get takes name of the applicationProfile, and returns the corresponding applicationProfile object, and an error if there is any. +func (c *FakeApplicationProfiles) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.ApplicationProfile, err error) { + emptyResult := &v1alpha1.ApplicationProfile{} + obj, err := c.Fake. + Invokes(testing.NewGetActionWithOptions(applicationprofilesResource, c.ns, name, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.ApplicationProfile), err +} + +// List takes label and field selectors, and returns the list of ApplicationProfiles that match those selectors. +func (c *FakeApplicationProfiles) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.ApplicationProfileList, err error) { + emptyResult := &v1alpha1.ApplicationProfileList{} + obj, err := c.Fake. + Invokes(testing.NewListActionWithOptions(applicationprofilesResource, applicationprofilesKind, c.ns, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.ApplicationProfileList{ListMeta: obj.(*v1alpha1.ApplicationProfileList).ListMeta} + for _, item := range obj.(*v1alpha1.ApplicationProfileList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested applicationProfiles. +func (c *FakeApplicationProfiles) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchActionWithOptions(applicationprofilesResource, c.ns, opts)) + +} + +// Create takes the representation of a applicationProfile and creates it. Returns the server's representation of the applicationProfile, and an error, if there is any. +func (c *FakeApplicationProfiles) Create(ctx context.Context, applicationProfile *v1alpha1.ApplicationProfile, opts v1.CreateOptions) (result *v1alpha1.ApplicationProfile, err error) { + emptyResult := &v1alpha1.ApplicationProfile{} + obj, err := c.Fake. + Invokes(testing.NewCreateActionWithOptions(applicationprofilesResource, c.ns, applicationProfile, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.ApplicationProfile), err +} + +// Update takes the representation of a applicationProfile and updates it. Returns the server's representation of the applicationProfile, and an error, if there is any. +func (c *FakeApplicationProfiles) Update(ctx context.Context, applicationProfile *v1alpha1.ApplicationProfile, opts v1.UpdateOptions) (result *v1alpha1.ApplicationProfile, err error) { + emptyResult := &v1alpha1.ApplicationProfile{} + obj, err := c.Fake. + Invokes(testing.NewUpdateActionWithOptions(applicationprofilesResource, c.ns, applicationProfile, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.ApplicationProfile), err +} + +// Delete takes name of the applicationProfile and deletes it. Returns an error if one occurs. +func (c *FakeApplicationProfiles) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(applicationprofilesResource, c.ns, name, opts), &v1alpha1.ApplicationProfile{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeApplicationProfiles) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionActionWithOptions(applicationprofilesResource, c.ns, opts, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.ApplicationProfileList{}) + return err +} + +// Patch applies the patch and returns the patched applicationProfile. +func (c *FakeApplicationProfiles) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.ApplicationProfile, err error) { + emptyResult := &v1alpha1.ApplicationProfile{} + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceActionWithOptions(applicationprofilesResource, c.ns, name, pt, data, opts, subresources...), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.ApplicationProfile), err +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_jobtemplate.go b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_jobtemplate.go new file mode 100644 index 0000000..456d176 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_jobtemplate.go @@ -0,0 +1,133 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// FakeJobTemplates implements JobTemplateInterface +type FakeJobTemplates struct { + Fake *FakeKjobctlV1alpha1 + ns string +} + +var jobtemplatesResource = v1alpha1.SchemeGroupVersion.WithResource("jobtemplates") + +var jobtemplatesKind = v1alpha1.SchemeGroupVersion.WithKind("JobTemplate") + +// Get takes name of the jobTemplate, and returns the corresponding jobTemplate object, and an error if there is any. +func (c *FakeJobTemplates) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.JobTemplate, err error) { + emptyResult := &v1alpha1.JobTemplate{} + obj, err := c.Fake. + Invokes(testing.NewGetActionWithOptions(jobtemplatesResource, c.ns, name, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.JobTemplate), err +} + +// List takes label and field selectors, and returns the list of JobTemplates that match those selectors. +func (c *FakeJobTemplates) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.JobTemplateList, err error) { + emptyResult := &v1alpha1.JobTemplateList{} + obj, err := c.Fake. + Invokes(testing.NewListActionWithOptions(jobtemplatesResource, jobtemplatesKind, c.ns, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.JobTemplateList{ListMeta: obj.(*v1alpha1.JobTemplateList).ListMeta} + for _, item := range obj.(*v1alpha1.JobTemplateList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested jobTemplates. +func (c *FakeJobTemplates) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchActionWithOptions(jobtemplatesResource, c.ns, opts)) + +} + +// Create takes the representation of a jobTemplate and creates it. Returns the server's representation of the jobTemplate, and an error, if there is any. +func (c *FakeJobTemplates) Create(ctx context.Context, jobTemplate *v1alpha1.JobTemplate, opts v1.CreateOptions) (result *v1alpha1.JobTemplate, err error) { + emptyResult := &v1alpha1.JobTemplate{} + obj, err := c.Fake. + Invokes(testing.NewCreateActionWithOptions(jobtemplatesResource, c.ns, jobTemplate, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.JobTemplate), err +} + +// Update takes the representation of a jobTemplate and updates it. Returns the server's representation of the jobTemplate, and an error, if there is any. +func (c *FakeJobTemplates) Update(ctx context.Context, jobTemplate *v1alpha1.JobTemplate, opts v1.UpdateOptions) (result *v1alpha1.JobTemplate, err error) { + emptyResult := &v1alpha1.JobTemplate{} + obj, err := c.Fake. + Invokes(testing.NewUpdateActionWithOptions(jobtemplatesResource, c.ns, jobTemplate, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.JobTemplate), err +} + +// Delete takes name of the jobTemplate and deletes it. Returns an error if one occurs. +func (c *FakeJobTemplates) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(jobtemplatesResource, c.ns, name, opts), &v1alpha1.JobTemplate{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeJobTemplates) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionActionWithOptions(jobtemplatesResource, c.ns, opts, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.JobTemplateList{}) + return err +} + +// Patch applies the patch and returns the patched jobTemplate. +func (c *FakeJobTemplates) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.JobTemplate, err error) { + emptyResult := &v1alpha1.JobTemplate{} + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceActionWithOptions(jobtemplatesResource, c.ns, name, pt, data, opts, subresources...), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.JobTemplate), err +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_rayclustertemplate.go b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_rayclustertemplate.go new file mode 100644 index 0000000..4e0cb86 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_rayclustertemplate.go @@ -0,0 +1,133 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// FakeRayClusterTemplates implements RayClusterTemplateInterface +type FakeRayClusterTemplates struct { + Fake *FakeKjobctlV1alpha1 + ns string +} + +var rayclustertemplatesResource = v1alpha1.SchemeGroupVersion.WithResource("rayclustertemplates") + +var rayclustertemplatesKind = v1alpha1.SchemeGroupVersion.WithKind("RayClusterTemplate") + +// Get takes name of the rayClusterTemplate, and returns the corresponding rayClusterTemplate object, and an error if there is any. +func (c *FakeRayClusterTemplates) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.RayClusterTemplate, err error) { + emptyResult := &v1alpha1.RayClusterTemplate{} + obj, err := c.Fake. + Invokes(testing.NewGetActionWithOptions(rayclustertemplatesResource, c.ns, name, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.RayClusterTemplate), err +} + +// List takes label and field selectors, and returns the list of RayClusterTemplates that match those selectors. +func (c *FakeRayClusterTemplates) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.RayClusterTemplateList, err error) { + emptyResult := &v1alpha1.RayClusterTemplateList{} + obj, err := c.Fake. + Invokes(testing.NewListActionWithOptions(rayclustertemplatesResource, rayclustertemplatesKind, c.ns, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.RayClusterTemplateList{ListMeta: obj.(*v1alpha1.RayClusterTemplateList).ListMeta} + for _, item := range obj.(*v1alpha1.RayClusterTemplateList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested rayClusterTemplates. +func (c *FakeRayClusterTemplates) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchActionWithOptions(rayclustertemplatesResource, c.ns, opts)) + +} + +// Create takes the representation of a rayClusterTemplate and creates it. Returns the server's representation of the rayClusterTemplate, and an error, if there is any. +func (c *FakeRayClusterTemplates) Create(ctx context.Context, rayClusterTemplate *v1alpha1.RayClusterTemplate, opts v1.CreateOptions) (result *v1alpha1.RayClusterTemplate, err error) { + emptyResult := &v1alpha1.RayClusterTemplate{} + obj, err := c.Fake. + Invokes(testing.NewCreateActionWithOptions(rayclustertemplatesResource, c.ns, rayClusterTemplate, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.RayClusterTemplate), err +} + +// Update takes the representation of a rayClusterTemplate and updates it. Returns the server's representation of the rayClusterTemplate, and an error, if there is any. +func (c *FakeRayClusterTemplates) Update(ctx context.Context, rayClusterTemplate *v1alpha1.RayClusterTemplate, opts v1.UpdateOptions) (result *v1alpha1.RayClusterTemplate, err error) { + emptyResult := &v1alpha1.RayClusterTemplate{} + obj, err := c.Fake. + Invokes(testing.NewUpdateActionWithOptions(rayclustertemplatesResource, c.ns, rayClusterTemplate, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.RayClusterTemplate), err +} + +// Delete takes name of the rayClusterTemplate and deletes it. Returns an error if one occurs. +func (c *FakeRayClusterTemplates) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(rayclustertemplatesResource, c.ns, name, opts), &v1alpha1.RayClusterTemplate{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeRayClusterTemplates) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionActionWithOptions(rayclustertemplatesResource, c.ns, opts, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.RayClusterTemplateList{}) + return err +} + +// Patch applies the patch and returns the patched rayClusterTemplate. +func (c *FakeRayClusterTemplates) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.RayClusterTemplate, err error) { + emptyResult := &v1alpha1.RayClusterTemplate{} + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceActionWithOptions(rayclustertemplatesResource, c.ns, name, pt, data, opts, subresources...), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.RayClusterTemplate), err +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_rayjobtemplate.go b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_rayjobtemplate.go new file mode 100644 index 0000000..2c9ea71 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_rayjobtemplate.go @@ -0,0 +1,133 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// FakeRayJobTemplates implements RayJobTemplateInterface +type FakeRayJobTemplates struct { + Fake *FakeKjobctlV1alpha1 + ns string +} + +var rayjobtemplatesResource = v1alpha1.SchemeGroupVersion.WithResource("rayjobtemplates") + +var rayjobtemplatesKind = v1alpha1.SchemeGroupVersion.WithKind("RayJobTemplate") + +// Get takes name of the rayJobTemplate, and returns the corresponding rayJobTemplate object, and an error if there is any. +func (c *FakeRayJobTemplates) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.RayJobTemplate, err error) { + emptyResult := &v1alpha1.RayJobTemplate{} + obj, err := c.Fake. + Invokes(testing.NewGetActionWithOptions(rayjobtemplatesResource, c.ns, name, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.RayJobTemplate), err +} + +// List takes label and field selectors, and returns the list of RayJobTemplates that match those selectors. +func (c *FakeRayJobTemplates) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.RayJobTemplateList, err error) { + emptyResult := &v1alpha1.RayJobTemplateList{} + obj, err := c.Fake. + Invokes(testing.NewListActionWithOptions(rayjobtemplatesResource, rayjobtemplatesKind, c.ns, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.RayJobTemplateList{ListMeta: obj.(*v1alpha1.RayJobTemplateList).ListMeta} + for _, item := range obj.(*v1alpha1.RayJobTemplateList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested rayJobTemplates. +func (c *FakeRayJobTemplates) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchActionWithOptions(rayjobtemplatesResource, c.ns, opts)) + +} + +// Create takes the representation of a rayJobTemplate and creates it. Returns the server's representation of the rayJobTemplate, and an error, if there is any. +func (c *FakeRayJobTemplates) Create(ctx context.Context, rayJobTemplate *v1alpha1.RayJobTemplate, opts v1.CreateOptions) (result *v1alpha1.RayJobTemplate, err error) { + emptyResult := &v1alpha1.RayJobTemplate{} + obj, err := c.Fake. + Invokes(testing.NewCreateActionWithOptions(rayjobtemplatesResource, c.ns, rayJobTemplate, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.RayJobTemplate), err +} + +// Update takes the representation of a rayJobTemplate and updates it. Returns the server's representation of the rayJobTemplate, and an error, if there is any. +func (c *FakeRayJobTemplates) Update(ctx context.Context, rayJobTemplate *v1alpha1.RayJobTemplate, opts v1.UpdateOptions) (result *v1alpha1.RayJobTemplate, err error) { + emptyResult := &v1alpha1.RayJobTemplate{} + obj, err := c.Fake. + Invokes(testing.NewUpdateActionWithOptions(rayjobtemplatesResource, c.ns, rayJobTemplate, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.RayJobTemplate), err +} + +// Delete takes name of the rayJobTemplate and deletes it. Returns an error if one occurs. +func (c *FakeRayJobTemplates) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(rayjobtemplatesResource, c.ns, name, opts), &v1alpha1.RayJobTemplate{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeRayJobTemplates) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionActionWithOptions(rayjobtemplatesResource, c.ns, opts, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.RayJobTemplateList{}) + return err +} + +// Patch applies the patch and returns the patched rayJobTemplate. +func (c *FakeRayJobTemplates) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.RayJobTemplate, err error) { + emptyResult := &v1alpha1.RayJobTemplate{} + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceActionWithOptions(rayjobtemplatesResource, c.ns, name, pt, data, opts, subresources...), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.RayJobTemplate), err +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_volumebundle.go b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_volumebundle.go new file mode 100644 index 0000000..7626eb0 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_volumebundle.go @@ -0,0 +1,133 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// FakeVolumeBundles implements VolumeBundleInterface +type FakeVolumeBundles struct { + Fake *FakeKjobctlV1alpha1 + ns string +} + +var volumebundlesResource = v1alpha1.SchemeGroupVersion.WithResource("volumebundles") + +var volumebundlesKind = v1alpha1.SchemeGroupVersion.WithKind("VolumeBundle") + +// Get takes name of the volumeBundle, and returns the corresponding volumeBundle object, and an error if there is any. +func (c *FakeVolumeBundles) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.VolumeBundle, err error) { + emptyResult := &v1alpha1.VolumeBundle{} + obj, err := c.Fake. + Invokes(testing.NewGetActionWithOptions(volumebundlesResource, c.ns, name, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.VolumeBundle), err +} + +// List takes label and field selectors, and returns the list of VolumeBundles that match those selectors. +func (c *FakeVolumeBundles) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.VolumeBundleList, err error) { + emptyResult := &v1alpha1.VolumeBundleList{} + obj, err := c.Fake. + Invokes(testing.NewListActionWithOptions(volumebundlesResource, volumebundlesKind, c.ns, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.VolumeBundleList{ListMeta: obj.(*v1alpha1.VolumeBundleList).ListMeta} + for _, item := range obj.(*v1alpha1.VolumeBundleList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested volumeBundles. +func (c *FakeVolumeBundles) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchActionWithOptions(volumebundlesResource, c.ns, opts)) + +} + +// Create takes the representation of a volumeBundle and creates it. Returns the server's representation of the volumeBundle, and an error, if there is any. +func (c *FakeVolumeBundles) Create(ctx context.Context, volumeBundle *v1alpha1.VolumeBundle, opts v1.CreateOptions) (result *v1alpha1.VolumeBundle, err error) { + emptyResult := &v1alpha1.VolumeBundle{} + obj, err := c.Fake. + Invokes(testing.NewCreateActionWithOptions(volumebundlesResource, c.ns, volumeBundle, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.VolumeBundle), err +} + +// Update takes the representation of a volumeBundle and updates it. Returns the server's representation of the volumeBundle, and an error, if there is any. +func (c *FakeVolumeBundles) Update(ctx context.Context, volumeBundle *v1alpha1.VolumeBundle, opts v1.UpdateOptions) (result *v1alpha1.VolumeBundle, err error) { + emptyResult := &v1alpha1.VolumeBundle{} + obj, err := c.Fake. + Invokes(testing.NewUpdateActionWithOptions(volumebundlesResource, c.ns, volumeBundle, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.VolumeBundle), err +} + +// Delete takes name of the volumeBundle and deletes it. Returns an error if one occurs. +func (c *FakeVolumeBundles) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(volumebundlesResource, c.ns, name, opts), &v1alpha1.VolumeBundle{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeVolumeBundles) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionActionWithOptions(volumebundlesResource, c.ns, opts, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.VolumeBundleList{}) + return err +} + +// Patch applies the patch and returns the patched volumeBundle. +func (c *FakeVolumeBundles) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.VolumeBundle, err error) { + emptyResult := &v1alpha1.VolumeBundle{} + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceActionWithOptions(volumebundlesResource, c.ns, name, pt, data, opts, subresources...), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.VolumeBundle), err +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/generated_expansion.go b/client-go/clientset/versioned/typed/apis/v1alpha1/generated_expansion.go new file mode 100644 index 0000000..5b06194 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/generated_expansion.go @@ -0,0 +1,28 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +type ApplicationProfileExpansion interface{} + +type JobTemplateExpansion interface{} + +type RayClusterTemplateExpansion interface{} + +type RayJobTemplateExpansion interface{} + +type VolumeBundleExpansion interface{} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/jobtemplate.go b/client-go/clientset/versioned/typed/apis/v1alpha1/jobtemplate.go new file mode 100644 index 0000000..3d94830 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/jobtemplate.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + scheme "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/scheme" +) + +// JobTemplatesGetter has a method to return a JobTemplateInterface. +// A group's client should implement this interface. +type JobTemplatesGetter interface { + JobTemplates(namespace string) JobTemplateInterface +} + +// JobTemplateInterface has methods to work with JobTemplate resources. +type JobTemplateInterface interface { + Create(ctx context.Context, jobTemplate *v1alpha1.JobTemplate, opts v1.CreateOptions) (*v1alpha1.JobTemplate, error) + Update(ctx context.Context, jobTemplate *v1alpha1.JobTemplate, opts v1.UpdateOptions) (*v1alpha1.JobTemplate, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.JobTemplate, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.JobTemplateList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.JobTemplate, err error) + JobTemplateExpansion +} + +// jobTemplates implements JobTemplateInterface +type jobTemplates struct { + *gentype.ClientWithList[*v1alpha1.JobTemplate, *v1alpha1.JobTemplateList] +} + +// newJobTemplates returns a JobTemplates +func newJobTemplates(c *KjobctlV1alpha1Client, namespace string) *jobTemplates { + return &jobTemplates{ + gentype.NewClientWithList[*v1alpha1.JobTemplate, *v1alpha1.JobTemplateList]( + "jobtemplates", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *v1alpha1.JobTemplate { return &v1alpha1.JobTemplate{} }, + func() *v1alpha1.JobTemplateList { return &v1alpha1.JobTemplateList{} }), + } +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/rayclustertemplate.go b/client-go/clientset/versioned/typed/apis/v1alpha1/rayclustertemplate.go new file mode 100644 index 0000000..02ca456 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/rayclustertemplate.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + scheme "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/scheme" +) + +// RayClusterTemplatesGetter has a method to return a RayClusterTemplateInterface. +// A group's client should implement this interface. +type RayClusterTemplatesGetter interface { + RayClusterTemplates(namespace string) RayClusterTemplateInterface +} + +// RayClusterTemplateInterface has methods to work with RayClusterTemplate resources. +type RayClusterTemplateInterface interface { + Create(ctx context.Context, rayClusterTemplate *v1alpha1.RayClusterTemplate, opts v1.CreateOptions) (*v1alpha1.RayClusterTemplate, error) + Update(ctx context.Context, rayClusterTemplate *v1alpha1.RayClusterTemplate, opts v1.UpdateOptions) (*v1alpha1.RayClusterTemplate, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.RayClusterTemplate, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.RayClusterTemplateList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.RayClusterTemplate, err error) + RayClusterTemplateExpansion +} + +// rayClusterTemplates implements RayClusterTemplateInterface +type rayClusterTemplates struct { + *gentype.ClientWithList[*v1alpha1.RayClusterTemplate, *v1alpha1.RayClusterTemplateList] +} + +// newRayClusterTemplates returns a RayClusterTemplates +func newRayClusterTemplates(c *KjobctlV1alpha1Client, namespace string) *rayClusterTemplates { + return &rayClusterTemplates{ + gentype.NewClientWithList[*v1alpha1.RayClusterTemplate, *v1alpha1.RayClusterTemplateList]( + "rayclustertemplates", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *v1alpha1.RayClusterTemplate { return &v1alpha1.RayClusterTemplate{} }, + func() *v1alpha1.RayClusterTemplateList { return &v1alpha1.RayClusterTemplateList{} }), + } +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/rayjobtemplate.go b/client-go/clientset/versioned/typed/apis/v1alpha1/rayjobtemplate.go new file mode 100644 index 0000000..3c61696 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/rayjobtemplate.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + scheme "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/scheme" +) + +// RayJobTemplatesGetter has a method to return a RayJobTemplateInterface. +// A group's client should implement this interface. +type RayJobTemplatesGetter interface { + RayJobTemplates(namespace string) RayJobTemplateInterface +} + +// RayJobTemplateInterface has methods to work with RayJobTemplate resources. +type RayJobTemplateInterface interface { + Create(ctx context.Context, rayJobTemplate *v1alpha1.RayJobTemplate, opts v1.CreateOptions) (*v1alpha1.RayJobTemplate, error) + Update(ctx context.Context, rayJobTemplate *v1alpha1.RayJobTemplate, opts v1.UpdateOptions) (*v1alpha1.RayJobTemplate, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.RayJobTemplate, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.RayJobTemplateList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.RayJobTemplate, err error) + RayJobTemplateExpansion +} + +// rayJobTemplates implements RayJobTemplateInterface +type rayJobTemplates struct { + *gentype.ClientWithList[*v1alpha1.RayJobTemplate, *v1alpha1.RayJobTemplateList] +} + +// newRayJobTemplates returns a RayJobTemplates +func newRayJobTemplates(c *KjobctlV1alpha1Client, namespace string) *rayJobTemplates { + return &rayJobTemplates{ + gentype.NewClientWithList[*v1alpha1.RayJobTemplate, *v1alpha1.RayJobTemplateList]( + "rayjobtemplates", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *v1alpha1.RayJobTemplate { return &v1alpha1.RayJobTemplate{} }, + func() *v1alpha1.RayJobTemplateList { return &v1alpha1.RayJobTemplateList{} }), + } +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/volumebundle.go b/client-go/clientset/versioned/typed/apis/v1alpha1/volumebundle.go new file mode 100644 index 0000000..afa0590 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/volumebundle.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + scheme "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/scheme" +) + +// VolumeBundlesGetter has a method to return a VolumeBundleInterface. +// A group's client should implement this interface. +type VolumeBundlesGetter interface { + VolumeBundles(namespace string) VolumeBundleInterface +} + +// VolumeBundleInterface has methods to work with VolumeBundle resources. +type VolumeBundleInterface interface { + Create(ctx context.Context, volumeBundle *v1alpha1.VolumeBundle, opts v1.CreateOptions) (*v1alpha1.VolumeBundle, error) + Update(ctx context.Context, volumeBundle *v1alpha1.VolumeBundle, opts v1.UpdateOptions) (*v1alpha1.VolumeBundle, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.VolumeBundle, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.VolumeBundleList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.VolumeBundle, err error) + VolumeBundleExpansion +} + +// volumeBundles implements VolumeBundleInterface +type volumeBundles struct { + *gentype.ClientWithList[*v1alpha1.VolumeBundle, *v1alpha1.VolumeBundleList] +} + +// newVolumeBundles returns a VolumeBundles +func newVolumeBundles(c *KjobctlV1alpha1Client, namespace string) *volumeBundles { + return &volumeBundles{ + gentype.NewClientWithList[*v1alpha1.VolumeBundle, *v1alpha1.VolumeBundleList]( + "volumebundles", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *v1alpha1.VolumeBundle { return &v1alpha1.VolumeBundle{} }, + func() *v1alpha1.VolumeBundleList { return &v1alpha1.VolumeBundleList{} }), + } +} diff --git a/cmd/kjobctl-docs/generators/doc.go b/cmd/kjobctl-docs/generators/doc.go new file mode 100644 index 0000000..e5007da --- /dev/null +++ b/cmd/kjobctl-docs/generators/doc.go @@ -0,0 +1,376 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generators + +import ( + "fmt" + "html" + "html/template" + "io" + "os" + "os/user" + "path/filepath" + "sort" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +type Reference struct { + NoList bool + Title string + Description string + ShowUseLine bool + UseLine string + ShowExample bool + Example string + Options []Option + InheritedOptions []Option + Links []Link +} + +type Option struct { + Name string + Shorthand string + Usage string + ValueType string + DefVal string + NoOptDefVal string +} + +type Link struct { + Name string + Description string + Path string +} + +type byName []*cobra.Command + +func (s byName) Len() int { return len(s) } +func (s byName) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s byName) Less(i, j int) bool { return s[i].Name() < s[j].Name() } + +func GenMarkdownTree(cmd *cobra.Command, templatesDir, outputDir string) error { + identity := func(s string) string { return s } + emptyStr := func(s string) string { return "" } + return GenMarkdownTreeCustom(cmd, templatesDir, outputDir, "", emptyStr, identity) +} + +func GenMarkdownTreeCustom(cmd *cobra.Command, templatesDir, outputDir string, subdir string, filePrepender, linkHandler func(string) string) error { + for _, c := range cmd.Commands() { + if !c.IsAvailableCommand() || c.IsAdditionalHelpTopicCommand() { + continue + } + // CommandPath example: 'kjobctl create localqueue' + parts := strings.Split(c.CommandPath(), " ") + childSubdir := "" + if len(parts) > 1 { + childSubdir = parts[0] + "_" + parts[1] + fullPath := filepath.Join(outputDir, childSubdir) + if _, err := os.Stat(fullPath); os.IsNotExist(err) { + if err := os.Mkdir(fullPath, 0770); err != nil { + return err + } + } + } + if err := GenMarkdownTreeCustom(c, templatesDir, outputDir, childSubdir, filePrepender, linkHandler); err != nil { + return err + } + } + + fullname := strings.ReplaceAll(cmd.CommandPath(), " ", "_") + ".md" + indexFile := false + if len(subdir) > 0 { + parts := strings.Split(cmd.CommandPath(), " ") + if len(parts) == 2 { + indexFile = true + fullname = "_index.md" + } + fullname = filepath.Join(subdir, fullname) + } + filename := filepath.Join(outputDir, fullname) + + f, err := os.Create(filename) + if err != nil { + return err + } + defer f.Close() + + if _, err := io.WriteString(f, filePrepender(filename)); err != nil { + return err + } + if err := GenReference(cmd, templatesDir, f, linkHandler, indexFile); err != nil { + return err + } + return nil +} + +func GenReference(cmd *cobra.Command, templatesDir string, w io.Writer, linkHandler func(string) string, indexFile bool) error { + ref := getReference(cmd, linkHandler, indexFile) + + tmpl, err := template.New("main.md"). + Funcs(template.FuncMap{ + "html": htmlFuncMap, + "usage": usageFuncMap, + }). + ParseGlob(fmt.Sprintf("%s/*", templatesDir)) + if err != nil { + return err + } + if err := tmpl.Execute(w, ref); err != nil { + return err + } + + return nil +} + +func htmlFuncMap(s string) template.HTML { + return template.HTML(s) +} + +func usageFuncMap(usage string) template.HTML { + usage = html.EscapeString(usage) + usage = strings.TrimSuffix(usage, "\\n") + usage = strings.ReplaceAll(usage, "\\n", "
") + return htmlFuncMap(usage) +} + +func getReference(cmd *cobra.Command, linkHandler func(string) string, indexFile bool) *Reference { + cmd.InitDefaultHelpCmd() + cmd.InitDefaultHelpFlag() + + ref := &Reference{ + NoList: indexFile, + Title: strings.TrimSpace(cmd.CommandPath()), + Description: cmd.Long, + Example: cmd.Example, + Options: getOptions(cmd.NonInheritedFlags()), + InheritedOptions: getOptions(cmd.InheritedFlags()), + Links: getLinks(cmd, linkHandler, indexFile), + } + + if len(ref.Description) == 0 { + ref.Description = cmd.Short + } + + if cmd.Runnable() { + ref.UseLine = cmd.UseLine() + } + + return ref +} + +func getOptions(f *pflag.FlagSet) []Option { + var options []Option + + f.VisitAll(func(flag *pflag.Flag) { + if len(flag.Deprecated) > 0 || flag.Hidden { + return + } + + option := Option{ + Name: flag.Name, + Usage: flag.Usage, + ValueType: getValueType(flag), + DefVal: getDefaultVal(flag), + NoOptDefVal: getNoOptDefVal(flag), + } + + if len(flag.ShorthandDeprecated) == 0 { + option.Shorthand = flag.Shorthand + } + + options = append(options, option) + }) + + return options +} + +func getValueType(flag *pflag.Flag) string { + name := flag.Value.Type() + switch name { + case "bool": + return "" + case "float64", "float32": + return "float" + case "int64", "int32", "int8": + return "int" + case "uint64", "uint32", "uint8": + return "uint" + case "stringSlice", "stringArray": + return "strings" + case "float32Slice", "float64Slice": + return "floats" + case "intSlice", "int32Slice", "int64Slice": + return "ints" + case "uintSlice": + return "uints" + case "boolSlice": + return "bools" + case "durationSlice": + return "durations" + case "stringToString": + return "" + case "stringToInt": + return "" + default: + return name + } +} + +func getNoOptDefVal(flag *pflag.Flag) string { + if len(flag.NoOptDefVal) == 0 { + return "" + } + + switch flag.Value.Type() { + case "string": + return fmt.Sprintf("[=\"%s\"]", flag.NoOptDefVal) + case "bool": + if flag.NoOptDefVal != "true" { + return fmt.Sprintf("[=%s]", flag.NoOptDefVal) + } else { + return "" + } + default: + return fmt.Sprintf("[=%s]", flag.NoOptDefVal) + } +} + +func getDefaultVal(flag *pflag.Flag) string { + var defaultVal string + + if !defaultIsZeroValue(flag) { + defaultVal = flag.DefValue + switch flag.Value.Type() { + case "string": + // There are cases where the string is very-very long, split + // it to mutiple lines manually + if len(defaultVal) > 40 { + defaultVal = strings.ReplaceAll(defaultVal, ",", ",
") + } + // clean up kjobctl cache-dir flag value + if flag.Name == "cache-dir" { + myUser, err := user.Current() + if err == nil { + noprefix := strings.TrimPrefix(defaultVal, myUser.HomeDir) + defaultVal = fmt.Sprintf("$HOME%s", noprefix) + } + } + defaultVal = fmt.Sprintf("\"%s\"", defaultVal) + case "stringSlice": + // For string slices, the default value should not contain '[' ]r ']' + defaultVal = strings.TrimPrefix(defaultVal, "[") + defaultVal = strings.TrimSuffix(defaultVal, "]") + defaultVal = strings.ReplaceAll(defaultVal, " ", "") + defaultVal = fmt.Sprintf("\"%s\"", defaultVal) + } + } + + return defaultVal +} + +func getLinks(cmd *cobra.Command, linkHandler func(string) string, indexFile bool) []Link { + var links []Link + + if cmd.HasParent() { + parent := cmd.Parent() + + parts := strings.Split(parent.CommandPath(), " ") + commandPath := strings.Join(parts, "_") + + var path string + if indexFile { + path = "../" + commandPath + ".md" + } else { + path = "_index.md" + } + + link := Link{ + Name: commandPath, + Path: linkHandler(path), + Description: parent.Short, + } + + cmd.VisitParents(func(c *cobra.Command) { + if c.DisableAutoGenTag { + cmd.DisableAutoGenTag = c.DisableAutoGenTag + } + }) + + links = append(links, link) + } + + name := cmd.CommandPath() + children := cmd.Commands() + sort.Sort(byName(children)) + + for _, child := range children { + if !child.IsAvailableCommand() || child.IsAdditionalHelpTopicCommand() { + continue + } + cname := name + " " + child.Name() + path := strings.ReplaceAll(cname, " ", "_") + if indexFile { + path += ".md" + } else { + path += "/_index.md" + } + + link := Link{ + Name: cname, + Path: linkHandler(path), + Description: child.Short, + } + + links = append(links, link) + } + + return links +} + +func defaultIsZeroValue(f *pflag.Flag) bool { + switch f.Value.Type() { + case "bool": + return f.DefValue == "false" + case "duration": + return f.DefValue == "0" || f.DefValue == "0s" + case "int", "int8", "int32", "int64", "uint", "uint8", "uint16", "uint32", "count", "float32", "float64": + return f.DefValue == "0" + case "string": + return f.DefValue == "" + case "ip", "ipMask", "ipNet": + return f.DefValue == "" + case "intSlice", "stringSlice", "stringArray": + return f.DefValue == "[]" + case "namedCertKey": + return f.DefValue == "[]" + default: + switch f.Value.String() { + case "false": + return true + case "": + return true + case "": + return true + case "0": + return true + } + return false + } +} diff --git a/cmd/kjobctl-docs/main.go b/cmd/kjobctl-docs/main.go new file mode 100644 index 0000000..218c57a --- /dev/null +++ b/cmd/kjobctl-docs/main.go @@ -0,0 +1,51 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + goflag "flag" + "log" + "os" + + "github.com/spf13/pflag" + cliflag "k8s.io/component-base/cli/flag" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/cmd/kjobctl-docs/generators" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd" +) + +func main() { + var ( + templatesDir string + outputDir string + ) + + if len(os.Args) == 3 { + templatesDir = os.Args[1] + outputDir = os.Args[2] + } else { + log.Fatalf("usage: %s ", os.Args[0]) + } + + cmd := cmd.NewDefaultKjobctlCmd() + pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) + pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) + + if err := generators.GenMarkdownTree(cmd, templatesDir, outputDir); err != nil { + panic(err) + } +} diff --git a/cmd/kjobctl-docs/templates/main.md b/cmd/kjobctl-docs/templates/main.md new file mode 100644 index 0000000..f66042c --- /dev/null +++ b/cmd/kjobctl-docs/templates/main.md @@ -0,0 +1,50 @@ +{{ html ``}} + +# {{.Title}} + + +## Synopsis + + +{{.Description}} +{{- if .UseLine}} + +``` +{{.UseLine}} +``` +{{- end}} +{{- if .Example}} + + +## Examples + +``` +{{html .Example}} +``` +{{- end}} +{{- if .Options}} + + +## Options + +{{template "options" .Options}} +{{- end}} +{{- if .InheritedOptions}} + + +## Options inherited from parent commands + +{{- template "options" .InheritedOptions}} +{{- end}} +{{- if .Links}} + + +## See Also +{{range .Links}} +* [{{.Name}}]({{.Path}}){{"\t"}} - {{.Description}} +{{- end}} +{{- end}} + diff --git a/cmd/kjobctl-docs/templates/options.html b/cmd/kjobctl-docs/templates/options.html new file mode 100644 index 0000000..55280da --- /dev/null +++ b/cmd/kjobctl-docs/templates/options.html @@ -0,0 +1,24 @@ +{{define "options"}} + + + + + + + {{- range .}} + + + + + + + + {{- end}} + +
{{if .Shorthand}}-{{.Shorthand}}, {{end}}--{{.Name}} + {{- if .ValueType}} {{.ValueType}}{{end}} + {{- if .NoOptDefVal}} {{.NoOptDefVal}}{{end}} + {{- if .DefVal}}     Default: {{.DefVal}}{{end}}
+

{{usage .Usage}}

+
+{{end}} \ No newline at end of file diff --git a/cmd/kjobctl/main.go b/cmd/kjobctl/main.go new file mode 100644 index 0000000..6697caf --- /dev/null +++ b/cmd/kjobctl/main.go @@ -0,0 +1,29 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd" +) + +func main() { + if err := cmd.NewDefaultKjobctlCmd().Execute(); err != nil { + os.Exit(1) + } +} diff --git a/config/crd/bases/kjobctl.x-k8s.io_applicationprofiles.yaml b/config/crd/bases/kjobctl.x-k8s.io_applicationprofiles.yaml new file mode 100644 index 0000000..474db35 --- /dev/null +++ b/config/crd/bases/kjobctl.x-k8s.io_applicationprofiles.yaml @@ -0,0 +1,210 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + name: applicationprofiles.kjobctl.x-k8s.io +spec: + group: kjobctl.x-k8s.io + names: + kind: ApplicationProfile + listKind: ApplicationProfileList + plural: applicationprofiles + singular: applicationprofile + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ApplicationProfile is the Schema for the applicationprofiles + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ApplicationProfileSpec defines the desired state of ApplicationProfile + properties: + supportedModes: + items: + properties: + name: + description: |- + name determines which template will be used and which object will eventually be created. + Possible values are Interactive, Job, RayJob, RayCluster and Slurm. + enum: + - Interactive + - Job + - RayJob + - RayCluster + - Slurm + type: string + requiredFlags: + description: |- + requiredFlags point which cli flags are required to be passed in order to fill the gaps in the templates. + Possible values are cmd, parallelism, completions, replicas, min-replicas, max-replicas, request, localqueue, and raycluster. + replicas, min-replicas, and max-replicas flags used only for RayJob and RayCluster mode. + The raycluster flag used only for the RayJob mode. + The request flag used only for Interactive and Job modes. + The cmd flag used only for Interactive, Job, and RayJob. + The time and priority flags can be used in all modes. + If the raycluster flag are set, none of localqueue, replicas, min-replicas, or max-replicas can be set. + For the Slurm mode, the possible values are: array, cpus-per-task, error, gpus-per-task, input, job-name, mem, mem-per-cpu, + mem-per-gpu, mem-per-task, nodes, ntasks, output, partition, localqueue. + + cmd and requests values are going to be added only to the first primary container. + items: + enum: + - cmd + - parallelism + - completions + - replicas + - min-replicas + - max-replicas + - request + - localqueue + - raycluster + - array + - cpus-per-task + - error + - gpus-per-task + - input + - job-name + - mem-per-cpu + - mem-per-gpu + - mem-per-task + - nodes + - ntasks + - output + - partition + type: string + maxItems: 14 + type: array + x-kubernetes-list-type: set + template: + description: |- + template is the name of the template. + Template type depends on ApplicationProfileMode: + - on Interactive mode it must be v1/PodTemplate + - on Job mode it must be kjobctl.x-k8s.io/v1alpha1/JobTemplate + - on RayJob mode it must be kjobctl.x-k8s.io/v1alpha1/RayJobTemplate + - on RayCluster mode it must be kjobctl.x-k8s.io/v1alpha1/RayClusterTemplate + - on Slurm mode it must be kjobctl.x-k8s.io/v1alpha1/JobTemplate + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + - template + type: object + x-kubernetes-validations: + - message: replicas flag can be used only on RayJob and RayCluster + modes + rule: '!has(self.requiredFlags) || !(''replicas'' in self.requiredFlags) + || self.name in [''RayJob'', ''RayCluster'']' + - message: min-replicas flag can be used only on RayJob and RayCluster + modes + rule: '!has(self.requiredFlags) || !(''min-replicas'' in self.requiredFlags) + || self.name in [''RayJob'', ''RayCluster'']' + - message: max-replicas flag can be used only on RayJob and RayCluster + modes + rule: '!has(self.requiredFlags) || !(''max-replicas'' in self.requiredFlags) + || self.name in [''RayJob'', ''RayCluster'']' + - message: request flag can be used only on Job and Interactive + modes + rule: '!has(self.requiredFlags) || !(''request'' in self.requiredFlags) + || self.name in [''Job'', ''Interactive'', ''RayJob'']' + - message: cmd flag can be used only on Job, Interactive and RayJob + modes + rule: '!has(self.requiredFlags) || !(''cmd'' in self.requiredFlags) + || self.name in [''Job'', ''Interactive'', ''RayJob'']' + - message: raycluster flag can be used only on RayJob mode + rule: '!has(self.requiredFlags) || !(''raycluster'' in self.requiredFlags) + || self.name == ''RayJob''' + - message: if raycluster flag are set none of localqueue, replicas, + min-replicas and max-replicas can be + rule: '!has(self.requiredFlags) || !(''raycluster'' in self.requiredFlags) + || !(''localqueue'' in self.requiredFlags || ''replicas'' in + self.requiredFlags || ''min-replicas'' in self.requiredFlags + || ''max-replicas'' in self.requiredFlags)' + - message: array flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''array'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: cpus-per-task flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''cpus-per-task'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: error flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''error'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: gpus-per-task flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''gpus-per-task'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: input flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''input'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: job-name flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''job-name'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: mem flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''mem'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: mem-per-cpu flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''mem-per-cpu'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: mem-per-gpu flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''mem-per-gpu'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: mem-per-task flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''mem-per-task'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: nodes flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''nodes'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: ntasks flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''ntasks'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: output flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''output'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: partition flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''partition'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: parallelism flag can't be used on Slurm mode + rule: '!has(self.requiredFlags) || self.name != ''Slurm'' || !(''parallelism'' + in self.requiredFlags)' + - message: completions flag can't be used on Slurm mode + rule: '!has(self.requiredFlags) || self.name != ''Slurm'' || !(''completions'' + in self.requiredFlags)' + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + volumeBundles: + items: + description: VolumeBundleReference is the name of the VolumeBundle. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + type: array + x-kubernetes-list-type: set + required: + - supportedModes + type: object + type: object + served: true + storage: true diff --git a/config/crd/bases/kjobctl.x-k8s.io_jobtemplates.yaml b/config/crd/bases/kjobctl.x-k8s.io_jobtemplates.yaml new file mode 100644 index 0000000..e171d88 --- /dev/null +++ b/config/crd/bases/kjobctl.x-k8s.io_jobtemplates.yaml @@ -0,0 +1,8406 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + name: jobtemplates.kjobctl.x-k8s.io +spec: + group: kjobctl.x-k8s.io + names: + kind: JobTemplate + listKind: JobTemplateList + plural: jobtemplates + singular: jobtemplate + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: JobTemplate is the Schema for the jobtemplate API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + template: + description: Template defines the jobs that will be created from this + pod template. + properties: + metadata: + description: Standard object's metadata. + type: object + spec: + description: Specification of the desired behavior of the job. + properties: + activeDeadlineSeconds: + description: |- + Specifies the duration in seconds relative to the startTime that the job + may be continuously active before the system tries to terminate it; value + must be positive integer. If a Job is suspended (at creation or through an + update), this timer will effectively be stopped and reset when the Job is + resumed again. + format: int64 + type: integer + backoffLimit: + description: |- + Specifies the number of retries before marking this job failed. + Defaults to 6 + format: int32 + type: integer + backoffLimitPerIndex: + description: |- + Specifies the limit for the number of retries within an + index before marking this index as failed. When enabled the number of + failures per index is kept in the pod's + batch.kubernetes.io/job-index-failure-count annotation. It can only + be set when Job's completionMode=Indexed, and the Pod's restart + policy is Never. The field is immutable. + This field is beta-level. It can be used when the `JobBackoffLimitPerIndex` + feature gate is enabled (enabled by default). + format: int32 + type: integer + completionMode: + description: |- + completionMode specifies how Pod completions are tracked. It can be + `NonIndexed` (default) or `Indexed`. + + `NonIndexed` means that the Job is considered complete when there have + been .spec.completions successfully completed Pods. Each Pod completion is + homologous to each other. + + `Indexed` means that the Pods of a + Job get an associated completion index from 0 to (.spec.completions - 1), + available in the annotation batch.kubernetes.io/job-completion-index. + The Job is considered complete when there is one successfully completed Pod + for each index. + When value is `Indexed`, .spec.completions must be specified and + `.spec.parallelism` must be less than or equal to 10^5. + In addition, The Pod name takes the form + `$(job-name)-$(index)-$(random-string)`, + the Pod hostname takes the form `$(job-name)-$(index)`. + + More completion modes can be added in the future. + If the Job controller observes a mode that it doesn't recognize, which + is possible during upgrades due to version skew, the controller + skips updates for the Job. + type: string + completions: + description: |- + Specifies the desired number of successfully finished pods the + job should be run with. Setting to null means that the success of any + pod signals the success of all pods, and allows parallelism to have any positive + value. Setting to 1 means that parallelism is limited to 1 and the success of that + pod signals the success of the job. + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ + format: int32 + type: integer + managedBy: + description: |- + ManagedBy field indicates the controller that manages a Job. The k8s Job + controller reconciles jobs which don't have this field at all or the field + value is the reserved string `kubernetes.io/job-controller`, but skips + reconciling Jobs with a custom value for this field. + The value must be a valid domain-prefixed path (e.g. acme.io/foo) - + all characters before the first "/" must be a valid subdomain as defined + by RFC 1123. All characters trailing the first "/" must be valid HTTP Path + characters as defined by RFC 3986. The value cannot exceed 63 characters. + This field is immutable. + + This field is alpha-level. The job controller accepts setting the field + when the feature gate JobManagedBy is enabled (disabled by default). + type: string + manualSelector: + description: |- + manualSelector controls generation of pod labels and pod selectors. + Leave `manualSelector` unset unless you are certain what you are doing. + When false or unset, the system pick labels unique to this job + and appends those labels to the pod template. When true, + the user is responsible for picking unique labels and specifying + the selector. Failure to pick a unique label may cause this + and other jobs to not function correctly. However, You may see + `manualSelector=true` in jobs that were created with the old `extensions/v1beta1` + API. + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#specifying-your-own-pod-selector + type: boolean + maxFailedIndexes: + description: |- + Specifies the maximal number of failed indexes before marking the Job as + failed, when backoffLimitPerIndex is set. Once the number of failed + indexes exceeds this number the entire Job is marked as Failed and its + execution is terminated. When left as null the job continues execution of + all of its indexes and is marked with the `Complete` Job condition. + It can only be specified when backoffLimitPerIndex is set. + It can be null or up to completions. It is required and must be + less than or equal to 10^4 when is completions greater than 10^5. + This field is beta-level. It can be used when the `JobBackoffLimitPerIndex` + feature gate is enabled (enabled by default). + format: int32 + type: integer + parallelism: + description: |- + Specifies the maximum desired number of pods the job should + run at any given time. The actual number of pods running in steady state will + be less than this number when ((.spec.completions - .status.successful) < .spec.parallelism), + i.e. when the work left to do is less than max parallelism. + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ + format: int32 + type: integer + podFailurePolicy: + description: |- + Specifies the policy of handling failed pods. In particular, it allows to + specify the set of actions and conditions which need to be + satisfied to take the associated action. + If empty, the default behaviour applies - the counter of failed pods, + represented by the jobs's .status.failed field, is incremented and it is + checked against the backoffLimit. This field cannot be used in combination + with restartPolicy=OnFailure. + properties: + rules: + description: |- + A list of pod failure policy rules. The rules are evaluated in order. + Once a rule matches a Pod failure, the remaining of the rules are ignored. + When no rule matches the Pod failure, the default handling applies - the + counter of pod failures is incremented and it is checked against + the backoffLimit. At most 20 elements are allowed. + items: + description: |- + PodFailurePolicyRule describes how a pod failure is handled when the requirements are met. + One of onExitCodes and onPodConditions, but not both, can be used in each rule. + properties: + action: + description: |- + Specifies the action taken on a pod failure when the requirements are satisfied. + Possible values are: + + - FailJob: indicates that the pod's job is marked as Failed and all + running pods are terminated. + - FailIndex: indicates that the pod's index is marked as Failed and will + not be restarted. + This value is beta-level. It can be used when the + `JobBackoffLimitPerIndex` feature gate is enabled (enabled by default). + - Ignore: indicates that the counter towards the .backoffLimit is not + incremented and a replacement pod is created. + - Count: indicates that the pod is handled in the default way - the + counter towards the .backoffLimit is incremented. + Additional values are considered to be added in the future. Clients should + react to an unknown action by skipping the rule. + type: string + onExitCodes: + description: Represents the requirement on the container + exit codes. + properties: + containerName: + description: |- + Restricts the check for exit codes to the container with the + specified name. When null, the rule applies to all containers. + When specified, it should match one the container or initContainer + names in the pod template. + type: string + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Containers completed with success (exit code 0) are + excluded from the requirement check. Possible values are: + + - In: the requirement is satisfied if at least one container exit code + (might be multiple if there are multiple containers not restricted + by the 'containerName' field) is in the set of specified values. + - NotIn: the requirement is satisfied if at least one container exit code + (might be multiple if there are multiple containers not restricted + by the 'containerName' field) is not in the set of specified values. + Additional values are considered to be added in the future. Clients should + react to an unknown operator by assuming the requirement is not satisfied. + type: string + values: + description: |- + Specifies the set of values. Each returned container exit code (might be + multiple in case of multiple containers) is checked against this set of + values with respect to the operator. The list of values must be ordered + and must not contain duplicates. Value '0' cannot be used for the In operator. + At least one element is required. At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + - values + type: object + onPodConditions: + description: |- + Represents the requirement on the pod conditions. The requirement is represented + as a list of pod condition patterns. The requirement is satisfied if at + least one pattern matches an actual pod condition. At most 20 elements are allowed. + items: + description: |- + PodFailurePolicyOnPodConditionsPattern describes a pattern for matching + an actual pod condition type. + properties: + status: + description: |- + Specifies the required Pod condition status. To match a pod condition + it is required that the specified status equals the pod condition status. + Defaults to True. + type: string + type: + description: |- + Specifies the required Pod condition type. To match a pod condition + it is required that specified type equals the pod condition type. + type: string + required: + - status + - type + type: object + type: array + x-kubernetes-list-type: atomic + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + required: + - rules + type: object + podReplacementPolicy: + description: |- + podReplacementPolicy specifies when to create replacement Pods. + Possible values are: + - TerminatingOrFailed means that we recreate pods + when they are terminating (has a metadata.deletionTimestamp) or failed. + - Failed means to wait until a previously created Pod is fully terminated (has phase + Failed or Succeeded) before creating a replacement Pod. + + When using podFailurePolicy, Failed is the the only allowed value. + TerminatingOrFailed and Failed are allowed values when podFailurePolicy is not in use. + This is an beta field. To use this, enable the JobPodReplacementPolicy feature toggle. + This is on by default. + type: string + selector: + description: |- + A label query over pods that should match the pod count. + Normally, the system sets this field for you. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + successPolicy: + description: |- + successPolicy specifies the policy when the Job can be declared as succeeded. + If empty, the default behavior applies - the Job is declared as succeeded + only when the number of succeeded pods equals to the completions. + When the field is specified, it must be immutable and works only for the Indexed Jobs. + Once the Job meets the SuccessPolicy, the lingering pods are terminated. + + This field is beta-level. To use this field, you must enable the + `JobSuccessPolicy` feature gate (enabled by default). + properties: + rules: + description: |- + rules represents the list of alternative rules for the declaring the Jobs + as successful before `.status.succeeded >= .spec.completions`. Once any of the rules are met, + the "SucceededCriteriaMet" condition is added, and the lingering pods are removed. + The terminal state for such a Job has the "Complete" condition. + Additionally, these rules are evaluated in order; Once the Job meets one of the rules, + other rules are ignored. At most 20 elements are allowed. + items: + description: |- + SuccessPolicyRule describes rule for declaring a Job as succeeded. + Each rule must have at least one of the "succeededIndexes" or "succeededCount" specified. + properties: + succeededCount: + description: |- + succeededCount specifies the minimal required size of the actual set of the succeeded indexes + for the Job. When succeededCount is used along with succeededIndexes, the check is + constrained only to the set of indexes specified by succeededIndexes. + For example, given that succeededIndexes is "1-4", succeededCount is "3", + and completed indexes are "1", "3", and "5", the Job isn't declared as succeeded + because only "1" and "3" indexes are considered in that rules. + When this field is null, this doesn't default to any value and + is never evaluated at any time. + When specified it needs to be a positive integer. + format: int32 + type: integer + succeededIndexes: + description: |- + succeededIndexes specifies the set of indexes + which need to be contained in the actual set of the succeeded indexes for the Job. + The list of indexes must be within 0 to ".spec.completions-1" and + must not contain duplicates. At least one element is required. + The indexes are represented as intervals separated by commas. + The intervals can be a decimal integer or a pair of decimal integers separated by a hyphen. + The number are listed in represented by the first and last element of the series, + separated by a hyphen. + For example, if the completed indexes are 1, 3, 4, 5 and 7, they are + represented as "1,3-5,7". + When this field is null, this field doesn't default to any value + and is never evaluated at any time. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + required: + - rules + type: object + suspend: + description: |- + suspend specifies whether the Job controller should create Pods or not. If + a Job is created with suspend set to true, no Pods are created by the Job + controller. If a Job is suspended after creation (i.e. the flag goes from + false to true), the Job controller will delete all active Pods associated + with this Job. Users must design their workload to gracefully handle this. + Suspending a Job will reset the StartTime field of the Job, effectively + resetting the ActiveDeadlineSeconds timer too. Defaults to false. + type: boolean + template: + description: |- + Describes the pod that will be created when executing a job. + The only allowed template.spec.restartPolicy values are "Never" or "OnFailure". + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling rules + for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, in + the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules + (e.g. co-locate this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added per-node + to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same node, + zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added per-node + to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates whether + a service account token should be automatically mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a + C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will be + mapped to. + type: string + name: + description: name must match the name of a + persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a + Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver + options of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a + C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral + containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral + containers. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will be + mapped to. + type: string + name: + description: name must match the name of a + persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a + Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a + C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will be + mapped to. + type: string + name: + description: name must match the name of a + persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a + Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference + to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition + in the pod's condition list with matching type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated to a Pod + to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to + be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies how + to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume in a pod + that may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data + Disk mount on the host and bind mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching + mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data + disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk + in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: + multiple blob disks per storage account Dedicated: + single blob disk per storage account Managed: + azure managed data disk (only in managed availability + set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File + Service mount on the host and bind mount to the + pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret + that contains Azure Storage Account Name and + Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount on + the host that shares a pod's lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the + mounted root, rather than the full Ceph tree, + default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that + should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents + ephemeral storage that is handled by certain external + CSI drivers (Beta feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API + about the pod that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API + volume file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. + Must not be absolute or contain the + ''..'' path. Must be utf-8 encoded. + The first item of the relative path + must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of + resource being referenced + type: string + name: + description: Name is the name of + resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of + resource being referenced + type: string + name: + description: Name is the name of + resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query + over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding + reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource + that is attached to a kubelet's host machine and + then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun + number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target + worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver + to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field + holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume + attached to a kubelet's host machine. This depends + on the Flocker control service being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the + dataset. This is unique identifier of a Flocker + dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for + the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether + support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified + Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun + number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for + iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets + host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx + volume attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a + Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources + secrets, configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the + volume root to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about + the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about + the downwardAPI data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile + represents information to create + the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only annotations, + labels, name, namespace and + uid are supported.' + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path + is the relative path name + of the file to be created. + Must not be absolute or contain + the ''..'' path. Must be utf-8 + encoded. The first item of + the relative path must not + start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about + the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify + whether the Secret or its key must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information + about the serviceAccountToken data to + project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte mount + on the host that shares a pod's lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references + an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent + volume attached and mounted on Kubernetes nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of + the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of + the ScaleIO Protection Domain for the configured + storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage + Pool associated with the protection domain. + type: string + system: + description: system is the name of the storage + system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether + the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere + volume attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile ID + associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage + Policy Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies + vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + ttlSecondsAfterFinished: + description: |- + ttlSecondsAfterFinished limits the lifetime of a Job that has finished + execution (either Complete or Failed). If this field is set, + ttlSecondsAfterFinished after the Job finishes, it is eligible to be + automatically deleted. When the Job is being deleted, its lifecycle + guarantees (e.g. finalizers) will be honored. If this field is unset, + the Job won't be automatically deleted. If this field is set to zero, + the Job becomes eligible to be deleted immediately after it finishes. + format: int32 + type: integer + required: + - template + type: object + required: + - spec + type: object + required: + - template + type: object + served: true + storage: true diff --git a/config/crd/bases/kjobctl.x-k8s.io_rayclustertemplates.yaml b/config/crd/bases/kjobctl.x-k8s.io_rayclustertemplates.yaml new file mode 100644 index 0000000..657bcf5 --- /dev/null +++ b/config/crd/bases/kjobctl.x-k8s.io_rayclustertemplates.yaml @@ -0,0 +1,17400 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + name: rayclustertemplates.kjobctl.x-k8s.io +spec: + group: kjobctl.x-k8s.io + names: + kind: RayClusterTemplate + listKind: RayClusterTemplateList + plural: rayclustertemplates + singular: rayclustertemplate + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: RayClusterTemplate is the Schema for the rayclustertemplate API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + template: + description: Template defines rayclusters that will be created from this + raycluster template. + properties: + metadata: + description: Standard object's metadata. + type: object + spec: + description: Specification of the desired behavior of the raycluster. + properties: + autoscalerOptions: + description: AutoscalerOptions specifies optional configuration + for the Ray autoscaler. + properties: + env: + description: Optional list of environment variables to set + in the autoscaler container. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + description: Optional list of sources to populate environment + variables in the autoscaler container. + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + idleTimeoutSeconds: + description: |- + IdleTimeoutSeconds is the number of seconds to wait before scaling down a worker pod which is not using Ray resources. + Defaults to 60 (one minute). It is not read by the KubeRay operator but by the Ray autoscaler. + format: int32 + type: integer + image: + description: Image optionally overrides the autoscaler's container + image. This override is for provided for autoscaler testing + and development. + type: string + imagePullPolicy: + description: ImagePullPolicy optionally overrides the autoscaler + container's image pull policy. This override is for provided + for autoscaler testing and development. + type: string + resources: + description: |- + Resources specifies optional resource request and limit overrides for the autoscaler container. + Default values: 500m CPU request and limit. 512Mi memory request and limit. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of + the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + upscalingMode: + description: |- + UpscalingMode is "Conservative", "Default", or "Aggressive." + Conservative: Upscaling is rate-limited; the number of pending worker pods is at most the size of the Ray cluster. + Default: Upscaling is not rate-limited. + Aggressive: An alias for Default; upscaling is not rate-limited. + It is not read by the KubeRay operator but by the Ray autoscaler. + enum: + - Default + - Aggressive + - Conservative + type: string + volumeMounts: + description: Optional list of volumeMounts. This is needed + for enabling TLS for the autoscaler container. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + type: object + enableInTreeAutoscaling: + description: EnableInTreeAutoscaling indicates whether operator + should create in tree autoscaling configs + type: boolean + headGroupSpec: + description: |- + INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + Important: Run "make" to regenerate code after modifying this file + HeadGroupSpecs are the spec for the head pod + properties: + enableIngress: + description: EnableIngress indicates whether operator should + create ingress object for head service or not. + type: boolean + headService: + description: HeadService is the Kubernetes service of the + head pod. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + type: object + spec: + description: |- + Spec defines the behavior of a service. + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + allocateLoadBalancerNodePorts: + description: |- + allocateLoadBalancerNodePorts defines if NodePorts will be automatically + allocated for services with type LoadBalancer. Default is "true". It + may be set to "false" if the cluster load-balancer does not rely on + NodePorts. If the caller requests specific NodePorts (by specifying a + value), those requests will be respected, regardless of this field. + This field may only be set for services with type LoadBalancer and will + be cleared if the type is changed to any other type. + type: boolean + clusterIP: + description: |- + clusterIP is the IP address of the service and is usually assigned + randomly. If an address is specified manually, is in-range (as per + system configuration), and is not in use, it will be allocated to the + service; otherwise creation of the service will fail. This field may not + be changed through updates unless the type field is also being changed + to ExternalName (which requires this field to be blank) or the type + field is being changed from ExternalName (in which case this field may + optionally be specified, as describe above). Valid values are "None", + empty string (""), or a valid IP address. Setting this to "None" makes a + "headless service" (no virtual IP), which is useful when direct endpoint + connections are preferred and proxying is not required. Only applies to + types ClusterIP, NodePort, and LoadBalancer. If this field is specified + when creating a Service of type ExternalName, creation will fail. This + field will be wiped when updating a Service to type ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + clusterIPs: + description: |- + ClusterIPs is a list of IP addresses assigned to this service, and are + usually assigned randomly. If an address is specified manually, is + in-range (as per system configuration), and is not in use, it will be + allocated to the service; otherwise creation of the service will fail. + This field may not be changed through updates unless the type field is + also being changed to ExternalName (which requires this field to be + empty) or the type field is being changed from ExternalName (in which + case this field may optionally be specified, as describe above). Valid + values are "None", empty string (""), or a valid IP address. Setting + this to "None" makes a "headless service" (no virtual IP), which is + useful when direct endpoint connections are preferred and proxying is + not required. Only applies to types ClusterIP, NodePort, and + LoadBalancer. If this field is specified when creating a Service of type + ExternalName, creation will fail. This field will be wiped when updating + a Service to type ExternalName. If this field is not specified, it will + be initialized from the clusterIP field. If this field is specified, + clients must ensure that clusterIPs[0] and clusterIP have the same + value. + + This field may hold a maximum of two entries (dual-stack IPs, in either order). + These IPs must correspond to the values of the ipFamilies field. Both + clusterIPs and ipFamilies are governed by the ipFamilyPolicy field. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + description: |- + externalIPs is a list of IP addresses for which nodes in the cluster + will also accept traffic for this service. These IPs are not managed by + Kubernetes. The user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external load-balancers + that are not part of the Kubernetes system. + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + healthCheckNodePort: + description: |- + healthCheckNodePort specifies the healthcheck nodePort for the service. + This only applies when type is set to LoadBalancer and + externalTrafficPolicy is set to Local. If a value is specified, is + in-range, and is not in use, it will be used. If not specified, a value + will be automatically allocated. External systems (e.g. load-balancers) + can use this port to determine if a given node holds endpoints for this + service or not. If this field is specified when creating a Service + which does not need it, creation will fail. This field will be wiped + when updating a Service to no longer need it (e.g. changing type). + This field cannot be updated once set. + format: int32 + type: integer + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilies: + description: |- + IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this + service. This field is usually assigned automatically based on cluster + configuration and the ipFamilyPolicy field. If this field is specified + manually, the requested family is available in the cluster, + and ipFamilyPolicy allows it, it will be used; otherwise creation of + the service will fail. This field is conditionally mutable: it allows + for adding or removing a secondary IP family, but it does not allow + changing the primary IP family of the Service. Valid values are "IPv4" + and "IPv6". This field only applies to Services of types ClusterIP, + NodePort, and LoadBalancer, and does apply to "headless" services. + This field will be wiped when updating a Service to type ExternalName. + + This field may hold a maximum of two entries (dual-stack families, in + either order). These families must correspond to the values of the + clusterIPs field, if specified. Both clusterIPs and ipFamilies are + governed by the ipFamilyPolicy field. + items: + description: |- + IPFamily represents the IP Family (IPv4 or IPv6). This type is used + to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerIP: + description: |- + Only applies to Service Type: LoadBalancer. + This feature depends on whether the underlying cloud-provider supports specifying + the loadBalancerIP when a load balancer is created. + This field will be ignored if the cloud-provider does not support the feature. + Deprecated: This field was under-specified and its meaning varies across implementations. + Using it is non-portable and it may not support dual-stack. + Users are encouraged to use implementation-specific annotations when available. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + ports: + description: |- + The list of ports that are exposed by this service. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + items: + description: ServicePort contains information on + service's port. + properties: + appProtocol: + description: |- + The application protocol for this port. + This is used as a hint for implementations to offer richer behavior for protocols that they understand. + This field follows standard Kubernetes label syntax. + Valid values are either: + + * Un-prefixed protocol names - reserved for IANA standard service names (as per + RFC-6335 and https://www.iana.org/assignments/service-names). + + * Kubernetes-defined prefixed names: + * 'kubernetes.io/h2c' - HTTP/2 prior knowledge over cleartext as described in https://www.rfc-editor.org/rfc/rfc9113.html#name-starting-http-2-with-prior- + * 'kubernetes.io/ws' - WebSocket over cleartext as described in https://www.rfc-editor.org/rfc/rfc6455 + * 'kubernetes.io/wss' - WebSocket over TLS as described in https://www.rfc-editor.org/rfc/rfc6455 + + * Other protocols should use implementation-defined prefixed names such as + mycompany.com/my-custom-protocol. + type: string + name: + description: |- + The name of this port within the service. This must be a DNS_LABEL. + All ports within a ServiceSpec must have unique names. When considering + the endpoints for a Service, this must match the 'name' field in the + EndpointPort. + Optional if only one ServicePort is defined on this service. + type: string + nodePort: + description: |- + The port on each node on which this service is exposed when type is + NodePort or LoadBalancer. Usually assigned by the system. If a value is + specified, in-range, and not in use it will be used, otherwise the + operation will fail. If not specified, a port will be allocated if this + Service requires one. If this field is specified when creating a + Service which does not need it, creation will fail. This field will be + wiped when updating a Service to no longer need it (e.g. changing type + from NodePort to ClusterIP). + More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + format: int32 + type: integer + port: + description: The port that will be exposed by + this service. + format: int32 + type: integer + protocol: + default: TCP + description: |- + The IP protocol for this port. Supports "TCP", "UDP", and "SCTP". + Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the pods targeted by the service. + Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named port in the + target Pod's container ports. If this is not specified, the value + of the 'port' field is used (an identity map). + This field is ignored for services with clusterIP=None, and should be + omitted or set equal to the 'port' field. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + description: |- + publishNotReadyAddresses indicates that any agent which deals with endpoints for this + Service should disregard any indications of ready/not-ready. + The primary use case for setting this field is for a StatefulSet's Headless Service to + propagate SRV DNS records for its Pods for the purpose of peer discovery. + The Kubernetes controllers that generate Endpoints and EndpointSlice resources for + Services interpret this to mean that all endpoints are considered "ready" even if the + Pods themselves are not. Agents which consume only Kubernetes generated endpoints + through the Endpoints or EndpointSlice resources can safely assume this behavior. + type: boolean + selector: + additionalProperties: + type: string + description: |- + Route service traffic to pods with label keys and values matching this + selector. If empty or not present, the service is assumed to have an + external process managing its endpoints, which Kubernetes will not + modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + trafficDistribution: + description: |- + TrafficDistribution offers a way to express preferences for how traffic is + distributed to Service endpoints. Implementations can use this field as a + hint, but are not required to guarantee strict adherence. If the field is + not set, the implementation will apply its default routing strategy. If set + to "PreferClose", implementations should prioritize endpoints that are + topologically close (e.g., same zone). + This is an alpha field and requires enabling ServiceTrafficDistribution feature. + type: string + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + status: + description: |- + Most recently observed status of the service. + Populated by the system. + Read-only. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + conditions: + description: Current service state + items: + description: Condition contains details for one + aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of + True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase + or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + loadBalancer: + description: |- + LoadBalancer contains the current status of the load-balancer, + if one is present. + properties: + ingress: + description: |- + Ingress is a list containing ingress points for the load-balancer. + Traffic intended for the service should be sent to these ingress points. + items: + description: |- + LoadBalancerIngress represents the status of a load-balancer ingress point: + traffic intended for the service should be sent to an ingress point. + properties: + hostname: + description: |- + Hostname is set for load-balancer ingress points that are DNS based + (typically AWS load-balancers) + type: string + ip: + description: |- + IP is set for load-balancer ingress points that are IP based + (typically GCE or OpenStack load-balancers) + type: string + ipMode: + description: |- + IPMode specifies how the load-balancer IP behaves, and may only be specified when the ip field is specified. + Setting this to "VIP" indicates that traffic is delivered to the node with + the destination set to the load-balancer's IP and port. + Setting this to "Proxy" indicates that traffic is delivered to the node or pod with + the destination set to the node's IP and node port or the pod's IP and port. + Service implementations may use this information to adjust traffic routing. + type: string + ports: + description: |- + Ports is a list of records of service ports + If used, every port defined in the service should have an entry in it + items: + properties: + error: + description: |- + Error is to record the problem with the service port + The format of the error shall comply with the following rules: + - built-in error values shall be specified in this file and those shall use + CamelCase names + - cloud provider specific error values must have names that comply with the + format foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number + of the service port of which status + is recorded here + format: int32 + type: integer + protocol: + description: |- + Protocol is the protocol of the service port of which status is recorded here + The supported values are: "TCP", "UDP", "SCTP" + type: string + required: + - error + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + type: object + rayStartParams: + additionalProperties: + type: string + description: 'RayStartParams are the params of the start command: + node-manager-port, object-store-memory, ...' + type: object + serviceType: + description: ServiceType is Kubernetes service type of the + head service. it will be used by the workers to connect + to the head pod + type: string + template: + description: Template is the exact pod template used in K8s + depoyments, statefulsets, etc. + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling + rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with + matching the corresponding nodeSelectorTerm, + in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node + selector terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling + rules (e.g. co-locate this pod in the same node, + zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same + node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates + whether a service account token should be automatically + mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container that + you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if + value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in + terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified API + version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the + source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to + prepend to each key in the ConfigMap. + Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the + external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the + container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name + of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name + of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS + resolver options of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if + value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in + terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified API + version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the + source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to + prepend to each key in the ConfigMap. + Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral + containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral + containers. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the + external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the + container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name + of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name + of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container that + you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if + value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in + terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified API + version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the + source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to + prepend to each key in the ConfigMap. + Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the + external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the + container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name + of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name + of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference + to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition + in the pod's condition list with matching + type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated to + a Pod to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter + to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies + how to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume in + a pod that may be accessed by any container in + the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data + Disk mount on the host and bind mount to the + pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching + mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the + data disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data + disk in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: + multiple blob disks per storage account Dedicated: + single blob disk per storage account Managed: + azure managed data disk (only in managed + availability set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File + Service mount on the host and bind mount to + the pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of + secret that contains Azure Storage Account + Name and Key + type: string + shareName: + description: shareName is the azure share + Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount + on the host that shares a pod's lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as + the mounted root, rather than the full + Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap + that should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the + ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) + represents ephemeral storage that is handled + by certain external CSI drivers (Beta feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward + API about the pod that should populate this + volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward + API volume file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a + field of the pod: only annotations, + labels, name, namespace and uid + are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in + terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified API + version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file to + be created. Must not be absolute + or contain the ''..'' path. Must + be utf-8 encoded. The first item + of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label + query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding + reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource + that is attached to a kubelet's host machine + and then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target + lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC + target worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver + to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this + field holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume + attached to a kubelet's host machine. This + depends on the Flocker control service being + running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of + the dataset. This is unique identifier + of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash + for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether + support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified + Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target + Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret + for iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents + a PhotonController persistent disk attached + and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx + volume attached and mounted on kubelets host + machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies + a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one + resources secrets, configmaps, and downward + API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from + the volume root to write the + bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information + about the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key + to a path within a volume. + properties: + key: + description: key is the + key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify + whether the ConfigMap or its + keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information + about the downwardAPI data to project + properties: + items: + description: Items is a list of + DownwardAPIVolume file + items: + description: DownwardAPIVolumeFile + represents information to + create the file containing + the pod field + properties: + fieldRef: + description: 'Required: + Selects a field of the + pod: only annotations, + labels, name, namespace + and uid are supported.' + properties: + apiVersion: + description: Version + of the schema the + FieldPath is written + in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of + the field to select + in the specified API + version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: + Path is the relative + path name of the file + to be created. Must not + be absolute or contain + the ''..'' path. Must + be utf-8 encoded. The + first item of the relative + path must not start with + ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required for + volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies + the output format + of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about + the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key + to a path within a volume. + properties: + key: + description: key is the + key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify + whether the Secret or its key + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is + information about the serviceAccountToken + data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte mount + on the host that shares a pod's lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references + an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent + volume attached and mounted on Kubernetes + nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address + of the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name + of the ScaleIO Protection Domain for the + configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default + false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO + Storage Pool associated with the protection + domain. + type: string + system: + description: system is the name of the storage + system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether + the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS + volume attached and mounted on Kubernetes + nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere + volume attached and mounted on kubelets host + machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile + ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage + Policy Based Management (SPBM) profile + name. + type: string + volumePath: + description: volumePath is the path that + identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + required: + - rayStartParams + - template + type: object + headServiceAnnotations: + additionalProperties: + type: string + type: object + rayVersion: + description: RayVersion is used to determine the command for the + Kubernetes Job managed by RayJob + type: string + suspend: + description: |- + Suspend indicates whether a RayCluster should be suspended. + A suspended RayCluster will have head pods and worker pods deleted. + type: boolean + workerGroupSpecs: + description: WorkerGroupSpecs are the specs for the worker pods + items: + description: WorkerGroupSpec are the specs for the worker pods + properties: + groupName: + description: we can have multiple worker groups, we distinguish + them by name + type: string + maxReplicas: + default: 2147483647 + description: MaxReplicas denotes the maximum number of desired + Pods for this worker group, and the default value is maxInt32. + format: int32 + type: integer + minReplicas: + default: 0 + description: MinReplicas denotes the minimum number of desired + Pods for this worker group. + format: int32 + type: integer + numOfHosts: + default: 1 + description: NumOfHosts denotes the number of hosts to create + per replica. The default value is 1. + format: int32 + type: integer + rayStartParams: + additionalProperties: + type: string + description: 'RayStartParams are the params of the start + command: address, object-store-memory, ...' + type: object + replicas: + default: 0 + description: Replicas is the number of desired Pods for + this worker group. See https://github.com/ray-project/kuberay/pull/1443 + for more details about the reason for making this field + optional. + format: int32 + type: integer + scaleStrategy: + description: ScaleStrategy defines which pods to remove + properties: + workersToDelete: + description: WorkersToDelete workers to be deleted + items: + type: string + type: array + type: object + template: + description: Template is a pod template for the worker + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling + constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling + rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, + associated with the corresponding + weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with + matching the corresponding nodeSelectorTerm, + in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node + selector terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling + rules (e.g. co-locate this pod in the same + node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the + matched WeightedPodAffinityTerm fields + are added per-node to find the most + preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the + same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the + matched WeightedPodAffinityTerm fields + are added per-node to find the most + preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates + whether a service account token should be automatically + mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container that + you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment + variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if + value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a + ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the + schema the FieldPath is written + in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a + secret in the pod's namespace + properties: + key: + description: The key of the + secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the + source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier + to prepend to each key in the ConfigMap. + Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a + network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the + external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the + container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of + block devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path + inside of the container that the device + will be mapped to. + type: string + name: + description: name must match the name + of a persistentVolumeClaim in the + pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name + of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS + resolver options of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment + variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if + value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a + ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the + schema the FieldPath is written + in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a + secret in the pod's namespace + properties: + key: + description: The key of the + secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the + source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier + to prepend to each key in the ConfigMap. + Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for + ephemeral containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral + containers. + items: + description: ContainerPort represents a + network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the + external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the + container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of + block devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path + inside of the container that the device + will be mapped to. + type: string + name: + description: name must match the name + of a persistentVolumeClaim in the + pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name + of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container that + you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment + variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if + value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a + ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the + schema the FieldPath is written + in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a + secret in the pod's namespace + properties: + key: + description: The key of the + secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the + source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier + to prepend to each key in the ConfigMap. + Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a + network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the + external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the + container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of + block devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path + inside of the container that the device + will be mapped to. + type: string + name: + description: name must match the name + of a persistentVolumeClaim in the + pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name + of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference + to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition + in the pod's condition list with matching + type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated to + a Pod to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter + to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies + how to spread matching pods among the given + topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume + in a pod that may be accessed by any container + in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure + Data Disk mount on the host and bind mount + to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host + Caching mode: None, Read Only, Read + Write.' + type: string + diskName: + description: diskName is the Name of the + data disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data + disk in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are + Shared: multiple blob disks per storage + account Dedicated: single blob disk + per storage account Managed: azure + managed data disk (only in managed availability + set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure + File Service mount on the host and bind + mount to the pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of + secret that contains Azure Storage Account + Name and Key + type: string + shareName: + description: shareName is the azure share + Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount + on the host that shares a pod's lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as + the mounted root, rather than the full + Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap + that should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a + path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) + represents ephemeral storage that is handled + by certain external CSI drivers (Beta feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward + API about the pod that should populate this + volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward + API volume file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only annotations, + labels, name, namespace and uid + are supported.' + properties: + apiVersion: + description: Version of the + schema the FieldPath is written + in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file + to be created. Must not be absolute + or contain the ''..'' path. Must + be utf-8 encoded. The first item + of the relative path must not + start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label + query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the + binding reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel + resource that is attached to a kubelet's + host machine and then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target + lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: + FC target worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the + driver to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this + field holds extra command options if + any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker + volume attached to a kubelet's host machine. + This depends on the Flocker control service + being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of + the dataset. This is unique identifier + of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash + for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines + whether support iSCSI Discovery CHAP + authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified + Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target + Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret + for iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents + a PhotonController persistent disk attached + and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx + volume attached and mounted on kubelets + host machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies + a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one + resources secrets, configmaps, and downward + API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from + the volume root to write the + bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information + about the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string + key to a path within a volume. + properties: + key: + description: key is the + key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify + whether the ConfigMap or its + keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information + about the downwardAPI data to + project + properties: + items: + description: Items is a list + of DownwardAPIVolume file + items: + description: DownwardAPIVolumeFile + represents information to + create the file containing + the pod field + properties: + fieldRef: + description: 'Required: + Selects a field of the + pod: only annotations, + labels, name, namespace + and uid are supported.' + properties: + apiVersion: + description: Version + of the schema the + FieldPath is written + in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path + of the field to + select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: + Path is the relative + path name of the file + to be created. Must + not be absolute or contain + the ''..'' path. Must + be utf-8 encoded. The + first item of the relative + path must not start + with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required for + volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies + the output format + of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information + about the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string + key to a path within a volume. + properties: + key: + description: key is the + key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field + specify whether the Secret + or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken + is information about the serviceAccountToken + data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte + mount on the host that shares a pod's lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references + an already created Quobyte volume by + name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO + persistent volume attached and mounted on + Kubernetes nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address + of the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name + of the ScaleIO Protection Domain for + the configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default + false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO + Storage Pool associated with the protection + domain. + type: string + system: + description: system is the name of the + storage system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a + path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether + the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS + volume attached and mounted on Kubernetes + nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere + volume attached and mounted on kubelets + host machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile + ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the + storage Policy Based Management (SPBM) + profile name. + type: string + volumePath: + description: volumePath is the path that + identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + required: + - groupName + - maxReplicas + - minReplicas + - rayStartParams + - template + type: object + type: array + required: + - headGroupSpec + type: object + required: + - spec + type: object + required: + - template + type: object + served: true + storage: true diff --git a/config/crd/bases/kjobctl.x-k8s.io_rayjobtemplates.yaml b/config/crd/bases/kjobctl.x-k8s.io_rayjobtemplates.yaml new file mode 100644 index 0000000..8a0405c --- /dev/null +++ b/config/crd/bases/kjobctl.x-k8s.io_rayjobtemplates.yaml @@ -0,0 +1,25625 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + name: rayjobtemplates.kjobctl.x-k8s.io +spec: + group: kjobctl.x-k8s.io + names: + kind: RayJobTemplate + listKind: RayJobTemplateList + plural: rayjobtemplates + singular: rayjobtemplate + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: RayJobTemplate is the Schema for the rayjobtemplate API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + template: + description: Template defines rayjobs that will be created from this rayjob + template. + properties: + metadata: + description: Standard object's metadata. + type: object + spec: + description: Specification of the desired behavior of the rayjob. + properties: + activeDeadlineSeconds: + description: |- + ActiveDeadlineSeconds is the duration in seconds that the RayJob may be active before + KubeRay actively tries to terminate the RayJob; value must be positive integer. + format: int32 + type: integer + backoffLimit: + default: 0 + description: |- + Specifies the number of retries before marking this job failed. + Each retry creates a new RayCluster. + format: int32 + type: integer + clusterSelector: + additionalProperties: + type: string + description: clusterSelector is used to select running rayclusters + by labels + type: object + entrypoint: + description: |- + INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + Important: Run "make" to regenerate code after modifying this file + type: string + entrypointNumCpus: + description: EntrypointNumCpus specifies the number of cpus to + reserve for the entrypoint command. + type: number + entrypointNumGpus: + description: EntrypointNumGpus specifies the number of gpus to + reserve for the entrypoint command. + type: number + entrypointResources: + description: |- + EntrypointResources specifies the custom resources and quantities to reserve for the + entrypoint command. + type: string + jobId: + description: If jobId is not set, a new jobId will be auto-generated. + type: string + metadata: + additionalProperties: + type: string + description: Metadata is data to store along with this job. + type: object + rayClusterSpec: + description: RayClusterSpec is the cluster template to run the + job + properties: + autoscalerOptions: + description: AutoscalerOptions specifies optional configuration + for the Ray autoscaler. + properties: + env: + description: Optional list of environment variables to + set in the autoscaler container. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the + pod's namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + description: Optional list of sources to populate environment + variables in the autoscaler container. + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + idleTimeoutSeconds: + description: |- + IdleTimeoutSeconds is the number of seconds to wait before scaling down a worker pod which is not using Ray resources. + Defaults to 60 (one minute). It is not read by the KubeRay operator but by the Ray autoscaler. + format: int32 + type: integer + image: + description: Image optionally overrides the autoscaler's + container image. This override is for provided for autoscaler + testing and development. + type: string + imagePullPolicy: + description: ImagePullPolicy optionally overrides the + autoscaler container's image pull policy. This override + is for provided for autoscaler testing and development. + type: string + resources: + description: |- + Resources specifies optional resource request and limit overrides for the autoscaler container. + Default values: 500m CPU request and limit. 512Mi memory request and limit. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + upscalingMode: + description: |- + UpscalingMode is "Conservative", "Default", or "Aggressive." + Conservative: Upscaling is rate-limited; the number of pending worker pods is at most the size of the Ray cluster. + Default: Upscaling is not rate-limited. + Aggressive: An alias for Default; upscaling is not rate-limited. + It is not read by the KubeRay operator but by the Ray autoscaler. + enum: + - Default + - Aggressive + - Conservative + type: string + volumeMounts: + description: Optional list of volumeMounts. This is needed + for enabling TLS for the autoscaler container. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + type: object + enableInTreeAutoscaling: + description: EnableInTreeAutoscaling indicates whether operator + should create in tree autoscaling configs + type: boolean + headGroupSpec: + description: |- + INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + Important: Run "make" to regenerate code after modifying this file + HeadGroupSpecs are the spec for the head pod + properties: + enableIngress: + description: EnableIngress indicates whether operator + should create ingress object for head service or not. + type: boolean + headService: + description: HeadService is the Kubernetes service of + the head pod. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + type: object + spec: + description: |- + Spec defines the behavior of a service. + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + allocateLoadBalancerNodePorts: + description: |- + allocateLoadBalancerNodePorts defines if NodePorts will be automatically + allocated for services with type LoadBalancer. Default is "true". It + may be set to "false" if the cluster load-balancer does not rely on + NodePorts. If the caller requests specific NodePorts (by specifying a + value), those requests will be respected, regardless of this field. + This field may only be set for services with type LoadBalancer and will + be cleared if the type is changed to any other type. + type: boolean + clusterIP: + description: |- + clusterIP is the IP address of the service and is usually assigned + randomly. If an address is specified manually, is in-range (as per + system configuration), and is not in use, it will be allocated to the + service; otherwise creation of the service will fail. This field may not + be changed through updates unless the type field is also being changed + to ExternalName (which requires this field to be blank) or the type + field is being changed from ExternalName (in which case this field may + optionally be specified, as describe above). Valid values are "None", + empty string (""), or a valid IP address. Setting this to "None" makes a + "headless service" (no virtual IP), which is useful when direct endpoint + connections are preferred and proxying is not required. Only applies to + types ClusterIP, NodePort, and LoadBalancer. If this field is specified + when creating a Service of type ExternalName, creation will fail. This + field will be wiped when updating a Service to type ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + clusterIPs: + description: |- + ClusterIPs is a list of IP addresses assigned to this service, and are + usually assigned randomly. If an address is specified manually, is + in-range (as per system configuration), and is not in use, it will be + allocated to the service; otherwise creation of the service will fail. + This field may not be changed through updates unless the type field is + also being changed to ExternalName (which requires this field to be + empty) or the type field is being changed from ExternalName (in which + case this field may optionally be specified, as describe above). Valid + values are "None", empty string (""), or a valid IP address. Setting + this to "None" makes a "headless service" (no virtual IP), which is + useful when direct endpoint connections are preferred and proxying is + not required. Only applies to types ClusterIP, NodePort, and + LoadBalancer. If this field is specified when creating a Service of type + ExternalName, creation will fail. This field will be wiped when updating + a Service to type ExternalName. If this field is not specified, it will + be initialized from the clusterIP field. If this field is specified, + clients must ensure that clusterIPs[0] and clusterIP have the same + value. + + This field may hold a maximum of two entries (dual-stack IPs, in either order). + These IPs must correspond to the values of the ipFamilies field. Both + clusterIPs and ipFamilies are governed by the ipFamilyPolicy field. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + description: |- + externalIPs is a list of IP addresses for which nodes in the cluster + will also accept traffic for this service. These IPs are not managed by + Kubernetes. The user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external load-balancers + that are not part of the Kubernetes system. + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + healthCheckNodePort: + description: |- + healthCheckNodePort specifies the healthcheck nodePort for the service. + This only applies when type is set to LoadBalancer and + externalTrafficPolicy is set to Local. If a value is specified, is + in-range, and is not in use, it will be used. If not specified, a value + will be automatically allocated. External systems (e.g. load-balancers) + can use this port to determine if a given node holds endpoints for this + service or not. If this field is specified when creating a Service + which does not need it, creation will fail. This field will be wiped + when updating a Service to no longer need it (e.g. changing type). + This field cannot be updated once set. + format: int32 + type: integer + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilies: + description: |- + IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this + service. This field is usually assigned automatically based on cluster + configuration and the ipFamilyPolicy field. If this field is specified + manually, the requested family is available in the cluster, + and ipFamilyPolicy allows it, it will be used; otherwise creation of + the service will fail. This field is conditionally mutable: it allows + for adding or removing a secondary IP family, but it does not allow + changing the primary IP family of the Service. Valid values are "IPv4" + and "IPv6". This field only applies to Services of types ClusterIP, + NodePort, and LoadBalancer, and does apply to "headless" services. + This field will be wiped when updating a Service to type ExternalName. + + This field may hold a maximum of two entries (dual-stack families, in + either order). These families must correspond to the values of the + clusterIPs field, if specified. Both clusterIPs and ipFamilies are + governed by the ipFamilyPolicy field. + items: + description: |- + IPFamily represents the IP Family (IPv4 or IPv6). This type is used + to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerIP: + description: |- + Only applies to Service Type: LoadBalancer. + This feature depends on whether the underlying cloud-provider supports specifying + the loadBalancerIP when a load balancer is created. + This field will be ignored if the cloud-provider does not support the feature. + Deprecated: This field was under-specified and its meaning varies across implementations. + Using it is non-portable and it may not support dual-stack. + Users are encouraged to use implementation-specific annotations when available. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + ports: + description: |- + The list of ports that are exposed by this service. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + items: + description: ServicePort contains information + on service's port. + properties: + appProtocol: + description: |- + The application protocol for this port. + This is used as a hint for implementations to offer richer behavior for protocols that they understand. + This field follows standard Kubernetes label syntax. + Valid values are either: + + * Un-prefixed protocol names - reserved for IANA standard service names (as per + RFC-6335 and https://www.iana.org/assignments/service-names). + + * Kubernetes-defined prefixed names: + * 'kubernetes.io/h2c' - HTTP/2 prior knowledge over cleartext as described in https://www.rfc-editor.org/rfc/rfc9113.html#name-starting-http-2-with-prior- + * 'kubernetes.io/ws' - WebSocket over cleartext as described in https://www.rfc-editor.org/rfc/rfc6455 + * 'kubernetes.io/wss' - WebSocket over TLS as described in https://www.rfc-editor.org/rfc/rfc6455 + + * Other protocols should use implementation-defined prefixed names such as + mycompany.com/my-custom-protocol. + type: string + name: + description: |- + The name of this port within the service. This must be a DNS_LABEL. + All ports within a ServiceSpec must have unique names. When considering + the endpoints for a Service, this must match the 'name' field in the + EndpointPort. + Optional if only one ServicePort is defined on this service. + type: string + nodePort: + description: |- + The port on each node on which this service is exposed when type is + NodePort or LoadBalancer. Usually assigned by the system. If a value is + specified, in-range, and not in use it will be used, otherwise the + operation will fail. If not specified, a port will be allocated if this + Service requires one. If this field is specified when creating a + Service which does not need it, creation will fail. This field will be + wiped when updating a Service to no longer need it (e.g. changing type + from NodePort to ClusterIP). + More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + format: int32 + type: integer + port: + description: The port that will be exposed + by this service. + format: int32 + type: integer + protocol: + default: TCP + description: |- + The IP protocol for this port. Supports "TCP", "UDP", and "SCTP". + Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the pods targeted by the service. + Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named port in the + target Pod's container ports. If this is not specified, the value + of the 'port' field is used (an identity map). + This field is ignored for services with clusterIP=None, and should be + omitted or set equal to the 'port' field. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + description: |- + publishNotReadyAddresses indicates that any agent which deals with endpoints for this + Service should disregard any indications of ready/not-ready. + The primary use case for setting this field is for a StatefulSet's Headless Service to + propagate SRV DNS records for its Pods for the purpose of peer discovery. + The Kubernetes controllers that generate Endpoints and EndpointSlice resources for + Services interpret this to mean that all endpoints are considered "ready" even if the + Pods themselves are not. Agents which consume only Kubernetes generated endpoints + through the Endpoints or EndpointSlice resources can safely assume this behavior. + type: boolean + selector: + additionalProperties: + type: string + description: |- + Route service traffic to pods with label keys and values matching this + selector. If empty or not present, the service is assumed to have an + external process managing its endpoints, which Kubernetes will not + modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the + configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + trafficDistribution: + description: |- + TrafficDistribution offers a way to express preferences for how traffic is + distributed to Service endpoints. Implementations can use this field as a + hint, but are not required to guarantee strict adherence. If the field is + not set, the implementation will apply its default routing strategy. If set + to "PreferClose", implementations should prioritize endpoints that are + topologically close (e.g., same zone). + This is an alpha field and requires enabling ServiceTrafficDistribution feature. + type: string + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + status: + description: |- + Most recently observed status of the service. + Populated by the system. + Read-only. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + conditions: + description: Current service state + items: + description: Condition contains details for + one aspect of the current state of this API + Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one + of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase + or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + loadBalancer: + description: |- + LoadBalancer contains the current status of the load-balancer, + if one is present. + properties: + ingress: + description: |- + Ingress is a list containing ingress points for the load-balancer. + Traffic intended for the service should be sent to these ingress points. + items: + description: |- + LoadBalancerIngress represents the status of a load-balancer ingress point: + traffic intended for the service should be sent to an ingress point. + properties: + hostname: + description: |- + Hostname is set for load-balancer ingress points that are DNS based + (typically AWS load-balancers) + type: string + ip: + description: |- + IP is set for load-balancer ingress points that are IP based + (typically GCE or OpenStack load-balancers) + type: string + ipMode: + description: |- + IPMode specifies how the load-balancer IP behaves, and may only be specified when the ip field is specified. + Setting this to "VIP" indicates that traffic is delivered to the node with + the destination set to the load-balancer's IP and port. + Setting this to "Proxy" indicates that traffic is delivered to the node or pod with + the destination set to the node's IP and node port or the pod's IP and port. + Service implementations may use this information to adjust traffic routing. + type: string + ports: + description: |- + Ports is a list of records of service ports + If used, every port defined in the service should have an entry in it + items: + properties: + error: + description: |- + Error is to record the problem with the service port + The format of the error shall comply with the following rules: + - built-in error values shall be specified in this file and those shall use + CamelCase names + - cloud provider specific error values must have names that comply with the + format foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port + number of the service port of + which status is recorded here + format: int32 + type: integer + protocol: + description: |- + Protocol is the protocol of the service port of which status is recorded here + The supported values are: "TCP", "UDP", "SCTP" + type: string + required: + - error + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + type: object + rayStartParams: + additionalProperties: + type: string + description: 'RayStartParams are the params of the start + command: node-manager-port, object-store-memory, ...' + type: object + serviceType: + description: ServiceType is Kubernetes service type of + the head service. it will be used by the workers to + connect to the head pod + type: string + template: + description: Template is the exact pod template used in + K8s depoyments, statefulsets, etc. + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling + constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling + rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, + associated with the corresponding + weight. + properties: + matchExpressions: + description: A list of node + selector requirements by node's + labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label + key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node + selector requirements by node's + fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label + key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with + matching the corresponding nodeSelectorTerm, + in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node + selector terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node + selector requirements by node's + labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label + key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node + selector requirements by node's + fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label + key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling + rules (e.g. co-locate this pod in the same + node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the + matched WeightedPodAffinityTerm fields + are added per-node to find the most + preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the + same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the + matched WeightedPodAffinityTerm fields + are added per-node to find the most + preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates + whether a service account token should be automatically + mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container + that you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment + variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used + if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of + a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the + schema the FieldPath is + written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of + a secret in the pod's namespace + properties: + key: + description: The key of the + secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents + the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier + to prepend to each key in the ConfigMap. + Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be in + the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents + a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind + the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be in + the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for + the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be in + the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of + block devices to be used by the container. + items: + description: volumeDevice describes a + mapping of a raw block device within + a container. + properties: + devicePath: + description: devicePath is the path + inside of the container that the + device will be mapped to. + type: string + name: + description: name must match the name + of a persistentVolumeClaim in the + pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name + of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines + DNS resolver options of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment + variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used + if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of + a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the + schema the FieldPath is + written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of + a secret in the pod's namespace + properties: + key: + description: The key of the + secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents + the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier + to prepend to each key in the ConfigMap. + Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for + ephemeral containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for + ephemeral containers. + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be in + the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral + containers. + items: + description: ContainerPort represents + a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind + the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for + ephemeral containers. + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be in + the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for + the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for + ephemeral containers. + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be in + the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of + block devices to be used by the container. + items: + description: volumeDevice describes a + mapping of a raw block device within + a container. + properties: + devicePath: + description: devicePath is the path + inside of the container that the + device will be mapped to. + type: string + name: + description: name must match the name + of a persistentVolumeClaim in the + pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name + of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP + address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file + entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container + that you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment + variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used + if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of + a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the + schema the FieldPath is + written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of + a secret in the pod's namespace + properties: + key: + description: The key of the + secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents + the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier + to prepend to each key in the ConfigMap. + Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be in + the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents + a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind + the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be in + the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for + the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be in + the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of + block devices to be used by the container. + items: + description: volumeDevice describes a + mapping of a raw block device within + a container. + properties: + devicePath: + description: devicePath is the path + inside of the container that the + device will be mapped to. + type: string + name: + description: name must match the name + of a persistentVolumeClaim in the + pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name + of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference + to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition + in the pod's condition list with matching + type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated + to a Pod to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter + to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to + set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies + how to spread matching pods among the given + topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume + in a pod that may be accessed by any container + in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure + Data Disk mount on the host and bind mount + to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host + Caching mode: None, Read Only, Read + Write.' + type: string + diskName: + description: diskName is the Name of + the data disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data + disk in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are + Shared: multiple blob disks per storage + account Dedicated: single blob disk + per storage account Managed: azure + managed data disk (only in managed + availability set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure + File Service mount on the host and bind + mount to the pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name + of secret that contains Azure Storage + Account Name and Key + type: string + shareName: + description: shareName is the azure + share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS + mount on the host that shares a pod's + lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used + as the mounted root, rather than the + full Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap + that should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key to + project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) + represents ephemeral storage that is handled + by certain external CSI drivers (Beta + feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward + API about the pod that should populate + this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward + API volume file + items: + description: DownwardAPIVolumeFile + represents information to create + the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only annotations, + labels, name, namespace and + uid are supported.' + properties: + apiVersion: + description: Version of the + schema the FieldPath is + written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file + to be created. Must not be absolute + or contain the ''..'' path. + Must be utf-8 encoded. The first + item of the relative path must + not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the + type of resource being + referenced + type: string + name: + description: Name is the + name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the + type of resource being + referenced + type: string + name: + description: Name is the + name of resource being + referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label + query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the + binding reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel + resource that is attached to a kubelet's + host machine and then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target + lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: + FC target worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the + driver to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this + field holds extra command options + if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker + volume attached to a kubelet's host machine. + This depends on the Flocker control service + being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID + of the dataset. This is unique identifier + of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit + hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines + whether support iSCSI Discovery CHAP + authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines + whether support iSCSI Session CHAP + authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI + Qualified Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target + Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret + for iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents + a PhotonController persistent disk attached + and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a + portworx volume attached and mounted on + kubelets host machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies + a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in + one resources secrets, configmaps, and + downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key + is the label key + that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path + from the volume root to + write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information + about the configMap data to + project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string + key to a path within a + volume. + properties: + key: + description: key is + the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify + whether the ConfigMap or + its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information + about the downwardAPI data to + project + properties: + items: + description: Items is a list + of DownwardAPIVolume file + items: + description: DownwardAPIVolumeFile + represents information + to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: + Selects a field of + the pod: only annotations, + labels, name, namespace + and uid are supported.' + properties: + apiVersion: + description: Version + of the schema + the FieldPath + is written in + terms of, defaults + to "v1". + type: string + fieldPath: + description: Path + of the field to + select in the + specified API + version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: + Path is the relative + path name of the file + to be created. Must + not be absolute or + contain the ''..'' + path. Must be utf-8 + encoded. The first + item of the relative + path must not start + with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required + for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies + the output format + of the exposed + resources, defaults + to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information + about the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string + key to a path within a + volume. + properties: + key: + description: key is + the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field + specify whether the Secret + or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken + is information about the serviceAccountToken + data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte + mount on the host that shares a pod's + lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that + references an already created Quobyte + volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO + persistent volume attached and mounted + on Kubernetes nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address + of the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the + name of the ScaleIO Protection Domain + for the configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default + false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO + Storage Pool associated with the protection + domain. + type: string + system: + description: system is the name of the + storage system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key to + project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify + whether the Secret or its keys must + be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS + volume attached and mounted on Kubernetes + nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a + vSphere volume attached and mounted on + kubelets host machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the + storage Policy Based Management (SPBM) + profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the + storage Policy Based Management (SPBM) + profile name. + type: string + volumePath: + description: volumePath is the path + that identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + required: + - rayStartParams + - template + type: object + headServiceAnnotations: + additionalProperties: + type: string + type: object + rayVersion: + description: RayVersion is used to determine the command for + the Kubernetes Job managed by RayJob + type: string + suspend: + description: |- + Suspend indicates whether a RayCluster should be suspended. + A suspended RayCluster will have head pods and worker pods deleted. + type: boolean + workerGroupSpecs: + description: WorkerGroupSpecs are the specs for the worker + pods + items: + description: WorkerGroupSpec are the specs for the worker + pods + properties: + groupName: + description: we can have multiple worker groups, we + distinguish them by name + type: string + maxReplicas: + default: 2147483647 + description: MaxReplicas denotes the maximum number + of desired Pods for this worker group, and the default + value is maxInt32. + format: int32 + type: integer + minReplicas: + default: 0 + description: MinReplicas denotes the minimum number + of desired Pods for this worker group. + format: int32 + type: integer + numOfHosts: + default: 1 + description: NumOfHosts denotes the number of hosts + to create per replica. The default value is 1. + format: int32 + type: integer + rayStartParams: + additionalProperties: + type: string + description: 'RayStartParams are the params of the start + command: address, object-store-memory, ...' + type: object + replicas: + default: 0 + description: Replicas is the number of desired Pods + for this worker group. See https://github.com/ray-project/kuberay/pull/1443 + for more details about the reason for making this + field optional. + format: int32 + type: integer + scaleStrategy: + description: ScaleStrategy defines which pods to remove + properties: + workersToDelete: + description: WorkersToDelete workers to be deleted + items: + type: string + type: array + type: object + template: + description: Template is a pod template for the worker + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling + constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling + rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, + associated with the corresponding + weight. + properties: + matchExpressions: + description: A list of node + selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label + key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node + selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label + key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated + with matching the corresponding + nodeSelectorTerm, in the range + 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of + node selector terms. The terms + are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node + selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label + key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node + selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label + key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling + rules (e.g. co-locate this pod in the + same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of + the matched WeightedPodAffinityTerm + fields are added per-node to find + the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key + is the label key + that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key + is the label key + that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity + scheduling rules (e.g. avoid putting this + pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of + the matched WeightedPodAffinityTerm + fields are added per-node to find + the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key + is the label key + that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key + is the label key + that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates + whether a service account token should be + automatically mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container + that you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment + variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used + if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of + a ConfigMap. + properties: + key: + description: The key to + select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of + a secret in the pod's namespace + properties: + key: + description: The key of + the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents + the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier + to prepend to each key in the + ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies + the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader + describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents + the duration that the container + should sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies + the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader + describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents + the duration that the container + should sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be + in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents + a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind + the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be + in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for + the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux + level label that applies to + the container. + type: string + role: + description: Role is a SELinux + role label that applies to the + container. + type: string + type: + description: Type is a SELinux + type label that applies to the + container. + type: string + user: + description: User is a SELinux + user label that applies to the + container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be + in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list + of block devices to be used by the container. + items: + description: volumeDevice describes + a mapping of a raw block device within + a container. + properties: + devicePath: + description: devicePath is the path + inside of the container that the + device will be mapped to. + type: string + name: + description: name must match the + name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a + mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the + Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines + DNS resolver options of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment + variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used + if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of + a ConfigMap. + properties: + key: + description: The key to + select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of + a secret in the pod's namespace + properties: + key: + description: The key of + the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents + the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier + to prepend to each key in the + ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed + for ephemeral containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies + the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader + describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents + the duration that the container + should sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies + the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader + describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents + the duration that the container + should sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for + ephemeral containers. + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be + in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for + ephemeral containers. + items: + description: ContainerPort represents + a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind + the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for + ephemeral containers. + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be + in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for + the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux + level label that applies to + the container. + type: string + role: + description: Role is a SELinux + role label that applies to the + container. + type: string + type: + description: Type is a SELinux + type label that applies to the + container. + type: string + user: + description: User is a SELinux + user label that applies to the + container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for + ephemeral containers. + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be + in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list + of block devices to be used by the container. + items: + description: volumeDevice describes + a mapping of a raw block device within + a container. + properties: + devicePath: + description: devicePath is the path + inside of the container that the + device will be mapped to. + type: string + name: + description: name must match the + name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a + mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the + Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP + address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file + entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container + that you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment + variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used + if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of + a ConfigMap. + properties: + key: + description: The key to + select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of + a secret in the pod's namespace + properties: + key: + description: The key of + the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents + the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier + to prepend to each key in the + ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies + the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader + describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents + the duration that the container + should sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies + the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader + describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents + the duration that the container + should sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be + in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents + a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind + the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be + in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for + the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux + level label that applies to + the container. + type: string + role: + description: Role is a SELinux + role label that applies to the + container. + type: string + type: + description: Type is a SELinux + type label that applies to the + container. + type: string + user: + description: User is a SELinux + user label that applies to the + container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be + in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list + of block devices to be used by the container. + items: + description: volumeDevice describes + a mapping of a raw block device within + a container. + properties: + devicePath: + description: devicePath is the path + inside of the container that the + device will be mapped to. + type: string + name: + description: name must match the + name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a + mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the + Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the + reference to a pod condition + properties: + conditionType: + description: ConditionType refers to a + condition in the pod's condition list + with matching type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated + to a Pod to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter + to be set + properties: + name: + description: Name of a property to + set + type: string + value: + description: Value of a property to + set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies + how to spread matching pods among the given + topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume + in a pod that may be accessed by any container + in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure + Data Disk mount on the host and bind + mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host + Caching mode: None, Read Only, Read + Write.' + type: string + diskName: + description: diskName is the Name + of the data disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of + data disk in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values + are Shared: multiple blob disks + per storage account Dedicated: + single blob disk per storage account Managed: + azure managed data disk (only in + managed availability set). defaults + to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure + File Service mount on the host and bind + mount to the pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name + of secret that contains Azure Storage + Account Name and Key + type: string + shareName: + description: shareName is the azure + share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph + FS mount on the host that shares a pod's + lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used + as the mounted root, rather than + the full Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap + that should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) + represents ephemeral storage that is + handled by certain external CSI drivers + (Beta feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward + API about the pod that should populate + this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward + API volume file + items: + description: DownwardAPIVolumeFile + represents information to create + the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only annotations, + labels, name, namespace and + uid are supported.' + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path + is the relative path name + of the file to be created. + Must not be absolute or contain + the ''..'' path. Must be utf-8 + encoded. The first item of + the relative path must not + start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the + type of resource being + referenced + type: string + name: + description: Name is the + name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the + type of resource being + referenced + type: string + name: + description: Name is the + name of resource being + referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a + label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key + is the label key + that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is + the binding reference to + the PersistentVolume backing + this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel + resource that is attached to a kubelet's + host machine and then exposed to the + pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC + target lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: + FC target worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of + the driver to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: + this field holds extra command options + if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker + volume attached to a kubelet's host + machine. This depends on the Flocker + control service being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID + of the dataset. This is unique identifier + of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit + hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines + whether support iSCSI Discovery + CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines + whether support iSCSI Session CHAP + authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI + Qualified Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI + Target Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP + Secret for iSCSI target and initiator + authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents + a PhotonController persistent disk attached + and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents + a portworx volume attached and mounted + on kubelets host machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies + a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in + one resources secrets, configmaps, and + downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label + selector requirements. + The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key + is the label + key that the + selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path + from the volume root to + write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information + about the configMap data to + project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string + key to a path within + a volume. + properties: + key: + description: key is + the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify + whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information + about the downwardAPI data + to project + properties: + items: + description: Items is a + list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile + represents information + to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: + Selects a field + of the pod: only + annotations, labels, + name, namespace + and uid are supported.' + properties: + apiVersion: + description: Version + of the schema + the FieldPath + is written in + terms of, defaults + to "v1". + type: string + fieldPath: + description: Path + of the field + to select in + the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: + Path is the relative + path name of the + file to be created. + Must not be absolute + or contain the ''..'' + path. Must be utf-8 + encoded. The first + item of the relative + path must not start + with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required + for volumes, + optional for + env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies + the output format + of the exposed + resources, defaults + to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information + about the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string + key to a path within + a volume. + properties: + key: + description: key is + the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field + specify whether the Secret + or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken + is information about the serviceAccountToken + data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte + mount on the host that shares a pod's + lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that + references an already created Quobyte + volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO + persistent volume attached and mounted + on Kubernetes nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address + of the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the + name of the ScaleIO Protection Domain + for the configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, + default false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO + Storage Pool associated with the + protection domain. + type: string + system: + description: system is the name of + the storage system as configured + in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify + whether the Secret or its keys must + be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS + volume attached and mounted on Kubernetes + nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents + a vSphere volume attached and mounted + on kubelets host machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the + storage Policy Based Management + (SPBM) profile ID associated with + the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is + the storage Policy Based Management + (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path + that identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + required: + - groupName + - maxReplicas + - minReplicas + - rayStartParams + - template + type: object + type: array + required: + - headGroupSpec + type: object + runtimeEnvYAML: + description: |- + RuntimeEnvYAML represents the runtime environment configuration + provided as a multi-line YAML string. + type: string + shutdownAfterJobFinishes: + description: ShutdownAfterJobFinishes will determine whether to + delete the ray cluster once rayJob succeed or failed. + type: boolean + submissionMode: + default: K8sJobMode + description: |- + SubmissionMode specifies how RayJob submits the Ray job to the RayCluster. + In "K8sJobMode", the KubeRay operator creates a submitter Kubernetes Job to submit the Ray job. + In "HTTPMode", the KubeRay operator sends a request to the RayCluster to create a Ray job. + type: string + submitterConfig: + description: Configurations of submitter k8s job. + properties: + backoffLimit: + description: BackoffLimit of the submitter k8s job. + format: int32 + type: integer + type: object + submitterPodTemplate: + description: SubmitterPodTemplate is the template for the pod + that will run `ray job submit`. + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling rules + for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, in + the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules + (e.g. co-locate this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added per-node + to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same node, + zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added per-node + to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates whether + a service account token should be automatically mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a + C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will be + mapped to. + type: string + name: + description: name must match the name of a + persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a + Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver + options of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a + C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral + containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral + containers. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will be + mapped to. + type: string + name: + description: name must match the name of a + persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a + Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a + C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will be + mapped to. + type: string + name: + description: name must match the name of a + persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a + Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference + to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition + in the pod's condition list with matching type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated to a Pod + to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to + be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies how + to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume in a pod + that may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data + Disk mount on the host and bind mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching + mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data + disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk + in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: + multiple blob disks per storage account Dedicated: + single blob disk per storage account Managed: + azure managed data disk (only in managed availability + set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File + Service mount on the host and bind mount to the + pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret + that contains Azure Storage Account Name and + Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount on + the host that shares a pod's lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the + mounted root, rather than the full Ceph tree, + default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that + should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents + ephemeral storage that is handled by certain external + CSI drivers (Beta feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API + about the pod that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API + volume file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. + Must not be absolute or contain the + ''..'' path. Must be utf-8 encoded. + The first item of the relative path + must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of + resource being referenced + type: string + name: + description: Name is the name of + resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of + resource being referenced + type: string + name: + description: Name is the name of + resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query + over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding + reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource + that is attached to a kubelet's host machine and + then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun + number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target + worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver + to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field + holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume + attached to a kubelet's host machine. This depends + on the Flocker control service being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the + dataset. This is unique identifier of a Flocker + dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for + the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether + support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified + Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun + number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for + iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets + host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx + volume attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a + Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources + secrets, configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the + volume root to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about + the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about + the downwardAPI data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile + represents information to create + the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only annotations, + labels, name, namespace and + uid are supported.' + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path + is the relative path name + of the file to be created. + Must not be absolute or contain + the ''..'' path. Must be utf-8 + encoded. The first item of + the relative path must not + start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about + the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify + whether the Secret or its key must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information + about the serviceAccountToken data to + project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte mount + on the host that shares a pod's lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references + an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent + volume attached and mounted on Kubernetes nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of + the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of + the ScaleIO Protection Domain for the configured + storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage + Pool associated with the protection domain. + type: string + system: + description: system is the name of the storage + system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether + the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere + volume attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile ID + associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage + Policy Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies + vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + suspend: + description: |- + suspend specifies whether the RayJob controller should create a RayCluster instance + If a job is applied with the suspend field set to true, + the RayCluster will not be created and will wait for the transition to false. + If the RayCluster is already created, it will be deleted. + In case of transition to false a new RayCluster will be created. + type: boolean + ttlSecondsAfterFinished: + default: 0 + description: |- + TTLSecondsAfterFinished is the TTL to clean up RayCluster. + It's only working when ShutdownAfterJobFinishes set to true. + format: int32 + type: integer + type: object + required: + - spec + type: object + required: + - template + type: object + served: true + storage: true diff --git a/config/crd/bases/kjobctl.x-k8s.io_volumebundles.yaml b/config/crd/bases/kjobctl.x-k8s.io_volumebundles.yaml new file mode 100644 index 0000000..8e1d4b4 --- /dev/null +++ b/config/crd/bases/kjobctl.x-k8s.io_volumebundles.yaml @@ -0,0 +1,2007 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + name: volumebundles.kjobctl.x-k8s.io +spec: + group: kjobctl.x-k8s.io + names: + kind: VolumeBundle + listKind: VolumeBundleList + plural: volumebundles + singular: volumebundle + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: VolumeBundle is the Schema for the volumebundles API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: VolumeBundleSpec defines the desired state of VolumeBundle + properties: + containerVolumeMounts: + description: containerVolumeMounts is a list of locations in each + container of a pod where the volumes will be mounted. + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + envVars: + description: |- + envVars are environment variables that refer to absolute paths in the container filesystem. + These key/value pairs will be available in containers as environment variables. + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + volumes: + description: volumes is a set of volumes that will be added to all + pods of the job. + items: + description: Volume represents a named volume in a pod that may + be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data Disk mount on + the host and bind mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching mode: None, + Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk in the + blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in the blob + storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple + blob disks per storage account Dedicated: single blob + disk per storage account Managed: azure managed data + disk (only in managed availability set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File Service mount + on the host and bind mount to the pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that contains + Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount on the host that + shares a pod's lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the mounted root, + rather than the full Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that should populate + this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap or its + keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents ephemeral + storage that is handled by certain external CSI drivers (Beta + feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about the pod + that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API volume file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: + only annotations, labels, name, namespace and uid + are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative path + name of the file to be created. Must not be absolute + or contain the ''..'' path. Must be utf-8 encoded. + The first item of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource that is + attached to a kubelet's host machine and then exposed to the + pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide + names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver to use for + this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds extra + command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume attached to + a kubelet's host machine. This depends on the Flocker control + service being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. This + is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the specified + revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support iSCSI + Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support iSCSI + Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI target + and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon Controller + persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx volume attached + and mounted on kubelets host machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources secrets, + configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the volume root + to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about the configMap + data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about the downwardAPI + data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must + not be absolute or contain the ''..'' + path. Must be utf-8 encoded. The first + item of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults + to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about the secret data + to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify whether the + Secret or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information about + the serviceAccountToken data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte mount on the host + that shares a pod's lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of the ScaleIO + API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the ScaleIO + Protection Domain for the configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage Pool associated + with the protection domain. + type: string + system: + description: system is the name of the storage system as + configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether the Secret or + its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy Based + Management (SPBM) profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage Policy Based + Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies vSphere + volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containerVolumeMounts + - volumes + type: object + type: object + served: true + storage: true diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml new file mode 100644 index 0000000..d3e935f --- /dev/null +++ b/config/crd/kustomization.yaml @@ -0,0 +1,15 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/kjobctl.x-k8s.io_volumebundles.yaml +- bases/kjobctl.x-k8s.io_jobtemplates.yaml +- bases/kjobctl.x-k8s.io_rayjobtemplates.yaml +- bases/kjobctl.x-k8s.io_applicationprofiles.yaml +- bases/kjobctl.x-k8s.io_rayclustertemplates.yaml +# +kubebuilder:scaffold:crdkustomizeresource + +patches: [] + +#configurations: +#- kustomizeconfig.yaml diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml new file mode 100644 index 0000000..4c2bb18 --- /dev/null +++ b/config/default/kustomization.yaml @@ -0,0 +1,18 @@ +# Adds namespace to all resources. +namespace: kjobctl-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: kjobctl- + +# Labels to add to all resources and selectors. +#labels: +#- includeSelectors: true +# pairs: +# someName: someValue + +resources: +- ../crd diff --git a/config/samples/README.md b/config/samples/README.md new file mode 100644 index 0000000..ab35943 --- /dev/null +++ b/config/samples/README.md @@ -0,0 +1,3 @@ +RayJob sample are straight from KubeRay (https://raw.githubusercontent.com/ray-project/kuberay/v1.1.1/ray-operator/config/samples/ray-job.sample.yaml). + +RayCluster sample are straight from KubeRay (https://raw.githubusercontent.com/ray-project/kuberay/master/ray-operator/config/samples/ray-cluster.sample.yaml). diff --git a/config/samples/interactive-sample.yaml b/config/samples/interactive-sample.yaml new file mode 100644 index 0000000..aa6c50b --- /dev/null +++ b/config/samples/interactive-sample.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: ApplicationProfile +metadata: + name: interactive-profile + namespace: default +spec: + supportedModes: + - name: Interactive + template: interactive-template +--- +apiVersion: v1 +kind: PodTemplate +metadata: + name: interactive-template + namespace: default +template: + spec: + containers: + - name: sample-container + image: busybox:1.28 + command: ['/bin/sh'] \ No newline at end of file diff --git a/config/samples/job-sample.yaml b/config/samples/job-sample.yaml new file mode 100644 index 0000000..674a379 --- /dev/null +++ b/config/samples/job-sample.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: ApplicationProfile +metadata: + name: job-profile + namespace: default +spec: + supportedModes: + - name: Job + template: job-template +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: JobTemplate +metadata: + name: job-template + namespace: default +template: + spec: + parallelism: 3 + completions: 3 + template: + spec: + containers: + - name: sample-container + image: gcr.io/k8s-staging-perf-tests/sleep:v0.1.0 + args: [ "30s" ] + resources: + requests: + cpu: "1" + memory: "200Mi" + restartPolicy: Never diff --git a/config/samples/ray-cluster-sample.yaml b/config/samples/ray-cluster-sample.yaml new file mode 100644 index 0000000..a263931 --- /dev/null +++ b/config/samples/ray-cluster-sample.yaml @@ -0,0 +1,120 @@ +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: ApplicationProfile +metadata: + name: ray-cluster-profile + namespace: default +spec: + supportedModes: + - name: RayCluster + template: ray-cluster-template + volumeBundles: ["ray-cluster-volume-bundle"] +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: RayClusterTemplate +metadata: + name: ray-cluster-template + namespace: default +template: + spec: + rayVersion: '2.9.0' # should match the Ray version in the image of the containers + # Ray head pod template + headGroupSpec: + rayStartParams: { } + #pod template + template: + spec: + containers: + - name: ray-head + image: rayproject/ray:2.9.0 + resources: + limits: + cpu: 1 + memory: 2Gi + requests: + cpu: 500m + memory: 2Gi + ports: + - containerPort: 6379 + name: gcs-server + - containerPort: 8265 # Ray dashboard + name: dashboard + - containerPort: 10001 + name: client + workerGroupSpecs: + # the pod replicas in this group typed worker + - replicas: 1 + minReplicas: 1 + maxReplicas: 5 + # logical group name, for this called small-group, also can be functional + groupName: small-group + rayStartParams: { } + #pod template + template: + spec: + containers: + - name: ray-worker # must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character (e.g. 'my-name', or '123-abc' + image: rayproject/ray:2.9.0 + resources: + limits: + cpu: 1 + memory: 1Gi + requests: + cpu: 500m + memory: 1Gi +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: VolumeBundle +metadata: + name: ray-cluster-volume-bundle + namespace: default +spec: + volumes: + - name: ray-cluster-code-sample + configMap: + name: ray-cluster-code-sample + items: + - key: sample_code.py + path: sample_code.py + containerVolumeMounts: + - name: ray-cluster-code-sample + mountPath: /home/ray/samples + envVars: + - name: ENTRYPOINT_PATH + value: /home/ray/samples +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ray-cluster-code-sample + namespace: default +data: + sample_code.py: | + import ray + import os + import requests + + ray.init() + + @ray.remote + class Counter: + def __init__(self): + # Used to verify runtimeEnv + self.name = os.getenv("counter_name") + assert self.name == "test_counter" + self.counter = 0 + + def inc(self): + self.counter += 1 + + def get_counter(self): + return "{} got {}".format(self.name, self.counter) + + counter = Counter.remote() + + for _ in range(5): + ray.get(counter.inc.remote()) + print(ray.get(counter.get_counter.remote())) + + # Verify that the correct runtime env was used for the job. + assert requests.__version__ == "2.26.0" \ No newline at end of file diff --git a/config/samples/ray-job-sample.yaml b/config/samples/ray-job-sample.yaml new file mode 100644 index 0000000..3d731fb --- /dev/null +++ b/config/samples/ray-job-sample.yaml @@ -0,0 +1,159 @@ +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: ApplicationProfile +metadata: + name: ray-job-profile + namespace: default +spec: + supportedModes: + - name: RayJob + template: ray-job-template + volumeBundles: ["ray-job-volume-bundle"] +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: RayJobTemplate +metadata: + name: ray-job-template + namespace: default +template: + spec: + # submissionMode specifies how RayJob submits the Ray job to the RayCluster. + # The default value is "K8sJobMode", meaning RayJob will submit the Ray job via a submitter Kubernetes Job. + # The alternative value is "HTTPMode", indicating that KubeRay will submit the Ray job by sending an HTTP request to the RayCluster. + # submissionMode: "K8sJobMode" + entrypoint: python ${ENTRYPOINT_PATH}/sample_code.py + # shutdownAfterJobFinishes specifies whether the RayCluster should be deleted after the RayJob finishes. Default is false. + # shutdownAfterJobFinishes: false + + # ttlSecondsAfterFinished specifies the number of seconds after which the RayCluster will be deleted after the RayJob finishes. + # ttlSecondsAfterFinished: 10 + + # activeDeadlineSeconds is the duration in seconds that the RayJob may be active before + # KubeRay actively tries to terminate the RayJob; value must be positive integer. + # activeDeadlineSeconds: 120 + + # RuntimeEnvYAML represents the runtime environment configuration provided as a multi-line YAML string. + # See https://docs.ray.io/en/latest/ray-core/handling-dependencies.html for details. + # (New in KubeRay version 1.0.) + runtimeEnvYAML: | + pip: + - requests==2.26.0 + - pendulum==2.1.2 + env_vars: + counter_name: "test_counter" + + # Suspend specifies whether the RayJob controller should create a RayCluster instance. + # If a job is applied with the suspend field set to true, the RayCluster will not be created and we will wait for the transition to false. + # If the RayCluster is already created, it will be deleted. In the case of transition to false, a new RayCluster will be created. + # suspend: false + + # rayClusterSpec specifies the RayCluster instance to be created by the RayJob controller. + rayClusterSpec: + rayVersion: '2.9.0' # should match the Ray version in the image of the containers + # Ray head pod template + headGroupSpec: + # The `rayStartParams` are used to configure the `ray start` command. + # See https://github.com/ray-project/kuberay/blob/master/docs/guidance/rayStartParams.md for the default settings of `rayStartParams` in KubeRay. + # See https://docs.ray.io/en/latest/cluster/cli.html#ray-start for all available options in `rayStartParams`. + rayStartParams: + dashboard-host: '0.0.0.0' + #pod template + template: + spec: + containers: + - name: ray-head + image: rayproject/ray:2.9.0 + ports: + - containerPort: 6379 + name: gcs-server + - containerPort: 8265 # Ray dashboard + name: dashboard + - containerPort: 10001 + name: client + resources: + limits: + cpu: "1" + requests: + cpu: "200m" + workerGroupSpecs: + # the pod replicas in this group typed worker + - replicas: 1 + minReplicas: 1 + maxReplicas: 5 + # logical group name, for this called small-group, also can be functional + groupName: small-group + # The `rayStartParams` are used to configure the `ray start` command. + # See https://github.com/ray-project/kuberay/blob/master/docs/guidance/rayStartParams.md for the default settings of `rayStartParams` in KubeRay. + # See https://docs.ray.io/en/latest/cluster/cli.html#ray-start for all available options in `rayStartParams`. + rayStartParams: {} + #pod template + template: + spec: + containers: + - name: ray-worker # must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character (e.g. 'my-name', or '123-abc' + image: rayproject/ray:2.9.0 + lifecycle: + preStop: + exec: + command: [ "/bin/sh","-c","ray stop" ] + resources: + limits: + cpu: "1" + requests: + cpu: "200m" +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: VolumeBundle +metadata: + name: ray-job-volume-bundle + namespace: default +spec: + volumes: + - name: ray-job-code-sample + configMap: + name: ray-job-code-sample + items: + - key: sample_code.py + path: sample_code.py + containerVolumeMounts: + - name: ray-job-code-sample + mountPath: /home/ray/samples + envVars: + - name: ENTRYPOINT_PATH + value: /home/ray/samples +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ray-job-code-sample + namespace: default +data: + sample_code.py: | + import ray + import os + import requests + + ray.init() + + @ray.remote + class Counter: + def __init__(self): + # Used to verify runtimeEnv + self.name = os.getenv("counter_name") + assert self.name == "test_counter" + self.counter = 0 + + def inc(self): + self.counter += 1 + + def get_counter(self): + return "{} got {}".format(self.name, self.counter) + + counter = Counter.remote() + + for _ in range(5): + ray.get(counter.inc.remote()) + print(ray.get(counter.get_counter.remote())) + + # Verify that the correct runtime env was used for the job. + assert requests.__version__ == "2.26.0" \ No newline at end of file diff --git a/config/samples/slurm/script.sh b/config/samples/slurm/script.sh new file mode 100644 index 0000000..176956a --- /dev/null +++ b/config/samples/slurm/script.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +#SBATCH --array=1-3%2 + +echo "now processing task id: ${SLURM_ARRAY_TASK_ID}" +python /home/slurm/samples/sample_code.py diff --git a/config/samples/slurm/slurm-sample.yaml b/config/samples/slurm/slurm-sample.yaml new file mode 100644 index 0000000..674fcda --- /dev/null +++ b/config/samples/slurm/slurm-sample.yaml @@ -0,0 +1,64 @@ +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: ApplicationProfile +metadata: + name: slurm-profile + namespace: default +spec: + supportedModes: + - name: Slurm + template: slurm-template + volumeBundles: ["slurm-volume-bundle"] +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: JobTemplate +metadata: + name: slurm-template + namespace: default +template: + spec: + parallelism: 3 + completions: 3 + completionMode: Indexed + template: + spec: + containers: + - name: sample-container + image: python:3-slim + restartPolicy: OnFailure +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: VolumeBundle +metadata: + name: slurm-volume-bundle + namespace: default +spec: + volumes: + - name: slurm-code-sample + configMap: + name: slurm-code-sample + items: + - key: sample_code.py + path: sample_code.py + containerVolumeMounts: + - name: slurm-code-sample + mountPath: /home/slurm/samples + envVars: + - name: ENTRYPOINT_PATH + value: /home/slurm/samples +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: slurm-code-sample + namespace: default +data: + sample_code.py: | + import time + + print('start at ' + time.strftime('%H:%M:%S')) + + print('sleep for 10 seconds ...') + time.sleep(10) + + print('stop at ' + time.strftime('%H:%M:%S')) diff --git a/docs/_index.md b/docs/_index.md new file mode 100644 index 0000000..b45f998 --- /dev/null +++ b/docs/_index.md @@ -0,0 +1,24 @@ +# Kubectl Kjob Plugin + +The `kubectl-kjob` plugin, `kjobctl`, allows you to list, describe and create jobs. + +## Syntax + +Use the following syntax to run `kubectl kjob` commands from your terminal window: + +```shell +kubectl kjob [OPERATION] [TYPE] [NAME] [flags] +``` + +or with shorter syntax `kjobctl`: + +```shell +kjobctl [OPERATION] [TYPE] [NAME] [flags] +``` + +You can go to the [commands](commands/_index.md) or run `kubectl kjob help` in the terminal to get the full list of commands, along with all possible flags. + +## See Also + +* [installation](installation.md) - Installation guide for the `kubectl-kjob` plugin, `kjobctl`. +* [commands](commands/kjobctl.md) - Full list of commands, along with all possible flags. \ No newline at end of file diff --git a/docs/commands/kjobctl.md b/docs/commands/kjobctl.md new file mode 100644 index 0000000..a449321 --- /dev/null +++ b/docs/commands/kjobctl.md @@ -0,0 +1,207 @@ + + +# kjobctl + + +## Synopsis + + +ML/AI/Batch Jobs Made Easy + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
-h, --help
+

help for kjobctl

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl create](kjobctl_create/_index.md) - Create a task +* [kjobctl delete](kjobctl_delete/_index.md) - Delete resources +* [kjobctl describe](kjobctl_describe/_index.md) - Show details of a specific resource or group of resources. +* [kjobctl list](kjobctl_list/_index.md) - Display resources +* [kjobctl printcrds](kjobctl_printcrds/_index.md) - Print the kjobctl CRDs + diff --git a/docs/commands/kjobctl_create/_index.md b/docs/commands/kjobctl_create/_index.md new file mode 100644 index 0000000..20c6ced --- /dev/null +++ b/docs/commands/kjobctl_create/_index.md @@ -0,0 +1,261 @@ + + +# kjobctl create + + +## Synopsis + + +Create a task + + +## Examples + +``` + # Create job + kjobctl create job \ + --profile my-application-profile \ + --cmd "sleep 5" \ + --parallelism 4 \ + --completions 4 \ + --request cpu=500m,memory=4Gi \ + --localqueue my-local-queue-name + + # Create interactive + kjobctl create interactive \ + --profile my-application-profile \ + --pod-running-timeout 30s \ + --rm + + # Create rayjob + kjobctl create rayjob \ + --profile my-application-profile \ + --cmd "python /home/ray/samples/sample_code.py" \ + --replicas small-group=1 \ + --min-replicas small-group=1 \ + --max-replicas small-group=5 \ + --localqueue my-local-queue-name + + # Create raycluster + kjobctl create raycluster \ + --profile my-application-profile \ + --replicas small-group=1 \ + --min-replicas small-group=1 \ + --max-replicas small-group=5 \ + --localqueue my-local-queue-name + + # Create slurm + kjobctl create slurm --profile my-application-profile -- \ + --array 0-5 --nodes 3 --ntasks 1 ./script.sh +``` + + +## Options + + + + + + + + + + + + + + + + +
-h, --help
+

help for create

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl](../kjobctl.md) - ML/AI/Batch Jobs Made Easy +* [kjobctl create interactive](kjobctl_create_interactive.md) - Create an interactive shell +* [kjobctl create job](kjobctl_create_job.md) - Create a job +* [kjobctl create raycluster](kjobctl_create_raycluster.md) - Create a raycluster +* [kjobctl create rayjob](kjobctl_create_rayjob.md) - Create a rayjob +* [kjobctl create slurm](kjobctl_create_slurm.md) - Create a slurm job + diff --git a/docs/commands/kjobctl_create/kjobctl_create_interactive.md b/docs/commands/kjobctl_create/kjobctl_create_interactive.md new file mode 100644 index 0000000..a4b3c58 --- /dev/null +++ b/docs/commands/kjobctl_create/kjobctl_create_interactive.md @@ -0,0 +1,369 @@ + + +# kjobctl create interactive + + +## Synopsis + + +Create an interactive shell + +``` +kjobctl create interactive --profile APPLICATION_PROFILE_NAME [--localqueue LOCAL_QUEUE_NAME] [--skip-localqueue-validation] [--priority NAME] [--skip-priority-validation] [--cmd COMMAND] [--request RESOURCE_NAME=QUANTITY] [--pod-running-timeout DURATION] [--time TIME_LIMIT] [--rm] +``` + + +## Examples + +``` + # Create interactive + kjobctl create interactive \ + --profile my-application-profile \ + --pod-running-timeout 30s \ + --rm +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--cmd string
+

Command which is associated with the resource.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
-h, --help
+

help for interactive

+
--localqueue string
+

Kueue localqueue name which is associated with the resource.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--pod-running-timeout duration     Default: 1m0s
+

The length of time (like 5s, 2m, or 3h, higher than zero) to wait until at least one pod is running.

+
--priority string
+

Apply priority for the entire workload.

+
-p, --profile string
+

Application profile contains a template (with defaults set) for running a specific type of application.

+
--request <comma-separated 'key=value' pairs>     Default: []
+

Request is a set of (resource name, quantity) pairs.

+
--rm
+

Remove pod when interactive session exits.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--skip-localqueue-validation
+

Skip local queue validation. Add local queue even if the queue does not exist.

+
--skip-priority-validation
+

Skip workload priority class validation. Add priority class label even if the class does not exist.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
-t, --time string
+

Set a limit on the total run time of the job. +A time limit of zero requests that no time limit be imposed. +Acceptable time formats include "minutes", "minutes:seconds", +"hours:minutes:seconds", "days-hours", "days-hours:minutes" +and "days-hours:minutes:seconds".

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_create](_index.md) - Create a task + diff --git a/docs/commands/kjobctl_create/kjobctl_create_job.md b/docs/commands/kjobctl_create/kjobctl_create_job.md new file mode 100644 index 0000000..b55a153 --- /dev/null +++ b/docs/commands/kjobctl_create/kjobctl_create_job.md @@ -0,0 +1,372 @@ + + +# kjobctl create job + + +## Synopsis + + +Create a job + +``` +kjobctl create job --profile APPLICATION_PROFILE_NAME [--localqueue LOCAL_QUEUE_NAME] [--skip-localqueue-validation] [--priority NAME] [--skip-priority-validation] [--cmd COMMAND] [--request RESOURCE_NAME=QUANTITY] [--parallelism PARALLELISM] [--completions COMPLETIONS] [--time TIME_LIMIT] +``` + + +## Examples + +``` + # Create job + kjobctl create job \ + --profile my-application-profile \ + --cmd "sleep 5" \ + --parallelism 4 \ + --completions 4 \ + --request cpu=500m,memory=4Gi \ + --localqueue my-local-queue-name +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--cmd string
+

Command which is associated with the resource.

+
--completions int
+

Completions specifies the desired number of successfully finished pods.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
-h, --help
+

help for job

+
--localqueue string
+

Kueue localqueue name which is associated with the resource.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--parallelism int
+

Parallelism specifies the maximum desired number of pods the job should run at any given time.

+
--priority string
+

Apply priority for the entire workload.

+
-p, --profile string
+

Application profile contains a template (with defaults set) for running a specific type of application.

+
--request <comma-separated 'key=value' pairs>     Default: []
+

Request is a set of (resource name, quantity) pairs.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--skip-localqueue-validation
+

Skip local queue validation. Add local queue even if the queue does not exist.

+
--skip-priority-validation
+

Skip workload priority class validation. Add priority class label even if the class does not exist.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
-t, --time string
+

Set a limit on the total run time of the job. +A time limit of zero requests that no time limit be imposed. +Acceptable time formats include "minutes", "minutes:seconds", +"hours:minutes:seconds", "days-hours", "days-hours:minutes" +and "days-hours:minutes:seconds".

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_create](_index.md) - Create a task + diff --git a/docs/commands/kjobctl_create/kjobctl_create_raycluster.md b/docs/commands/kjobctl_create/kjobctl_create_raycluster.md new file mode 100644 index 0000000..8015b5d --- /dev/null +++ b/docs/commands/kjobctl_create/kjobctl_create_raycluster.md @@ -0,0 +1,364 @@ + + +# kjobctl create raycluster + + +## Synopsis + + +Create a raycluster. + + KubeRay operator is required for RayCluster. How to install KubeRay operator you can find here https://ray-project.github.io/kuberay/deploy/installation/. + +``` +kjobctl create raycluster --profile APPLICATION_PROFILE_NAME [--localqueue LOCAL_QUEUE_NAME] [--skip-localqueue-validation] [--priority NAME] [--skip-priority-validation] [--replicas [WORKER_GROUP]=REPLICAS] [--min-replicas [WORKER_GROUP]=MIN_REPLICAS] [--max-replicas [WORKER_GROUP]=MAX_REPLICAS] [--time TIME_LIMIT] +``` + + +## Examples + +``` + # Create raycluster + kjobctl create raycluster \ + --profile my-application-profile \ + --replicas small-group=1 \ + --min-replicas small-group=1 \ + --max-replicas small-group=5 \ + --localqueue my-local-queue-name +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
-h, --help
+

help for raycluster

+
--localqueue string
+

Kueue localqueue name which is associated with the resource.

+
--max-replicas <comma-separated 'key=int' pairs>     Default: []
+

MaxReplicas denotes the maximum number of desired Pods for this worker group, and the default value is maxInt32.

+
--min-replicas <comma-separated 'key=int' pairs>     Default: []
+

MinReplicas denotes the minimum number of desired Pods for this worker group.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--priority string
+

Apply priority for the entire workload.

+
-p, --profile string
+

Application profile contains a template (with defaults set) for running a specific type of application.

+
--replicas <comma-separated 'key=int' pairs>     Default: []
+

Replicas is the number of desired Pods for this worker group.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--skip-localqueue-validation
+

Skip local queue validation. Add local queue even if the queue does not exist.

+
--skip-priority-validation
+

Skip workload priority class validation. Add priority class label even if the class does not exist.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
-t, --time string
+

Set a limit on the total run time of the job. +A time limit of zero requests that no time limit be imposed. +Acceptable time formats include "minutes", "minutes:seconds", +"hours:minutes:seconds", "days-hours", "days-hours:minutes" +and "days-hours:minutes:seconds".

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_create](_index.md) - Create a task + diff --git a/docs/commands/kjobctl_create/kjobctl_create_rayjob.md b/docs/commands/kjobctl_create/kjobctl_create_rayjob.md new file mode 100644 index 0000000..a192549 --- /dev/null +++ b/docs/commands/kjobctl_create/kjobctl_create_rayjob.md @@ -0,0 +1,383 @@ + + +# kjobctl create rayjob + + +## Synopsis + + +Create a rayjob. + + KubeRay operator is required for RayJob. How to install KubeRay operator you can find here https://ray-project.github.io/kuberay/deploy/installation/. + +``` +kjobctl create rayjob --profile APPLICATION_PROFILE_NAME [--localqueue LOCAL_QUEUE_NAME] [--skip-localqueue-validation] [--priority NAME] [--skip-priority-validation] [--cmd COMMAND] [--replicas [WORKER_GROUP]=REPLICAS] [--min-replicas [WORKER_GROUP]=MIN_REPLICAS] [--max-replicas [WORKER_GROUP]=MAX_REPLICAS] [--time TIME_LIMIT] +``` + + +## Examples + +``` + # Create rayjob + kjobctl create rayjob \ + --profile my-application-profile \ + --cmd "python /home/ray/samples/sample_code.py" \ + --replicas small-group=1 \ + --min-replicas small-group=1 \ + --max-replicas small-group=5 \ + --localqueue my-local-queue-name +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--cmd string
+

Command which is associated with the resource.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
-h, --help
+

help for rayjob

+
--localqueue string
+

Kueue localqueue name which is associated with the resource.

+
--max-replicas <comma-separated 'key=int' pairs>     Default: []
+

MaxReplicas denotes the maximum number of desired Pods for this worker group, and the default value is maxInt32.

+
--min-replicas <comma-separated 'key=int' pairs>     Default: []
+

MinReplicas denotes the minimum number of desired Pods for this worker group.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--priority string
+

Apply priority for the entire workload.

+
-p, --profile string
+

Application profile contains a template (with defaults set) for running a specific type of application.

+
--raycluster string
+

Existing ray cluster on which the job will be created.

+
--replicas <comma-separated 'key=int' pairs>     Default: []
+

Replicas is the number of desired Pods for this worker group.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--skip-localqueue-validation
+

Skip local queue validation. Add local queue even if the queue does not exist.

+
--skip-priority-validation
+

Skip workload priority class validation. Add priority class label even if the class does not exist.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
-t, --time string
+

Set a limit on the total run time of the job. +A time limit of zero requests that no time limit be imposed. +Acceptable time formats include "minutes", "minutes:seconds", +"hours:minutes:seconds", "days-hours", "days-hours:minutes" +and "days-hours:minutes:seconds".

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_create](_index.md) - Create a task + diff --git a/docs/commands/kjobctl_create/kjobctl_create_slurm.md b/docs/commands/kjobctl_create/kjobctl_create_slurm.md new file mode 100644 index 0000000..6f80852 --- /dev/null +++ b/docs/commands/kjobctl_create/kjobctl_create_slurm.md @@ -0,0 +1,354 @@ + + +# kjobctl create slurm + + +## Synopsis + + +Create a slurm job + +``` +kjobctl create slurm --profile APPLICATION_PROFILE_NAME [--localqueue LOCAL_QUEUE_NAME] [--skip-localqueue-validation] [--priority NAME] [--skip-priority-validation] [--ignore-unknown-flags] [--init-image IMAGE] [--first-node-ip] [--first-node-ip-timeout DURATION] -- [--array ARRAY] [--cpus-per-task QUANTITY] [--gpus-per-task QUANTITY] [--mem QUANTITY] [--mem-per-task QUANTITY] [--mem-per-cpu QUANTITY] [--mem-per-gpu QUANTITY] [--nodes COUNT] [--ntasks COUNT] [--output FILENAME_PATTERN] [--error FILENAME_PATTERN] [--input FILENAME_PATTERN] [--job-name NAME] [--partition NAME] SCRIPT +``` + + +## Examples + +``` + # Create slurm + kjobctl create slurm --profile my-application-profile -- \ + --array 0-5 --nodes 3 --ntasks 1 ./script.sh +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
--first-node-ip
+

Enable the retrieval of the first node's IP address.

+
--first-node-ip-timeout duration     Default: 1m0s
+

The timeout for the retrieval of the first node's IP address.

+
-h, --help
+

help for slurm

+
--ignore-unknown-flags
+

Ignore all the unsupported flags in the bash script.

+
--init-image string     Default: "registry.k8s.io/busybox:1.27.2"
+

The image used for the init container.

+
--localqueue string
+

Kueue localqueue name which is associated with the resource.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--priority string
+

Apply priority for the entire workload.

+
-p, --profile string
+

Application profile contains a template (with defaults set) for running a specific type of application.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--skip-localqueue-validation
+

Skip local queue validation. Add local queue even if the queue does not exist.

+
--skip-priority-validation
+

Skip workload priority class validation. Add priority class label even if the class does not exist.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_create](_index.md) - Create a task + diff --git a/docs/commands/kjobctl_delete/_index.md b/docs/commands/kjobctl_delete/_index.md new file mode 100644 index 0000000..0197832 --- /dev/null +++ b/docs/commands/kjobctl_delete/_index.md @@ -0,0 +1,237 @@ + + +# kjobctl delete + + +## Synopsis + + +Delete resources + + +## Examples + +``` + # Delete interactive shell + kjobctl delete interactive my-application-profile-interactive-k2wzd + + # Delete Job + kjobctl delete job my-application-profile-job-k2wzd + + # Delete RayJob + kjobctl delete rayjob my-application-profile-rayjob-k2wzd + + # Delete RayCluster + kjobctl delete raycluster my-application-profile-raycluster-k2wzd +``` + + +## Options + + + + + + + + + + + + + + + + +
-h, --help
+

help for delete

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl](../kjobctl.md) - ML/AI/Batch Jobs Made Easy +* [kjobctl delete interactive](kjobctl_delete_interactive.md) - Delete interactive shell +* [kjobctl delete job](kjobctl_delete_job.md) - Delete Job +* [kjobctl delete raycluster](kjobctl_delete_raycluster.md) - Delete RayCluster +* [kjobctl delete rayjob](kjobctl_delete_rayjob.md) - Delete RayJob +* [kjobctl delete slurm](kjobctl_delete_slurm.md) - Delete Slurm + diff --git a/docs/commands/kjobctl_delete/kjobctl_delete_interactive.md b/docs/commands/kjobctl_delete/kjobctl_delete_interactive.md new file mode 100644 index 0000000..3499a37 --- /dev/null +++ b/docs/commands/kjobctl_delete/kjobctl_delete_interactive.md @@ -0,0 +1,281 @@ + + +# kjobctl delete interactive + + +## Synopsis + + +Delete interactive shell + +``` +kjobctl delete interactive NAME [--cascade STRATEGY] [--dry-run STRATEGY] +``` + + +## Examples + +``` + # Delete interactive shell + kjobctl delete interactive my-application-profile-interactive-k2wzd +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--cascade string     Default: "background"
+

Must be "background", "orphan", or "foreground". Defaults to background.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
-h, --help
+

help for interactive

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_delete](_index.md) - Delete resources + diff --git a/docs/commands/kjobctl_delete/kjobctl_delete_job.md b/docs/commands/kjobctl_delete/kjobctl_delete_job.md new file mode 100644 index 0000000..752252d --- /dev/null +++ b/docs/commands/kjobctl_delete/kjobctl_delete_job.md @@ -0,0 +1,281 @@ + + +# kjobctl delete job + + +## Synopsis + + +Delete Job + +``` +kjobctl delete job NAME [--cascade STRATEGY] [--dry-run STRATEGY] +``` + + +## Examples + +``` + # Delete Job + kjobctl delete job my-application-profile-job-k2wzd +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--cascade string     Default: "background"
+

Must be "background", "orphan", or "foreground". Defaults to background.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
-h, --help
+

help for job

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_delete](_index.md) - Delete resources + diff --git a/docs/commands/kjobctl_delete/kjobctl_delete_raycluster.md b/docs/commands/kjobctl_delete/kjobctl_delete_raycluster.md new file mode 100644 index 0000000..8ddca7e --- /dev/null +++ b/docs/commands/kjobctl_delete/kjobctl_delete_raycluster.md @@ -0,0 +1,281 @@ + + +# kjobctl delete raycluster + + +## Synopsis + + +Delete RayCluster + +``` +kjobctl delete raycluster NAME [--cascade STRATEGY] [--dry-run STRATEGY] +``` + + +## Examples + +``` + # Delete RayCluster + kjobctl delete raycluster my-application-profile-raycluster-k2wzd +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--cascade string     Default: "background"
+

Must be "background", "orphan", or "foreground". Defaults to background.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
-h, --help
+

help for raycluster

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_delete](_index.md) - Delete resources + diff --git a/docs/commands/kjobctl_delete/kjobctl_delete_rayjob.md b/docs/commands/kjobctl_delete/kjobctl_delete_rayjob.md new file mode 100644 index 0000000..1f09351 --- /dev/null +++ b/docs/commands/kjobctl_delete/kjobctl_delete_rayjob.md @@ -0,0 +1,281 @@ + + +# kjobctl delete rayjob + + +## Synopsis + + +Delete RayJob + +``` +kjobctl delete rayjob NAME [--cascade STRATEGY] [--dry-run STRATEGY] +``` + + +## Examples + +``` + # Delete RayJob + kjobctl delete rayjob my-application-profile-rayjob-k2wzd +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--cascade string     Default: "background"
+

Must be "background", "orphan", or "foreground". Defaults to background.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
-h, --help
+

help for rayjob

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_delete](_index.md) - Delete resources + diff --git a/docs/commands/kjobctl_delete/kjobctl_delete_slurm.md b/docs/commands/kjobctl_delete/kjobctl_delete_slurm.md new file mode 100644 index 0000000..0c46583 --- /dev/null +++ b/docs/commands/kjobctl_delete/kjobctl_delete_slurm.md @@ -0,0 +1,281 @@ + + +# kjobctl delete slurm + + +## Synopsis + + +Delete Slurm + +``` +kjobctl delete slurm NAME [--cascade STRATEGY] [--dry-run STRATEGY] +``` + + +## Examples + +``` + # Delete Slurm + kjobctl delete slurm my-application-profile-slurm-k2wzd +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--cascade string     Default: "background"
+

Must be "background", "orphan", or "foreground". Defaults to background.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
-h, --help
+

help for slurm

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_delete](_index.md) - Delete resources + diff --git a/docs/commands/kjobctl_describe/_index.md b/docs/commands/kjobctl_describe/_index.md new file mode 100644 index 0000000..f6ad10c --- /dev/null +++ b/docs/commands/kjobctl_describe/_index.md @@ -0,0 +1,263 @@ + + +# kjobctl describe + + +## Synopsis + + +Show details of a specific resource or group of resources. + +``` +kjobctl describe MODE NAME +``` + + +## Examples + +``` + # Describe a task with job mode + kjobctl describe job sample-job + + # Describe a task with job mode + kjobctl describe job/sample-job + + # Describe all tasks with job mode + kjobctl describe job + + # Describe tasks by label name=myLabel + kjobctl describe job -l name=myLabel +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
-A, --all-namespaces
+

If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.

+
-h, --help
+

help for describe

+
-p, --profile string
+

Filter by profile name which is associated with the resource.

+
-l, --selector string
+

Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints.

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl](../kjobctl.md) - ML/AI/Batch Jobs Made Easy + diff --git a/docs/commands/kjobctl_list/_index.md b/docs/commands/kjobctl_list/_index.md new file mode 100644 index 0000000..593d582 --- /dev/null +++ b/docs/commands/kjobctl_list/_index.md @@ -0,0 +1,228 @@ + + +# kjobctl list + + +## Synopsis + + +Display resources + + +## Examples + +``` + # List Job + kjobctl list job +``` + + +## Options + + + + + + + + + + + + + + + + +
-h, --help
+

help for list

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl](../kjobctl.md) - ML/AI/Batch Jobs Made Easy +* [kjobctl list interactive](kjobctl_list_interactive.md) - List Interactive +* [kjobctl list job](kjobctl_list_job.md) - List Job +* [kjobctl list raycluster](kjobctl_list_raycluster.md) - List RayCluster +* [kjobctl list rayjob](kjobctl_list_rayjob.md) - List RayJob +* [kjobctl list slurm](kjobctl_list_slurm.md) - List Slurm + diff --git a/docs/commands/kjobctl_list/kjobctl_list_interactive.md b/docs/commands/kjobctl_list/kjobctl_list_interactive.md new file mode 100644 index 0000000..b0a3cba --- /dev/null +++ b/docs/commands/kjobctl_list/kjobctl_list_interactive.md @@ -0,0 +1,311 @@ + + +# kjobctl list interactive + + +## Synopsis + + +List Interactive + +``` +kjobctl list interactive [--profile PROFILE_NAME] [--localqueue LOCALQUEUE_NAME] [--selector key1=value1] [--field-selector key1=value1] [--all-namespaces] +``` + + +## Examples + +``` + # List Interactive + kjobctl list interactive + + # List Interactive with profile filter + kjobctl list interactive --profile my-profile +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
-A, --all-namespaces
+

If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.

+
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--field-selector string
+

Selector (field query) to filter on, supports '=', '==', and '!='.(e.g. --field-selector key1=value1,key2=value2). The server only supports a limited number of field queries per type.

+
-h, --help
+

help for interactive

+
-q, --localqueue string
+

Filter by localqueue which is associated with the resource.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
-p, --profile string
+

Filter by profile name which is associated with the resource.

+
-l, --selector string
+

Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_list](_index.md) - Display resources + diff --git a/docs/commands/kjobctl_list/kjobctl_list_job.md b/docs/commands/kjobctl_list/kjobctl_list_job.md new file mode 100644 index 0000000..10c3300 --- /dev/null +++ b/docs/commands/kjobctl_list/kjobctl_list_job.md @@ -0,0 +1,311 @@ + + +# kjobctl list job + + +## Synopsis + + +List Job + +``` +kjobctl list job [--profile PROFILE_NAME] [--localqueue LOCALQUEUE_NAME] [--selector key1=value1] [--field-selector key1=value1] [--all-namespaces] +``` + + +## Examples + +``` + # List Job + kjobctl list job + + # List Job with profile filter + kjobctl list job --profile my-profile +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
-A, --all-namespaces
+

If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.

+
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--field-selector string
+

Selector (field query) to filter on, supports '=', '==', and '!='.(e.g. --field-selector key1=value1,key2=value2). The server only supports a limited number of field queries per type.

+
-h, --help
+

help for job

+
-q, --localqueue string
+

Filter by localqueue which is associated with the resource.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
-p, --profile string
+

Filter by profile name which is associated with the resource.

+
-l, --selector string
+

Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_list](_index.md) - Display resources + diff --git a/docs/commands/kjobctl_list/kjobctl_list_raycluster.md b/docs/commands/kjobctl_list/kjobctl_list_raycluster.md new file mode 100644 index 0000000..49d981f --- /dev/null +++ b/docs/commands/kjobctl_list/kjobctl_list_raycluster.md @@ -0,0 +1,311 @@ + + +# kjobctl list raycluster + + +## Synopsis + + +List RayCluster + +``` +kjobctl list raycluster [--profile PROFILE_NAME] [--localqueue LOCALQUEUE_NAME] [--selector key1=value1] [--field-selector key1=value1] [--all-namespaces] +``` + + +## Examples + +``` + # List RayCluster + kjobctl list raycluster + + # List RayCluster with profile filter + kjobctl list raycluster --profile my-profile +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
-A, --all-namespaces
+

If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.

+
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--field-selector string
+

Selector (field query) to filter on, supports '=', '==', and '!='.(e.g. --field-selector key1=value1,key2=value2). The server only supports a limited number of field queries per type.

+
-h, --help
+

help for raycluster

+
-q, --localqueue string
+

Filter by localqueue which is associated with the resource.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
-p, --profile string
+

Filter by profile name which is associated with the resource.

+
-l, --selector string
+

Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_list](_index.md) - Display resources + diff --git a/docs/commands/kjobctl_list/kjobctl_list_rayjob.md b/docs/commands/kjobctl_list/kjobctl_list_rayjob.md new file mode 100644 index 0000000..5d28f86 --- /dev/null +++ b/docs/commands/kjobctl_list/kjobctl_list_rayjob.md @@ -0,0 +1,311 @@ + + +# kjobctl list rayjob + + +## Synopsis + + +List RayJob + +``` +kjobctl list rayjob [--profile PROFILE_NAME] [--localqueue LOCALQUEUE_NAME] [--selector key1=value1] [--field-selector key1=value1] [--all-namespaces] +``` + + +## Examples + +``` + # List RayJob + kjobctl list rayjob + + # List RayJob with profile filter + kjobctl list rayjob --profile my-profile +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
-A, --all-namespaces
+

If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.

+
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--field-selector string
+

Selector (field query) to filter on, supports '=', '==', and '!='.(e.g. --field-selector key1=value1,key2=value2). The server only supports a limited number of field queries per type.

+
-h, --help
+

help for rayjob

+
-q, --localqueue string
+

Filter by localqueue which is associated with the resource.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
-p, --profile string
+

Filter by profile name which is associated with the resource.

+
-l, --selector string
+

Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_list](_index.md) - Display resources + diff --git a/docs/commands/kjobctl_list/kjobctl_list_slurm.md b/docs/commands/kjobctl_list/kjobctl_list_slurm.md new file mode 100644 index 0000000..a00088e --- /dev/null +++ b/docs/commands/kjobctl_list/kjobctl_list_slurm.md @@ -0,0 +1,311 @@ + + +# kjobctl list slurm + + +## Synopsis + + +List Slurm + +``` +kjobctl list slurm [--profile PROFILE_NAME] [--localqueue LOCALQUEUE_NAME] [--selector key1=value1] [--field-selector key1=value1] [--all-namespaces] +``` + + +## Examples + +``` + # List Slurm + kjobctl list slurm + + # List Slurm with profile filter + kjobctl list slurm --profile my-profile +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
-A, --all-namespaces
+

If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.

+
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--field-selector string
+

Selector (field query) to filter on, supports '=', '==', and '!='.(e.g. --field-selector key1=value1,key2=value2). The server only supports a limited number of field queries per type.

+
-h, --help
+

help for slurm

+
-q, --localqueue string
+

Filter by localqueue which is associated with the resource.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
-p, --profile string
+

Filter by profile name which is associated with the resource.

+
-l, --selector string
+

Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_list](_index.md) - Display resources + diff --git a/docs/commands/kjobctl_printcrds/_index.md b/docs/commands/kjobctl_printcrds/_index.md new file mode 100644 index 0000000..b42e234 --- /dev/null +++ b/docs/commands/kjobctl_printcrds/_index.md @@ -0,0 +1,230 @@ + + +# kjobctl printcrds + + +## Synopsis + + +Print the kjobctl CRDs + +``` +kjobctl printcrds [flags] +``` + + +## Examples + +``` + # Install or update the kjobctl CRDs + kjobctl printcrds | kubectl apply --server-side -f - + + # Remove the kjobctl CRDs + kjobctl printcrds | kubectl delete --ignore-not-found=true -f - +``` + + +## Options + + + + + + + + + + + + + + + + +
-h, --help
+

help for printcrds

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl](../kjobctl.md) - ML/AI/Batch Jobs Made Easy + diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..4b874d6 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,59 @@ +# Installation + +Installing the `kubectl-kjob` plugin, `kjobctl`. + +## Installing + +### From source + +```bash +make kubectl-kjob +sudo mv ./bin/kubectl-kjob /usr/local/bin/kubectl-kjob +``` + +## Installing CRDs + +### Using printcrds command + +```bash +kubectl-kjob printcrds | kubectl apply --server-side -f - +``` + +### From source + +```bash +make install +``` + +## Kjobctl + +Additionally, you can create an alias `kjobctl` to allow shorter syntax. + +```bash +echo 'alias kjobctl="kubectl kjob"' >> ~/.bashrc +# Or if you are using ZSH +echo 'alias kjobctl="kubectl kjob"' >> ~/.zshrc +``` + +## Autocompletion + +```bash +echo '[[ $commands[kubectl-kjob] ]] && source <(kubectl-kjob completion bash)' >> ~/.bashrc +# Or if you are using ZSH +echo '[[ $commands[kubectl-kjob] ]] && source <(kubectl-kjob completion zsh)' >> ~/.zshrc + +cat <kubectl_complete-kjob +#!/usr/bin/env sh + +# Call the __complete command passing it all arguments +kubectl kjob __complete "\$@" +EOF + +chmod u+x kubectl_complete-kjob +sudo mv kubectl_complete-kjob /usr/local/bin/kubectl_complete-kjob +``` + +## See Also + +* [overview](_index.md) - `kubectl-kjob` plugin, `kjobctl` overview. +* [commands](commands/kjobctl.md) - Full list of commands, along with all possible flags. \ No newline at end of file diff --git a/docs/run_slurm.md b/docs/run_slurm.md new file mode 100644 index 0000000..fc4c7b7 --- /dev/null +++ b/docs/run_slurm.md @@ -0,0 +1,246 @@ +# Run a Slurm Job on Kubernetes + +This page demonstrates how to run a Slurm job on a Kubernetes cluster using Kjob. + +## Before You Begin + +Ensure the following prerequisites are met: + +- A Kubernetes cluster is operational. +- The [kjobctl](installation.md) plugin is installed. + +## Slurm Mode in Kjob + +This mode is specifically designed to offer a compatibility mode for [Slurm](https://slurm.schedmd.com/) jobs. Kjob does not directly incorporate Slurm's components. + +Instead, it simulates Slurm behavior to facilitate a smooth transition for users. + +By translating Slurm concepts such as nodes into Kubernetes-compatible terms like pods, Kjob ensures that users familiar with Slurm will find the Kjob environment intuitive and easy to use. + +The addition of the Slurm mode to the ApplicationProfile in Kjob aims to replicate the experience of using the sbatch command in Slurm as closely as possible, thereby enhancing user satisfaction and efficiency in managing cluster jobs within a Kubernetes context. + +### Supported Options + +Kjob provides support for executing Slurm scripts by offering several options that can be specified either through the command line interface, `Kjobctl`, or via the `#SBATCH` directive embedded within the bash script. These options are detailed in the table below: + +| Option | Description | +|---------------------|-------------| +| -a, --array | See [array option](https://slurm.schedmd.com/sbatch.html#OPT_array) for the specification. | +| -c, --cpus-per-task | Specifies how many CPUs a container inside a pod requires. | +| -e, --error | Specifies where to redirect the standard error stream of a task. If not passed, it proceeds to stdout and is available via `kubectl logs`. | +| --gpus-per-task | Specifies how many GPUs a container inside a pod requires. | +| -i, --input | Specifies what to pipe into the script. | +| -J, --job-name= | Specifies the job name. | +| --mem | Specifies how much memory a pod requires. | +| --mem-per-cpu | Specifies how much memory a container requires, multiplying the number of requested CPUs per task by mem-per-cpu. | +| --mem-per-gpu | Specifies how much memory a container requires, multiplying the number of requested GPUs per task by mem-per-gpu. | +| --mem-per-task | Specifies how much memory a container requires. | +| -N, --nodes | Specifies the number of pods to be used at a time - parallelism in indexed jobs. | +| -n, --ntasks | Specifies the number of identical containers inside of a pod, usually 1. | +| -o, --output | Specifies where to redirect the standard output stream of a task. If not passed, it proceeds to stdout and is available via `kubectl logs`. | +| --partition | Specifies the local queue name. See [Local Queue](https://kueue.sigs.k8s.io/docs/concepts/local_queue/) for more information. | +| -D, --chdir | Change directory before executing the script. | +| -t, --time | Set a limit on the total run time of the job. A time limit of zero requests that no time limit be imposed. Acceptable time formats include "minutes", "minutes:seconds", "hours:minutes:seconds", "days-hours", "days-hours:minutes" and "days-hours:minutes:seconds". | + +If an unsupported flag is passed in the script, the command will fail with an error unless `--ignore-unknown-flags` is given. + +### Supported Input Environment Variables + +> NOTE: Environment variables will override any options set in a batch script, and command line +> options will override any environment variables. + +| Name | Description | +|-----------------------|-------------------------| +| $SBATCH_ARRAY_INX | Same as -a, --array | +| $SBATCH_GPUS_PER_TASK | Same as --gpus-per-task | +| $SBATCH_MEM_PER_NODE | Same as --mem | +| $SBATCH_MEM_PER_CPU | Same as --mem-per-cpu | +| $SBATCH_MEM_PER_GPU | Same as --mem-per-gpu | +| $SBATCH_OUTPUT | Same as -o, --output | +| $SBATCH_ERROR | Same as -e, --error | +| $SBATCH_INPUT | Same as -i, --input | +| $SBATCH_JOB_NAME | Same as -J, --job-name | +| $SBATCH_PARTITION | Same as -p, --partition | +| $SBATCH_TIMELIMIT | Same as -t, --time | + +### Supported Output Environment Variables + +| Name | Description | +|------------------------------|-------------| +| $SLURM_ARRAY_TASK_ID | Job array ID (index) number. | +| $SLURM_JOB_ID | The Job ID. | +| $SLURM_JOBID | Deprecated. Same as $SLURM_JOB_ID. | +| $SLURM_SUBMIT_DIR | The path of the job submission directory. | +| $SLURM_SUBMIT_HOST | The hostname of the node used for job submission. | +| $SLURM_JOB_NODELIST | Contains the definition (list) of the nodes (actually pods) that is assigned to the job. | +| $SLURM_JOB_FIRST_NODE | First element of SLURM_JOB_NODELIST. | +| $SLURM_JOB_FIRST_NODE_IP | IP of the first element, obtained via nslookup. | +| $SLURM_CPUS_PER_TASK | Number of CPUs per task. | +| $SLURM_CPUS_ON_NODE | Number of CPUs on the allocated node (actually pod). | +| $SLURM_JOB_CPUS_PER_NODE | Count of processors available to the job on this node. | +| $SLURM_CPUS_PER_GPU | Number of CPUs requested per allocated GPU. | +| $SLURM_MEM_PER_CPU | Memory per CPU. Same as --mem-per-cpu. | +| $SLURM_MEM_PER_GPU | Memory per GPU. | +| $SLURM_MEM_PER_NODE | Memory per node. Same as --mem. | +| $SLURM_GPUS | Number of GPUs requested (in total). | +| $SLURM_NTASKS | Same as -n, --ntasks. The number of tasks. | +| $SLURM_NTASKS_PER_NODE | Number of tasks requested per node. | +| $SLURM_NTASKS_PER_SOCKET | Number of tasks requested per socket. To be supported later. | +| $SLURM_NTASKS_PER_CORE | Number of tasks requested per core. To be supported later. | +| $SLURM_NTASKS_PER_GPU | Number of tasks requested per GPU. To be supported later. | +| $SLURM_NPROCS | Same as -n, --ntasks. See $SLURM_NTASKS. | +| $SLURM_NNODES | Total number of nodes (actually pods) in the job’s resource allocation. | +| $SLURM_TASKS_PER_NODE | Number of tasks to be initiated on each node. | +| $SLURM_ARRAY_JOB_ID | Job array’s master job ID number. For now, same as $SLURM_JOB_ID. | +| $SLURM_ARRAY_TASK_COUNT | Total number of tasks in a job array. | +| $SLURM_ARRAY_TASK_MAX | Job array’s maximum ID (index) number. | +| $SLURM_ARRAY_TASK_MIN | Job array’s minimum ID (index) number. | + +## Example + +The following example demonstrates a use case for running a Python script. + +See [config/samples/slurm](../config/samples/slurm/) for the full samples used in this section. + +### 1. Create Slurm Template + +First, you need to create a template for your job using the `JobTemplate` kind. This template should specify the container image with all the dependencies to run your task. Below is an example of a template to run a Python script. + +```yaml +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: JobTemplate +metadata: + name: slurm-template + namespace: default +template: + spec: + parallelism: 3 + completions: 3 + completionMode: Indexed + template: + spec: + containers: + - name: sample-container + image: python:3-slim + restartPolicy: OnFailure +``` + +Once you have created your template, you need an `ApplicationProfile` containing `Slurm` and your new template in the supported modes as shown below: + +```yaml +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: ApplicationProfile +metadata: + name: slurm-profile + namespace: default +spec: + supportedModes: + - name: Slurm + template: slurm-template + volumeBundles: ["slurm-volume-bundle"] +``` + +Then, save the file as `application_profile.yaml` and create the ApplicationProfile by running: + +```bash +kubectl create -f application_profile.yaml +``` + +> Note: This setup process, including the creation of the JobTemplate and ApplicationProfile, only needs to be completed once and can be applied to subsequent Python script executions. + +### 2. Prepare the Python Script for the Job + +Before loading the Python script for use with the pods, it's crucial to consider how the script will be made accessible. + +For the purposes of this tutorial, we'll utilize a ConfigMap to store and pass the Python script. While this method simplifies the setup and is suitable for demonstration purposes, for a scalable setup, using NFS or a similar shared file system is necessary. + +#### Creating a ConfigMap with Your Python Script + +Create a file named `sample_code.py` and add your Python script. For example: + +```python +import time + +print('Start at ' + time.strftime('%H:%M:%S')) + +print('Sleep for 10 seconds...') +time.sleep(10) + +print('Stop at ' + time.strftime('%H:%M:%S')) +``` + +To create a `ConfigMap` from this Python script, save it and then run the following command: + +```bash +kubectl create configmap slurm-code-sample --from-file=sample_code.py +``` + +This command packages your script into a `ConfigMap` named `slurm-code-sample`. + +### 3. Integrate the Script with a VolumeBundle + +With your Python script now encapsulated in a `ConfigMap`, the next step is to make it accessible to your Slurm pods through a `VolumeBundle`. + +#### Creating a VolumeBundle + +Define a `VolumeBundle` object to specify how the containers should mount the volumes containing your script: + +```yaml +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: VolumeBundle +metadata: + name: slurm-volume-bundle + namespace: default +spec: + volumes: + - name: slurm-code-sample + configMap: + name: slurm-code-sample + items: + - key: sample_code.py + path: sample_code.py + containerVolumeMounts: + - name: slurm-code-sample + mountPath: /home/slurm/samples + envVars: + - name: ENTRYPOINT_PATH + value: /home/slurm/samples +``` + +Save this definition as a file named `slurm_volume_bundle.yaml`. And then, apply this configuration to your Kubernetes cluster by running: + +```bash +kubectl create -f slurm_volume_bundle.yaml +``` + +This setup mounts the `ConfigMap` as a volume at `/home/slurm/samples` on the pods, making your Python script readily accessible to be executed as part of your Slurm job. + +### 4. Submit Job + +After setting up the `ApplicationProfile` and `VolumeBundle`, you're ready to run tasks using your template. To submit a job, you need to pass a batch script to the `create slurm` command. You can run your script with custom configurations by the supported options via the `#SBATCH` directive or flags. + +Below is an example of a simple batch script which wraps the Python script loaded in the previous step and executes it. Additionally, it sets the array option to run 3 multiple jobs and the maximum number of simultaneously running tasks to 2. + +```bash +#!/bin/bash + +#SBATCH --array=1-3%2 + +echo "Now processing task ID: ${SLURM_ARRAY_TASK_ID}" +python /home/slurm/samples/sample_code.py +``` + +To run this batch script, save it as `slurm-sample.sh` and execute: + +```bash +kubectl-kjob create slurm --profile slurm-profile -- slurm-sample.sh +``` + +Now, after submitting the job, you should be able to see something similar to the following output in the logs: + +``` +Now processing task ID: 1 +Start at 17:08:04 +Sleep for 10 seconds... +Stop at 17:08:14 +``` diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6effa8d --- /dev/null +++ b/go.mod @@ -0,0 +1,102 @@ +module sigs.k8s.io/kueue/cmd/experimental/kjobctl + +go 1.23.0 + +require ( + github.com/google/go-cmp v0.6.0 + github.com/onsi/ginkgo/v2 v2.22.0 + github.com/onsi/gomega v1.36.0 + github.com/ray-project/kuberay/ray-operator v1.2.2 + github.com/spf13/cobra v1.8.1 + github.com/spf13/pflag v1.0.5 + k8s.io/api v0.31.3 + k8s.io/apimachinery v0.31.3 + k8s.io/cli-runtime v0.31.3 + k8s.io/client-go v0.31.3 + k8s.io/component-base v0.31.3 + k8s.io/klog/v2 v2.130.1 + k8s.io/kubectl v0.31.3 + k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 + sigs.k8s.io/controller-runtime v0.19.2 + sigs.k8s.io/kueue v0.9.1 +) + +require ( + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/MakeNowJust/heredoc v1.0.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/chai2010/gettext-go v1.0.2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect + github.com/fatih/camelcase v1.0.0 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-errors/errors v1.4.2 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.0.1 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.1 // indirect + github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect + github.com/imdario/mergo v0.3.16 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/moby/spdystream v0.4.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.57.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + github.com/xlab/treeprint v1.2.0 // indirect + go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/term v0.25.0 // indirect + golang.org/x/text v0.19.0 // indirect + golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.26.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/protobuf v1.35.1 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.31.1 // indirect + k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/kustomize/api v0.17.3 // indirect + sigs.k8s.io/kustomize/kyaml v0.17.2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..0c29ac5 --- /dev/null +++ b/go.sum @@ -0,0 +1,328 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= +github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= +github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= +github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jarcoal/httpmock v1.2.0 h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc= +github.com/jarcoal/httpmock v1.2.0/go.mod h1:oCoTsnAz4+UoOUIf5lJOWV2QQIW5UoeUI6aM2YnWAZk= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= +github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8= +github.com/moby/spdystream v0.4.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.36.0 h1:Pb12RlruUtj4XUuPUqeEWc6j5DkVVVA49Uf6YLfC95Y= +github.com/onsi/gomega v1.36.0/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVhoNcY= +github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/ray-project/kuberay/ray-operator v1.2.2 h1:wj4qe9SmJfD1ubgEaVPuAsnU/WFDvremzR8j3JslBdk= +github.com/ray-project/kuberay/ray-operator v1.2.2/go.mod h1:osTiIyaDoWi5IN1f0tOOtZ4TzVf+5kJXZor8VFvcEiI= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= +go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/api v0.31.3 h1:umzm5o8lFbdN/hIXbrK9oRpOproJO62CV1zqxXrLgk8= +k8s.io/api v0.31.3/go.mod h1:UJrkIp9pnMOI9K2nlL6vwpxRzzEX5sWgn8kGQe92kCE= +k8s.io/apiextensions-apiserver v0.31.1 h1:L+hwULvXx+nvTYX/MKM3kKMZyei+UiSXQWciX/N6E40= +k8s.io/apiextensions-apiserver v0.31.1/go.mod h1:tWMPR3sgW+jsl2xm9v7lAyRF1rYEK71i9G5dRtkknoQ= +k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4= +k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/cli-runtime v0.31.3 h1:fEQD9Xokir78y7pVK/fCJN090/iYNrLHpFbGU4ul9TI= +k8s.io/cli-runtime v0.31.3/go.mod h1:Q2jkyTpl+f6AtodQvgDI8io3jrfr+Z0LyQBPJJ2Btq8= +k8s.io/client-go v0.31.3 h1:CAlZuM+PH2cm+86LOBemaJI/lQ5linJ6UFxKX/SoG+4= +k8s.io/client-go v0.31.3/go.mod h1:2CgjPUTpv3fE5dNygAr2NcM8nhHzXvxB8KL5gYc3kJs= +k8s.io/component-base v0.31.3 h1:DMCXXVx546Rfvhj+3cOm2EUxhS+EyztH423j+8sOwhQ= +k8s.io/component-base v0.31.3/go.mod h1:xME6BHfUOafRgT0rGVBGl7TuSg8Z9/deT7qq6w7qjIU= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f h1:bnWtxXWdAl5bVOCEPoNdvMkyj6cTW3zxHuwKIakuV9w= +k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f/go.mod h1:G0W3eI9gG219NHRq3h5uQaRBl4pj4ZpwzRP5ti8y770= +k8s.io/kubectl v0.31.3 h1:3r111pCjPsvnR98oLLxDMwAeM6OPGmPty6gSKaLTQes= +k8s.io/kubectl v0.31.3/go.mod h1:lhMECDCbJN8He12qcKqs2QfmVo9Pue30geovBVpH5fs= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.19.2 h1:3sPrF58XQEPzbE8T81TN6selQIMGbtYwuaJ6eDssDF8= +sigs.k8s.io/controller-runtime v0.19.2/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/kueue v0.9.1 h1:4y8YCoWyZj/C1rOWLvjn9T3Hmjfe+YD7Syhii1B92Fo= +sigs.k8s.io/kueue v0.9.1/go.mod h1:9t5SwZ0XH/xK0+XSuLea4stzCC/PIja5JpFZzip2/Oo= +sigs.k8s.io/kustomize/api v0.17.3 h1:6GCuHSsxq7fN5yhF2XrC+AAr8gxQwhexgHflOAD/JJU= +sigs.k8s.io/kustomize/api v0.17.3/go.mod h1:TuDH4mdx7jTfK61SQ/j1QZM/QWR+5rmEiNjvYlhzFhc= +sigs.k8s.io/kustomize/kyaml v0.17.2 h1:+AzvoJUY0kq4QAhH/ydPHHMRLijtUKiyVyh7fOSshr0= +sigs.k8s.io/kustomize/kyaml v0.17.2/go.mod h1:9V0mCjIEYjlXuCdYsSXvyoy2BTsLESH7TlGV81S282U= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt new file mode 100644 index 0000000..4ad4385 --- /dev/null +++ b/hack/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ \ No newline at end of file diff --git a/hack/e2e-test.sh b/hack/e2e-test.sh new file mode 100755 index 0000000..d3d2f7d --- /dev/null +++ b/hack/e2e-test.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +SOURCE_DIR="$(cd "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +ROOT_DIR="$SOURCE_DIR/.." + +export GINKGO="$ROOT_DIR"/bin/ginkgo +export KIND="$ROOT_DIR"/bin/kind + +export E2E_TEST_BASH_IMAGE=registry.k8s.io/alpine-with-bash:1.0@sha256:0955672451201896cf9e2e5ce30bec0c7f10757af33bf78b7a6afa5672c596f5 + +# $1 cluster name +function cluster_create { + $KIND create cluster --name "$1" --image "$E2E_KIND_VERSION" --wait 1m -v 5 > "$ARTIFACTS/$1-create.log" 2>&1 \ + || { echo "unable to start the $1 cluster "; cat "$ARTIFACTS/$1-create.log" ; } + kubectl config use-context "kind-$1" + kubectl get nodes > "$ARTIFACTS/$1-nodes.log" || true + kubectl describe pods -n kube-system > "$ARTIFACTS/$1-system-pods.log" || true +} + +# $1 - cluster name +function cluster_cleanup { + kubectl config use-context "kind-$1" + $KIND export logs "$ARTIFACTS" --name "$1" || true + kubectl describe pods -n kueue-system > "$ARTIFACTS/$1-kueue-system-pods.log" || true + kubectl describe pods > "$ARTIFACTS/$1-default-pods.log" || true + $KIND delete cluster --name "$1" +} + +function startup { + if [ "$CREATE_KIND_CLUSTER" == 'true' ] + then + if [ ! -d "$ARTIFACTS" ]; then + mkdir -p "$ARTIFACTS" + fi + cluster_create "$KIND_CLUSTER_NAME" + fi +} + +function kind_load { + if [ "$CREATE_KIND_CLUSTER" == 'true' ] + then + cluster_kind_load "$KIND_CLUSTER_NAME" + fi +} + +# $1 cluster +function cluster_kind_load { + docker pull "${E2E_TEST_BASH_IMAGE}" + e2e_test_bash_image_without_sha=${E2E_TEST_BASH_IMAGE%%@*} + # We can load image by a digest but we cannot reference it by the digest that we pulled. + # For more information https://github.com/kubernetes-sigs/kind/issues/2394#issuecomment-888713831. + # Manually create tag for image with digest which is already pulled + docker tag $E2E_TEST_BASH_IMAGE "$e2e_test_bash_image_without_sha" + $KIND load docker-image "${e2e_test_bash_image_without_sha}" --name "$1" +} + +function cleanup { + if [ "$CREATE_KIND_CLUSTER" == 'true' ] + then + if [ ! -d "$ARTIFACTS" ]; then + mkdir -p "$ARTIFACTS" + fi + uninstall_kjobctl + cluster_cleanup "$KIND_CLUSTER_NAME" + fi +} + +function install_kjobctl { + cd "$SOURCE_DIR"/.. && make install +} + +function uninstall_kjobctl { + cd "$SOURCE_DIR"/.. && make uninstall +} + +trap cleanup EXIT +startup +kind_load +install_kjobctl +# shellcheck disable=SC2086 +$GINKGO $GINKGO_ARGS --junit-report=junit.xml --json-report=e2e.json --output-dir="$ARTIFACTS" -v ./test/e2e/... diff --git a/hack/multiplatform-build.sh b/hack/multiplatform-build.sh new file mode 100755 index 0000000..7c39952 --- /dev/null +++ b/hack/multiplatform-build.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +GO_CMD=${GO_CMD:-go} +LD_FLAGS=${LD_FLAGS:-} + +BUILD_NAME=${BUILD_NAME:-kueuectl} +PLATFORMS=${PLATFORMS:-linux/amd64} + +CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}") +ROOT_PATH=$(realpath "${CURRENT_DIR}/..") +BUILD_PATH=${ROOT_PATH}/${BUILD_DIR} + +mkdir -p "${BUILD_PATH}" + +IFS="," +for PLATFORM in ${PLATFORMS} ; do + export GOOS="${PLATFORM%/*}" + export GOARCH="${PLATFORM#*/}" + EXTENSION="" + + if [ "${GOOS}" == "windows" ]; then + EXTENSION=".exe" + fi + + echo "Building for $PLATFORM platform" + FULL_NAME=${BUILD_NAME}-${GOOS}-${GOARCH} + "${GO_CMD}" build -ldflags="${LD_FLAGS}" -o "${BUILD_PATH}/${FULL_NAME}${EXTENSION}" "$1" + + mkdir -p "${BUILD_PATH}/tmp" + cp "${ROOT_PATH}/LICENSE" "${BUILD_PATH}/tmp" + cp "${BUILD_PATH}/${FULL_NAME}${EXTENSION}" "${BUILD_PATH}/tmp/${BUILD_NAME}${EXTENSION}" + (cd "${BUILD_PATH}/tmp" && tar -czf "${BUILD_PATH}/${FULL_NAME}.tar.gz" ./*) + rm -R "${BUILD_PATH}/tmp" +done diff --git a/hack/shellcheck/Dockerfile b/hack/shellcheck/Dockerfile new file mode 100644 index 0000000..c5f0692 --- /dev/null +++ b/hack/shellcheck/Dockerfile @@ -0,0 +1 @@ +FROM docker.io/koalaman/shellcheck-alpine:v0.10.0 \ No newline at end of file diff --git a/hack/shellcheck/verify.sh b/hack/shellcheck/verify.sh new file mode 100755 index 0000000..aedb0df --- /dev/null +++ b/hack/shellcheck/verify.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# allow overriding docker cli, which should work fine for this script +DOCKER="${DOCKER:-docker}" +CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}") + +SHELLCHECK_IMAGE=$(grep '^FROM' "${CURRENT_DIR}/Dockerfile" | awk '{print $2}') + +# Initialize an empty array for scripts to check +scripts_to_check=() + +if [[ "$#" == 0 ]]; then + # Find all shell scripts excluding certain directories and patterns + while IFS=$'\n' read -r script; do + if ! git check-ignore -q "$script"; then + scripts_to_check+=("$script") + fi + done < <(find . -name "*.sh" \ + -not \( \ + -path ./_\* -o \ + -path ./.git\* -o \ + -path ./vendor\* -o \ + \( -path ./third_party\* -a -not -path ./third_party/forked\* \) \ + \)) +fi + +# Download shellcheck-alpine from Docker Hub +echo "Downloading ShellCheck Docker image..." +"${DOCKER}" pull "${SHELLCHECK_IMAGE}" + +# Run ShellCheck on all shell script files, excluding those in the 'vendor' directory. +# Configuration loaded from the .shelcheckrc file. +echo "Running ShellCheck..." +if [ "${#scripts_to_check[@]}" -ne 0 ]; then + "${DOCKER}" run --rm -v "$(pwd)":/mnt -w /mnt "${SHELLCHECK_IMAGE}" shellcheck "${scripts_to_check[@]}" >&2 +else + echo "No scripts to check" +fi + +echo "Shellcheck ran successfully" + diff --git a/hack/tools/go.mod b/hack/tools/go.mod new file mode 100644 index 0000000..6c0afbe --- /dev/null +++ b/hack/tools/go.mod @@ -0,0 +1,76 @@ +module sigs.k8s.io/kueue/cmd/experimental/kjobctl/hack/tools + +go 1.23 + +require ( + github.com/golangci/golangci-lint v1.62.2 + github.com/onsi/ginkgo/v2 v2.22.0 + gotest.tools/gotestsum v1.12.0 + k8s.io/code-generator v0.31.3 + sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20240813183042-b901db121e1f + sigs.k8s.io/controller-tools v0.16.5 + sigs.k8s.io/kind v0.25.0 + sigs.k8s.io/kustomize/kustomize/v5 v5.5.0 +) + +require ( + github.com/bitfield/gotestdox v0.2.2 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dnephin/pflag v1.0.7 // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-errors/errors v1.4.2 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gobuffalo/flect v1.0.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cobra v1.8.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + github.com/xlab/treeprint v1.2.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/mod v0.22.0 // indirect + golang.org/x/net v0.31.0 // indirect + golang.org/x/sync v0.9.0 // indirect + golang.org/x/sys v0.27.0 // indirect + golang.org/x/term v0.26.0 // indirect + golang.org/x/text v0.20.0 // indirect + golang.org/x/tools v0.27.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.31.2 // indirect + k8s.io/apimachinery v0.31.3 // indirect + k8s.io/gengo/v2 v2.0.0-20240404160639-a0386bf69313 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20240709000822-3c01b740850f // indirect + k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/kustomize/api v0.18.0 // indirect + sigs.k8s.io/kustomize/kyaml v0.18.1 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/hack/tools/go.sum b/hack/tools/go.sum new file mode 100644 index 0000000..95170e3 --- /dev/null +++ b/hack/tools/go.sum @@ -0,0 +1,289 @@ +github.com/bitfield/gotestdox v0.2.2 h1:x6RcPAbBbErKLnapz1QeAlf3ospg8efBsedU93CDsnE= +github.com/bitfield/gotestdox v0.2.2/go.mod h1:D+gwtS0urjBrzguAkTM2wodsTQYFHdpx8eqRJ3N+9pY= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= +github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gobuffalo/flect v1.0.3 h1:xeWBM2nui+qnVvNM4S3foBhCAL2XgPU+a7FdpelbTq4= +github.com/gobuffalo/flect v1.0.3/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golangci/golangci-lint v1.62.2 h1:b8K5K9PN+rZN1+mKLtsZHz2XXS9aYKzQ9i25x3Qnxxw= +github.com/golangci/golangci-lint v1.62.2/go.mod h1:ILWWyeFUrctpHVGMa1dg2xZPKoMUTc5OIMgW7HZr34g= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= +github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= +golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= +golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= +golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/tools v0.27.0 h1:qEKojBykQkQ4EynWy4S8Weg69NumxKdn40Fce3uc/8o= +golang.org/x/tools v0.27.0/go.mod h1:sUi0ZgbwW9ZPAq26Ekut+weQPR5eIM6GQLQ1Yjm1H0Q= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/gotestsum v1.12.0 h1:CmwtaGDkHxrZm4Ib0Vob89MTfpc3GrEFMJKovliPwGk= +gotest.tools/gotestsum v1.12.0/go.mod h1:fAvqkSptospfSbQw26CTYzNwnsE/ztqLeyhP0h67ARY= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= +k8s.io/apiextensions-apiserver v0.31.2 h1:W8EwUb8+WXBLu56ser5IudT2cOho0gAKeTOnywBLxd0= +k8s.io/apiextensions-apiserver v0.31.2/go.mod h1:i+Geh+nGCJEGiCGR3MlBDkS7koHIIKWVfWeRFiOsUcM= +k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4= +k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/code-generator v0.31.3 h1:Pj0fYOBms+ZrsulLi4DMsCEx1jG8fWKRLy44onHsLBI= +k8s.io/code-generator v0.31.3/go.mod h1:/umCIlT84g1+Yu5ZXtP1KGSRTnGiIzzX5AzUAxsNlts= +k8s.io/gengo/v2 v2.0.0-20240404160639-a0386bf69313 h1:bKcdZJOPICVmIIuaM9+MXmapE94dn5AYv5ODs1jA43o= +k8s.io/gengo/v2 v2.0.0-20240404160639-a0386bf69313/go.mod h1:VH3AT8AaQOqiGjMF9p0/IM1Dj+82ZwjfxUP1IxaHE+8= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240709000822-3c01b740850f h1:2sXuKesAYbRHxL3aE2PN6zX/gcJr22cjrsej+W784Tc= +k8s.io/kube-openapi v0.0.0-20240709000822-3c01b740850f/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20240813183042-b901db121e1f h1:RIqUbZQO5yizUt9nozxQdzyLRD1sG6s2oi/QZrlg/hs= +sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20240813183042-b901db121e1f/go.mod h1:IaDsO8xSPRxRG1/rm9CP7+jPmj0nMNAuNi/yiHnLX8k= +sigs.k8s.io/controller-tools v0.16.5 h1:5k9FNRqziBPwqr17AMEPPV/En39ZBplLAdOwwQHruP4= +sigs.k8s.io/controller-tools v0.16.5/go.mod h1:8vztuRVzs8IuuJqKqbXCSlXcw+lkAv/M2sTpg55qjMY= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/kind v0.25.0 h1:ugUvgesHKKA0yKmD6QtYTiEev+kPUpGxdTPbMGf8VTU= +sigs.k8s.io/kind v0.25.0/go.mod h1:t7ueEpzPYJvHA8aeLtI52rtFftNgUYUaCwvxjk7phfw= +sigs.k8s.io/kustomize/api v0.18.0 h1:hTzp67k+3NEVInwz5BHyzc9rGxIauoXferXyjv5lWPo= +sigs.k8s.io/kustomize/api v0.18.0/go.mod h1:f8isXnX+8b+SGLHQ6yO4JG1rdkZlvhaCf/uZbLVMb0U= +sigs.k8s.io/kustomize/kustomize/v5 v5.5.0 h1:o1mtt6vpxsxDYaZKrw3BnEtc+pAjLz7UffnIvHNbvW0= +sigs.k8s.io/kustomize/kustomize/v5 v5.5.0/go.mod h1:AeFCmgCrXzmvjWWaeZCyBp6XzG1Y0w1svYus8GhJEOE= +sigs.k8s.io/kustomize/kyaml v0.18.1 h1:WvBo56Wzw3fjS+7vBjN6TeivvpbW9GmRaWZ9CIVmt4E= +sigs.k8s.io/kustomize/kyaml v0.18.1/go.mod h1:C3L2BFVU1jgcddNBE1TxuVLgS46TjObMwW5FT9FcjYo= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/hack/tools/gzip/gzip.go b/hack/tools/gzip/gzip.go new file mode 100644 index 0000000..34945b2 --- /dev/null +++ b/hack/tools/gzip/gzip.go @@ -0,0 +1,37 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "compress/gzip" + "io" + "os" +) + +// Simple gzip encoder used to have the same output generated regardless of the OS. +func main() { + writer, err := gzip.NewWriterLevel(os.Stdout, gzip.BestCompression) + if err != nil { + panic(err) + } + // The header is initialized with Unknown OS and no additional fields, we should not change it. + _, err = io.Copy(writer, os.Stdin) + if err != nil { + panic(err) + } + writer.Close() +} diff --git a/hack/tools/pinversion.go b/hack/tools/pinversion.go new file mode 100644 index 0000000..ed79056 --- /dev/null +++ b/hack/tools/pinversion.go @@ -0,0 +1,34 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tools + +// Keep a reference to the code generators, so they are not removed by go mod tidy +import ( + _ "github.com/golangci/golangci-lint/pkg/exitcodes" + _ "github.com/onsi/ginkgo/v2/ginkgo/command" + _ "github.com/onsi/ginkgo/v2/ginkgo/run" + _ "gotest.tools/gotestsum/cmd" + _ "k8s.io/code-generator" + _ "sigs.k8s.io/controller-runtime/tools/setup-envtest/env" + + // since verify will error when referencing a cmd package + // we need to reference individual dependencies used by it + _ "sigs.k8s.io/controller-tools/pkg/crd" + _ "sigs.k8s.io/controller-tools/pkg/genall/help/pretty" + _ "sigs.k8s.io/kind/pkg/cmd" + _ "sigs.k8s.io/kustomize/kustomize/v5/commands/edit/listbuiltin" +) diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh new file mode 100755 index 0000000..bc9656b --- /dev/null +++ b/hack/update-codegen.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +GO_CMD=${1:-go} +CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}") +KJOBCTL_ROOT=$(realpath "$CURRENT_DIR/..") +KJOBCTL_PKG="sigs.k8s.io/kueue/cmd/experimental/kjobctl" +CODEGEN_PKG=$(cd "$TOOLS_DIR" && go mod download && $GO_CMD list -m -f "{{.Dir}}" k8s.io/code-generator) + +cd "$CURRENT_DIR/.." + +# shellcheck source=/dev/null +source "${CODEGEN_PKG}/kube_codegen.sh" + +# Generating conversion and defaults functions +kube::codegen::gen_helpers \ + --boilerplate "${KJOBCTL_ROOT}/hack/boilerplate.go.txt" \ + "${KJOBCTL_ROOT}" + +kube::codegen::gen_client \ + --boilerplate "${KJOBCTL_ROOT}/hack/boilerplate.go.txt" \ + --output-dir "${KJOBCTL_ROOT}/client-go" \ + --output-pkg "${KJOBCTL_PKG}/client-go" \ + "${KJOBCTL_ROOT}" + +# We need to clean up the go.mod file since code-generator adds temporary library to the go.mod file. +"${GO_CMD}" mod tidy diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go new file mode 100644 index 0000000..dae217c --- /dev/null +++ b/pkg/builder/builder.go @@ -0,0 +1,697 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + "errors" + "fmt" + "os" + "slices" + "strings" + "time" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilrand "k8s.io/apimachinery/pkg/util/rand" + k8s "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" + kueueversioned "sigs.k8s.io/kueue/client-go/clientset/versioned" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/parser" +) + +var ( + noNamespaceSpecifiedErr = errors.New("no namespace specified") + noApplicationProfileSpecifiedErr = errors.New("no application profile specified") + noApplicationProfileModeSpecifiedErr = errors.New("no application profile mode specified") + invalidApplicationProfileModeErr = errors.New("invalid application profile mode") + applicationProfileModeNotConfiguredErr = errors.New("application profile mode not configured") + noCommandSpecifiedErr = errors.New("no command specified") + noParallelismSpecifiedErr = errors.New("no parallelism specified") + noCompletionsSpecifiedErr = errors.New("no completions specified") + noReplicasSpecifiedErr = errors.New("no replicas specified") + noMinReplicasSpecifiedErr = errors.New("no min-replicas specified") + noMaxReplicasSpecifiedErr = errors.New("no max-replicas specified") + noRequestsSpecifiedErr = errors.New("no requests specified") + noLocalQueueSpecifiedErr = errors.New("no local queue specified") + noRayClusterSpecifiedErr = errors.New("no raycluster specified") + noArraySpecifiedErr = errors.New("no array specified") + noCpusPerTaskSpecifiedErr = errors.New("no cpus-per-task specified") + noErrorSpecifiedErr = errors.New("no error specified") + noGpusPerTaskSpecifiedErr = errors.New("no gpus-per-task specified") + noInputSpecifiedErr = errors.New("no input specified") + noJobNameSpecifiedErr = errors.New("no job-name specified") + noMemPerCPUSpecifiedErr = errors.New("no mem-per-cpu specified") + noMemPerGPUSpecifiedErr = errors.New("no mem-per-gpu specified") + noMemPerTaskSpecifiedErr = errors.New("no mem-per-task specified") + noNodesSpecifiedErr = errors.New("no nodes specified") + noNTasksSpecifiedErr = errors.New("no ntasks specified") + noOutputSpecifiedErr = errors.New("no output specified") + noPartitionSpecifiedErr = errors.New("no partition specified") + noPrioritySpecifiedErr = errors.New("no priority specified") + noTimeSpecifiedErr = errors.New("no time specified") +) + +type builder interface { + build(ctx context.Context) (rootObj runtime.Object, childObjs []runtime.Object, err error) +} + +type Builder struct { + clientGetter util.ClientGetter + kjobctlClientset versioned.Interface + k8sClientset k8s.Interface + kueueClientset kueueversioned.Interface + + namespace string + profileName string + modeName v1alpha1.ApplicationProfileMode + + command []string + parallelism *int32 + completions *int32 + replicas map[string]int + minReplicas map[string]int + maxReplicas map[string]int + requests corev1.ResourceList + localQueue string + rayCluster string + script string + array string + cpusPerTask *resource.Quantity + error string + gpusPerTask map[string]*resource.Quantity + input string + jobName string + memPerNode *resource.Quantity + memPerCPU *resource.Quantity + memPerGPU *resource.Quantity + memPerTask *resource.Quantity + nodes *int32 + nTasks *int32 + output string + partition string + priority string + initImage string + ignoreUnknown bool + skipLocalQueueValidation bool + skipPriorityValidation bool + firstNodeIP bool + firstNodeIPTimeout time.Duration + changeDir string + timeLimit string + + profile *v1alpha1.ApplicationProfile + mode *v1alpha1.SupportedMode + volumeBundles []v1alpha1.VolumeBundle + + buildTime time.Time +} + +func NewBuilder(clientGetter util.ClientGetter, buildTime time.Time) *Builder { + return &Builder{clientGetter: clientGetter, buildTime: buildTime} +} + +func (b *Builder) WithNamespace(namespace string) *Builder { + b.namespace = namespace + return b +} + +func (b *Builder) WithProfileName(profileName string) *Builder { + b.profileName = profileName + return b +} + +func (b *Builder) WithModeName(modeName v1alpha1.ApplicationProfileMode) *Builder { + b.modeName = modeName + return b +} + +func (b *Builder) WithCommand(command []string) *Builder { + b.command = command + return b +} + +func (b *Builder) WithParallelism(parallelism *int32) *Builder { + b.parallelism = parallelism + return b +} + +func (b *Builder) WithCompletions(completions *int32) *Builder { + b.completions = completions + return b +} + +func (b *Builder) WithReplicas(replicas map[string]int) *Builder { + b.replicas = replicas + return b +} + +func (b *Builder) WithMinReplicas(minReplicas map[string]int) *Builder { + b.minReplicas = minReplicas + return b +} + +func (b *Builder) WithMaxReplicas(maxReplicas map[string]int) *Builder { + b.maxReplicas = maxReplicas + return b +} + +func (b *Builder) WithRequests(requests corev1.ResourceList) *Builder { + b.requests = requests + return b +} + +func (b *Builder) WithLocalQueue(localQueue string) *Builder { + b.localQueue = localQueue + return b +} + +func (b *Builder) WithRayCluster(rayCluster string) *Builder { + b.rayCluster = rayCluster + return b +} + +func (b *Builder) WithScript(script string) *Builder { + b.script = script + return b +} + +func (b *Builder) WithArray(array string) *Builder { + b.array = array + return b +} + +func (b *Builder) WithCpusPerTask(cpusPerTask *resource.Quantity) *Builder { + b.cpusPerTask = cpusPerTask + return b +} + +func (b *Builder) WithError(error string) *Builder { + b.error = error + return b +} + +func (b *Builder) WithGpusPerTask(gpusPerTask map[string]*resource.Quantity) *Builder { + b.gpusPerTask = gpusPerTask + return b +} + +func (b *Builder) WithInput(input string) *Builder { + b.input = input + return b +} + +func (b *Builder) WithJobName(jobName string) *Builder { + b.jobName = jobName + return b +} + +func (b *Builder) WithMemPerNode(memPerNode *resource.Quantity) *Builder { + b.memPerNode = memPerNode + return b +} + +func (b *Builder) WithMemPerCPU(memPerCPU *resource.Quantity) *Builder { + b.memPerCPU = memPerCPU + return b +} + +func (b *Builder) WithMemPerGPU(memPerGPU *resource.Quantity) *Builder { + b.memPerGPU = memPerGPU + return b +} + +func (b *Builder) WithMemPerTask(memPerTask *resource.Quantity) *Builder { + b.memPerTask = memPerTask + return b +} + +func (b *Builder) WithNodes(nodes *int32) *Builder { + b.nodes = nodes + return b +} + +func (b *Builder) WithNTasks(nTasks *int32) *Builder { + b.nTasks = nTasks + return b +} + +func (b *Builder) WithOutput(output string) *Builder { + b.output = output + return b +} + +func (b *Builder) WithPartition(partition string) *Builder { + b.partition = partition + return b +} + +func (b *Builder) WithPriority(priority string) *Builder { + b.priority = priority + return b +} + +func (b *Builder) WithInitImage(initImage string) *Builder { + b.initImage = initImage + return b +} + +func (b *Builder) WithIgnoreUnknown(ignoreUnknown bool) *Builder { + b.ignoreUnknown = ignoreUnknown + return b +} + +func (b *Builder) WithChangeDir(chdir string) *Builder { + b.changeDir = chdir + return b +} + +func (b *Builder) WithSkipLocalQueueValidation(skip bool) *Builder { + b.skipLocalQueueValidation = skip + return b +} + +func (b *Builder) WithSkipPriorityValidation(skip bool) *Builder { + b.skipPriorityValidation = skip + return b +} + +func (b *Builder) WithFirstNodeIP(firstNodeIP bool) *Builder { + b.firstNodeIP = firstNodeIP + return b +} + +func (b *Builder) WithFirstNodeIPTimeout(timeout time.Duration) *Builder { + b.firstNodeIPTimeout = timeout + return b +} + +func (b *Builder) WithTimeLimit(timeLimit string) *Builder { + b.timeLimit = timeLimit + return b +} + +func (b *Builder) validateGeneral(ctx context.Context) error { + if b.namespace == "" { + return noNamespaceSpecifiedErr + } + + if b.profileName == "" { + return noApplicationProfileSpecifiedErr + } + + if b.modeName == "" { + return noApplicationProfileModeSpecifiedErr + } + + // check that local queue exists + if len(b.localQueue) != 0 && !b.skipLocalQueueValidation { + _, err := b.kueueClientset.KueueV1beta1().LocalQueues(b.namespace).Get(ctx, b.localQueue, metav1.GetOptions{}) + if err != nil { + return err + } + } + + // check that priority class exists + if len(b.priority) != 0 && !b.skipPriorityValidation { + _, err := b.kueueClientset.KueueV1beta1().WorkloadPriorityClasses().Get(ctx, b.priority, metav1.GetOptions{}) + if err != nil { + return err + } + } + + return nil +} + +func (b *Builder) complete(ctx context.Context) error { + var err error + + b.profile, err = b.kjobctlClientset.KjobctlV1alpha1().ApplicationProfiles(b.namespace).Get(ctx, b.profileName, metav1.GetOptions{}) + if err != nil { + return err + } + + for i, mode := range b.profile.Spec.SupportedModes { + if mode.Name == b.modeName { + b.mode = &b.profile.Spec.SupportedModes[i] + } + } + + if b.mode == nil { + return applicationProfileModeNotConfiguredErr + } + + for _, name := range b.profile.Spec.VolumeBundles { + volumeBundle, err := b.kjobctlClientset.KjobctlV1alpha1().VolumeBundles(b.profile.Namespace).Get(ctx, string(name), metav1.GetOptions{}) + if err != nil { + return err + } + b.volumeBundles = append(b.volumeBundles, *volumeBundle) + } + + return nil +} + +func (b *Builder) validateFlags() error { + if slices.Contains(b.mode.RequiredFlags, v1alpha1.CmdFlag) && len(b.command) == 0 { + return noCommandSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.ParallelismFlag) && b.parallelism == nil { + return noParallelismSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.CompletionsFlag) && b.completions == nil { + return noCompletionsSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.ReplicasFlag) && b.replicas == nil { + return noReplicasSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.MinReplicasFlag) && b.minReplicas == nil { + return noMinReplicasSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.MaxReplicasFlag) && b.maxReplicas == nil { + return noMaxReplicasSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.RequestFlag) && b.requests == nil { + return noRequestsSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.LocalQueueFlag) && b.localQueue == "" { + return noLocalQueueSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.RayClusterFlag) && b.rayCluster == "" { + return noRayClusterSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.ArrayFlag) && b.array == "" { + return noArraySpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.CpusPerTaskFlag) && b.cpusPerTask == nil { + return noCpusPerTaskSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.ErrorFlag) && b.error == "" { + return noErrorSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.GpusPerTaskFlag) && b.gpusPerTask == nil { + return noGpusPerTaskSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.InputFlag) && b.input == "" { + return noInputSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.JobNameFlag) && b.jobName == "" { + return noJobNameSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.MemPerCPUFlag) && b.memPerCPU == nil { + return noMemPerCPUSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.MemPerGPUFlag) && b.memPerGPU == nil { + return noMemPerGPUSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.MemPerTaskFlag) && b.memPerTask == nil { + return noMemPerTaskSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.NodesFlag) && b.nodes == nil { + return noNodesSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.NTasksFlag) && b.nTasks == nil { + return noNTasksSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.OutputFlag) && b.output == "" { + return noOutputSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.PartitionFlag) && b.partition == "" { + return noPartitionSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.PriorityFlag) && b.priority == "" { + return noPrioritySpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.TimeFlag) && b.timeLimit == "" { + return noTimeSpecifiedErr + } + + return nil +} + +func (b *Builder) Do(ctx context.Context) (runtime.Object, []runtime.Object, error) { + if err := b.setClients(); err != nil { + return nil, nil, err + } + + if err := b.validateGeneral(ctx); err != nil { + return nil, nil, err + } + + var bImpl builder + + switch b.modeName { + case v1alpha1.JobMode: + bImpl = newJobBuilder(b) + case v1alpha1.InteractiveMode: + bImpl = newInteractiveBuilder(b) + case v1alpha1.RayJobMode: + bImpl = newRayJobBuilder(b) + case v1alpha1.RayClusterMode: + bImpl = newRayClusterBuilder(b) + case v1alpha1.SlurmMode: + bImpl = newSlurmBuilder(b) + } + + if bImpl == nil { + return nil, nil, invalidApplicationProfileModeErr + } + + if err := b.complete(ctx); err != nil { + return nil, nil, err + } + + if err := b.validateFlags(); err != nil { + return nil, nil, err + } + + return bImpl.build(ctx) +} + +func (b *Builder) setClients() error { + var err error + + b.kjobctlClientset, err = b.clientGetter.KjobctlClientset() + if err != nil { + return err + } + + b.k8sClientset, err = b.clientGetter.K8sClientset() + if err != nil { + return err + } + + b.kueueClientset, err = b.clientGetter.KueueClientset() + if err != nil { + return err + } + + return nil +} + +func (b *Builder) buildObjectMeta(templateObjectMeta metav1.ObjectMeta, strictNaming bool) (metav1.ObjectMeta, error) { + objectMeta := metav1.ObjectMeta{ + Namespace: b.profile.Namespace, + Labels: templateObjectMeta.Labels, + Annotations: templateObjectMeta.Annotations, + } + + if strictNaming { + objectMeta.Name = b.generatePrefixName() + utilrand.String(5) + } else { + objectMeta.GenerateName = b.generatePrefixName() + } + + b.withKjobLabels(&objectMeta) + if err := b.withKueueLabels(&objectMeta); err != nil { + return metav1.ObjectMeta{}, err + } + + return objectMeta, nil +} + +func (b *Builder) buildChildObjectMeta(name string) metav1.ObjectMeta { + objectMeta := metav1.ObjectMeta{ + Name: name, + Namespace: b.profile.Namespace, + } + b.withKjobLabels(&objectMeta) + return objectMeta +} + +func (b *Builder) withKjobLabels(objectMeta *metav1.ObjectMeta) { + if objectMeta.Labels == nil { + objectMeta.Labels = map[string]string{} + } + + if b.profile != nil { + objectMeta.Labels[constants.ProfileLabel] = b.profile.Name + } + + if b.mode != nil { + objectMeta.Labels[constants.ModeLabel] = string(b.mode.Name) + } +} + +func (b *Builder) withKueueLabels(objectMeta *metav1.ObjectMeta) error { + if objectMeta.Labels == nil { + objectMeta.Labels = map[string]string{} + } + + if len(b.localQueue) > 0 { + objectMeta.Labels[kueueconstants.QueueLabel] = b.localQueue + } + + if len(b.priority) != 0 { + objectMeta.Labels[kueueconstants.WorkloadPriorityClassLabel] = b.priority + } + + if b.timeLimit != "" { + maxExecutionTimeSeconds, err := parser.TimeLimitToSeconds(b.timeLimit) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", b.timeLimit, err) + } + + if ptr.Deref(maxExecutionTimeSeconds, 0) > 0 { + objectMeta.Labels[kueueconstants.MaxExecTimeSecondsLabel] = fmt.Sprint(*maxExecutionTimeSeconds) + } + } + + return nil +} + +func (b *Builder) buildPodSpec(templateSpec corev1.PodSpec) corev1.PodSpec { + b.buildPodSpecVolumesAndEnv(&templateSpec) + + for i := range templateSpec.Containers { + container := &templateSpec.Containers[i] + + if i == 0 && len(b.command) > 0 { + container.Command = b.command + } + + if i == 0 && len(b.requests) > 0 { + container.Resources.Requests = b.requests + } + } + + return templateSpec +} + +func (b *Builder) buildPodSpecVolumesAndEnv(templateSpec *corev1.PodSpec) { + bundle := mergeBundles(b.volumeBundles) + + templateSpec.Volumes = append(templateSpec.Volumes, bundle.Spec.Volumes...) + for i := range templateSpec.Containers { + container := &templateSpec.Containers[i] + + container.VolumeMounts = append(container.VolumeMounts, bundle.Spec.ContainerVolumeMounts...) + container.Env = append(container.Env, bundle.Spec.EnvVars...) + container.Env = append(container.Env, b.additionalEnvironmentVariables()...) + } + + for i := range templateSpec.InitContainers { + initContainer := &templateSpec.InitContainers[i] + + initContainer.VolumeMounts = append(initContainer.VolumeMounts, bundle.Spec.ContainerVolumeMounts...) + initContainer.Env = append(initContainer.Env, bundle.Spec.EnvVars...) + initContainer.Env = append(initContainer.Env, b.additionalEnvironmentVariables()...) + } +} + +func (b *Builder) buildRayClusterSpec(spec *rayv1.RayClusterSpec) { + b.buildPodSpecVolumesAndEnv(&spec.HeadGroupSpec.Template.Spec) + + for index := range spec.WorkerGroupSpecs { + workerGroupSpec := &spec.WorkerGroupSpecs[index] + + if replicas, ok := b.replicas[workerGroupSpec.GroupName]; ok { + workerGroupSpec.Replicas = ptr.To(int32(replicas)) + } + if minReplicas, ok := b.minReplicas[workerGroupSpec.GroupName]; ok { + workerGroupSpec.MinReplicas = ptr.To(int32(minReplicas)) + } + if maxReplicas, ok := b.maxReplicas[workerGroupSpec.GroupName]; ok { + workerGroupSpec.MaxReplicas = ptr.To(int32(maxReplicas)) + } + + b.buildPodSpecVolumesAndEnv(&workerGroupSpec.Template.Spec) + } +} + +func (b *Builder) additionalEnvironmentVariables() []corev1.EnvVar { + userID := os.Getenv(constants.SystemEnvVarNameUser) + timestamp := b.buildTime.Format(time.RFC3339) + taskName := fmt.Sprintf("%s_%s", b.namespace, b.profileName) + + envVars := []corev1.EnvVar{ + {Name: constants.EnvVarNameUserID, Value: userID}, + {Name: constants.EnvVarTaskName, Value: taskName}, + {Name: constants.EnvVarTaskID, Value: fmt.Sprintf("%s_%s_%s", userID, timestamp, taskName)}, + {Name: constants.EnvVarNameProfile, Value: fmt.Sprintf("%s_%s", b.namespace, b.profileName)}, + {Name: constants.EnvVarNameTimestamp, Value: timestamp}, + } + + return envVars +} + +func mergeBundles(bundles []v1alpha1.VolumeBundle) v1alpha1.VolumeBundle { + var volumeBundle v1alpha1.VolumeBundle + for _, b := range bundles { + volumeBundle.Spec.Volumes = append(volumeBundle.Spec.Volumes, b.Spec.Volumes...) + volumeBundle.Spec.ContainerVolumeMounts = append(volumeBundle.Spec.ContainerVolumeMounts, b.Spec.ContainerVolumeMounts...) + volumeBundle.Spec.EnvVars = append(volumeBundle.Spec.EnvVars, b.Spec.EnvVars...) + } + + return volumeBundle +} + +func (b *Builder) generatePrefixName() string { + return strings.ToLower(fmt.Sprintf("%s-%s-", b.profile.Name, b.modeName)) +} diff --git a/pkg/builder/builder_test.go b/pkg/builder/builder_test.go new file mode 100644 index 0000000..527d809 --- /dev/null +++ b/pkg/builder/builder_test.go @@ -0,0 +1,472 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/fake" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestBuilder(t *testing.T) { + testStartTime := time.Now() + + testCases := map[string]struct { + namespace string + profile string + mode v1alpha1.ApplicationProfileMode + kjobctlObjs []runtime.Object + wantRootObj runtime.Object + wantErr error + }{ + "shouldn't build job because no namespace specified": { + wantErr: noNamespaceSpecifiedErr, + }, + "shouldn't build job because no application profile specified": { + namespace: metav1.NamespaceDefault, + wantErr: noApplicationProfileSpecifiedErr, + }, + "shouldn't build job because application profile not found": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + wantErr: apierrors.NewNotFound(schema.GroupResource{Group: "kjobctl.x-k8s.io", Resource: "applicationprofiles"}, "profile"), + }, + "shouldn't build job because no application profile mode specified": { + namespace: metav1.NamespaceDefault, + profile: "profile", + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.JobMode}). + Obj(), + }, + wantErr: noApplicationProfileModeSpecifiedErr, + }, + "shouldn't build job because application profile mode not configured": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.InteractiveMode}). + Obj(), + }, + wantErr: applicationProfileModeNotConfiguredErr, + }, + "shouldn't build job because invalid application profile mode": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: "Invalid", + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.InteractiveMode}). + Obj(), + }, + wantErr: invalidApplicationProfileModeErr, + }, + "shouldn't build job because command not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.JobMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.CmdFlag}, + }). + Obj(), + }, + wantErr: noCommandSpecifiedErr, + }, + "shouldn't build job because parallelism not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.JobMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.ParallelismFlag}, + }). + Obj(), + }, + wantErr: noParallelismSpecifiedErr, + }, + "shouldn't build job because completions not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.JobMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.CompletionsFlag}, + }). + Obj(), + }, + wantErr: noCompletionsSpecifiedErr, + }, + "shouldn't build job because replicas not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.JobMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.ReplicasFlag}, + }). + Obj(), + }, + wantErr: noReplicasSpecifiedErr, + }, + "shouldn't build job because min-replicas not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.JobMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.MinReplicasFlag}, + }). + Obj(), + }, + wantErr: noMinReplicasSpecifiedErr, + }, + "shouldn't build job because max-replicas not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.JobMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.MaxReplicasFlag}, + }). + Obj(), + }, + wantErr: noMaxReplicasSpecifiedErr, + }, + "shouldn't build job because request not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.JobMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.RequestFlag}, + }). + Obj(), + }, + wantErr: noRequestsSpecifiedErr, + }, + "shouldn't build job because local queue not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.JobMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.LocalQueueFlag}, + }). + Obj(), + }, + wantErr: noLocalQueueSpecifiedErr, + }, + "shouldn't build job because raycluster not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.RayJobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.RayJobMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.RayClusterFlag}, + }). + Obj(), + }, + wantErr: noRayClusterSpecifiedErr, + }, + "shouldn't build job because array not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.ArrayFlag}, + }). + Obj(), + }, + wantErr: noArraySpecifiedErr, + }, + "shouldn't build job because cpusPerTask not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.CpusPerTaskFlag}, + }). + Obj(), + }, + wantErr: noCpusPerTaskSpecifiedErr, + }, + "shouldn't build job because error not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.ErrorFlag}, + }). + Obj(), + }, + wantErr: noErrorSpecifiedErr, + }, + "shouldn't build job because gpusPerTask not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.GpusPerTaskFlag}, + }). + Obj(), + }, + wantErr: noGpusPerTaskSpecifiedErr, + }, + "shouldn't build job because input not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.InputFlag}, + }). + Obj(), + }, + wantErr: noInputSpecifiedErr, + }, + "shouldn't build job because jobName not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.JobNameFlag}, + }). + Obj(), + }, + wantErr: noJobNameSpecifiedErr, + }, + "shouldn't build job because memPerCPU not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.MemPerCPUFlag}, + }). + Obj(), + }, + wantErr: noMemPerCPUSpecifiedErr, + }, + "shouldn't build job because memPerGPU not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.MemPerGPUFlag}, + }). + Obj(), + }, + wantErr: noMemPerGPUSpecifiedErr, + }, + "shouldn't build job because memPerTask not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.MemPerTaskFlag}, + }). + Obj(), + }, + wantErr: noMemPerTaskSpecifiedErr, + }, + "shouldn't build job because nodes not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.NodesFlag}, + }). + Obj(), + }, + wantErr: noNodesSpecifiedErr, + }, + "shouldn't build job because nTasks not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.NTasksFlag}, + }). + Obj(), + }, + wantErr: noNTasksSpecifiedErr, + }, + "shouldn't build job because output not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.OutputFlag}, + }). + Obj(), + }, + wantErr: noOutputSpecifiedErr, + }, + "shouldn't build job because partition not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.PartitionFlag}, + }). + Obj(), + }, + wantErr: noPartitionSpecifiedErr, + }, + "shouldn't build job because priority not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.PriorityFlag}, + }). + Obj(), + }, + wantErr: noPrioritySpecifiedErr, + }, + "should build job": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault). + Label("foo", "bar"). + Annotation("foo", "baz"). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.JobMode, + Template: "job-template", + }). + Obj(), + }, + wantRootObj: wrappers.MakeJob("", metav1.NamespaceDefault).GenerateName("profile-job-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Obj(), + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tcg := cmdtesting.NewTestClientGetter(). + WithKjobctlClientset(fake.NewSimpleClientset(tc.kjobctlObjs...)) + gotRootObj, gotChildObjs, gotErr := NewBuilder(tcg, testStartTime). + WithNamespace(tc.namespace). + WithProfileName(tc.profile). + WithModeName(tc.mode). + Do(ctx) + + var opts []cmp.Option + var statusError *apierrors.StatusError + if !errors.As(tc.wantErr, &statusError) { + opts = append(opts, cmpopts.EquateErrors()) + } + if diff := cmp.Diff(tc.wantErr, gotErr, opts...); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + return + } + + if diff := cmp.Diff(tc.wantRootObj, gotRootObj, opts...); diff != "" { + t.Errorf("Root object after build (-want,+got):\n%s", diff) + } + + if diff := cmp.Diff([]runtime.Object(nil), gotChildObjs, opts...); diff != "" { + t.Errorf("Child objects after build (-want,+got):\n%s", diff) + } + }) + } +} diff --git a/pkg/builder/interactive_builder.go b/pkg/builder/interactive_builder.go new file mode 100644 index 0000000..0c8b302 --- /dev/null +++ b/pkg/builder/interactive_builder.go @@ -0,0 +1,65 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +type interactiveBuilder struct { + *Builder +} + +var _ builder = (*interactiveBuilder)(nil) + +func (b *interactiveBuilder) build(ctx context.Context) (runtime.Object, []runtime.Object, error) { + template, err := b.k8sClientset.CoreV1().PodTemplates(b.profile.Namespace).Get(ctx, string(b.mode.Template), metav1.GetOptions{}) + if err != nil { + return nil, nil, err + } + + objectMeta, err := b.buildObjectMeta(template.Template.ObjectMeta, false) + if err != nil { + return nil, nil, err + } + + pod := &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: objectMeta, + Spec: template.Template.Spec, + } + + pod.Spec = b.buildPodSpec(pod.Spec) + + if len(pod.Spec.Containers) > 0 { + pod.Spec.Containers[0].TTY = true + pod.Spec.Containers[0].Stdin = true + } + + return pod, nil, nil +} + +func newInteractiveBuilder(b *Builder) *interactiveBuilder { + return &interactiveBuilder{Builder: b} +} diff --git a/pkg/builder/interactive_builder_test.go b/pkg/builder/interactive_builder_test.go new file mode 100644 index 0000000..8971aa5 --- /dev/null +++ b/pkg/builder/interactive_builder_test.go @@ -0,0 +1,218 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + k8sfake "k8s.io/client-go/kubernetes/fake" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + kjobctlfake "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/fake" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestInteractiveBuilder(t *testing.T) { + testStartTime := time.Now() + userID := os.Getenv(constants.SystemEnvVarNameUser) + + testPodTemplateWrapper := wrappers.MakePodTemplate("pod-template", metav1.NamespaceDefault). + Label("foo", "bar"). + Annotation("foo", "baz"). + WithInitContainer( + *wrappers.MakeContainer("ic1", ""). + WithEnvVar(corev1.EnvVar{Name: "e0", Value: "default-value0"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm0", MountPath: "/etc/default-config0"}). + Obj(), + ). + WithContainer( + *wrappers.MakeContainer("c1", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("1")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithContainer(*wrappers.MakeContainer("c2", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithVolume("v1", "default-config1"). + WithVolume("v2", "default-config2") + + testCases := map[string]struct { + namespace string + profile string + mode v1alpha1.ApplicationProfileMode + command []string + requests corev1.ResourceList + localQueue string + k8sObjs []runtime.Object + kjobctlObjs []runtime.Object + wantRootObj runtime.Object + wantErr error + }{ + "shouldn't build pod because template not found": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.InteractiveMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.InteractiveMode, Template: "pod-template"}). + Obj(), + }, + wantErr: apierrors.NewNotFound(schema.GroupResource{Resource: "podtemplates"}, "pod-template"), + }, + "should build job without replacements": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.InteractiveMode, + k8sObjs: []runtime.Object{testPodTemplateWrapper.Clone().Obj()}, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.InteractiveMode, Template: "pod-template"}). + Obj(), + }, + wantRootObj: wrappers.MakePod("", metav1.NamespaceDefault).GenerateName("profile-interactive-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.InteractiveMode). + Spec( + testPodTemplateWrapper.Clone(). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + TTY(). + Stdin(). + Obj().Template.Spec, + ). + Obj(), + }, + "should build job with replacements": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.InteractiveMode, + command: []string{"sleep"}, + requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("3")}, + localQueue: "lq1", + k8sObjs: []runtime.Object{testPodTemplateWrapper.Clone().Obj()}, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.InteractiveMode, Template: "pod-template"}). + WithVolumeBundleReferences("vb1", "vb2"). + Obj(), + wrappers.MakeVolumeBundle("vb1", metav1.NamespaceDefault). + WithVolume("v3", "config3"). + WithVolumeMount(corev1.VolumeMount{Name: "vm3", MountPath: "/etc/config3"}). + WithEnvVar(corev1.EnvVar{Name: "e3", Value: "value3"}). + Obj(), + wrappers.MakeVolumeBundle("vb2", metav1.NamespaceDefault).Obj(), + }, + wantRootObj: wrappers.MakePod("", metav1.NamespaceDefault).GenerateName("profile-interactive-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.InteractiveMode). + Label(kueueconstants.QueueLabel, "lq1"). + Spec( + testPodTemplateWrapper.Clone(). + Command([]string{"sleep"}). + WithRequest(corev1.ResourceCPU, resource.MustParse("3")). + WithVolume("v3", "config3"). + WithVolumeMount(corev1.VolumeMount{Name: "vm3", MountPath: "/etc/config3"}). + WithEnvVar(corev1.EnvVar{Name: "e3", Value: "value3"}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + TTY(). + Stdin(). + Obj().Template.Spec, + ). + Obj(), + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tcg := cmdtesting.NewTestClientGetter(). + WithKjobctlClientset(kjobctlfake.NewSimpleClientset(tc.kjobctlObjs...)). + WithK8sClientset(k8sfake.NewSimpleClientset(tc.k8sObjs...)) + + gotRootObj, gotChildObjs, gotErr := NewBuilder(tcg, testStartTime). + WithNamespace(tc.namespace). + WithProfileName(tc.profile). + WithModeName(tc.mode). + WithCommand(tc.command). + WithRequests(tc.requests). + WithLocalQueue(tc.localQueue). + WithSkipLocalQueueValidation(true). + Do(ctx) + + var opts []cmp.Option + var statusError *apierrors.StatusError + if !errors.As(tc.wantErr, &statusError) { + opts = append(opts, cmpopts.EquateErrors()) + } + if diff := cmp.Diff(tc.wantErr, gotErr, opts...); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + return + } + + if diff := cmp.Diff(tc.wantRootObj, gotRootObj, opts...); diff != "" { + t.Errorf("Root object after build (-want,+got):\n%s", diff) + } + + if diff := cmp.Diff([]runtime.Object(nil), gotChildObjs, opts...); diff != "" { + t.Errorf("Child objects after build (-want,+got):\n%s", diff) + } + }) + } +} diff --git a/pkg/builder/job_builder.go b/pkg/builder/job_builder.go new file mode 100644 index 0000000..4bc543b --- /dev/null +++ b/pkg/builder/job_builder.go @@ -0,0 +1,69 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + + batchv1 "k8s.io/api/batch/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +type jobBuilder struct { + *Builder +} + +var _ builder = (*jobBuilder)(nil) + +func (b *jobBuilder) build(ctx context.Context) (runtime.Object, []runtime.Object, error) { + template, err := b.kjobctlClientset.KjobctlV1alpha1().JobTemplates(b.profile.Namespace). + Get(ctx, string(b.mode.Template), metav1.GetOptions{}) + if err != nil { + return nil, nil, err + } + + objectMeta, err := b.buildObjectMeta(template.Template.ObjectMeta, false) + if err != nil { + return nil, nil, err + } + + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + Kind: "Job", + APIVersion: "batch/v1", + }, + ObjectMeta: objectMeta, + Spec: template.Template.Spec, + } + + job.Spec.Template.Spec = b.buildPodSpec(job.Spec.Template.Spec) + + if b.parallelism != nil { + job.Spec.Parallelism = b.parallelism + } + + if b.completions != nil { + job.Spec.Completions = b.completions + } + + return job, nil, nil +} + +func newJobBuilder(b *Builder) *jobBuilder { + return &jobBuilder{Builder: b} +} diff --git a/pkg/builder/job_builder_test.go b/pkg/builder/job_builder_test.go new file mode 100644 index 0000000..71e9b9b --- /dev/null +++ b/pkg/builder/job_builder_test.go @@ -0,0 +1,222 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/ptr" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + kjobctlfake "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/fake" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestJobBuilder(t *testing.T) { + testStartTime := time.Now() + userID := os.Getenv(constants.SystemEnvVarNameUser) + + testJobTemplateWrapper := wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault). + Label("foo", "bar"). + Annotation("foo", "baz"). + Parallelism(1). + Completions(1). + WithInitContainer( + *wrappers.MakeContainer("ic1", ""). + WithEnvVar(corev1.EnvVar{Name: "e0", Value: "default-value0"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm0", MountPath: "/etc/default-config0"}). + Obj(), + ). + WithContainer( + *wrappers.MakeContainer("c1", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("1")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithContainer(*wrappers.MakeContainer("c2", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithVolume("v1", "default-config1"). + WithVolume("v2", "default-config2") + + testCases := map[string]struct { + namespace string + profile string + mode v1alpha1.ApplicationProfileMode + command []string + parallelism *int32 + completions *int32 + requests corev1.ResourceList + localQueue string + kjobctlObjs []runtime.Object + wantRootObj runtime.Object + wantErr error + }{ + "shouldn't build job because template not found": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.JobMode, Template: "job-template"}). + Obj(), + }, + wantErr: apierrors.NewNotFound(schema.GroupResource{Group: "kjobctl.x-k8s.io", Resource: "jobtemplates"}, "job-template"), + }, + "should build job without replacements": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + testJobTemplateWrapper.Clone().Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.JobMode, Template: "job-template"}). + Obj(), + }, + wantRootObj: wrappers.MakeJob("", metav1.NamespaceDefault).GenerateName("profile-job-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Spec( + testJobTemplateWrapper.Clone(). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj().Template.Spec, + ). + Obj(), + }, + "should build job with replacements": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + command: []string{"sleep"}, + parallelism: ptr.To[int32](2), + completions: ptr.To[int32](3), + requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("3")}, + localQueue: "lq1", + kjobctlObjs: []runtime.Object{ + testJobTemplateWrapper.Clone().Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.JobMode, Template: "job-template"}). + WithVolumeBundleReferences("vb1", "vb2"). + Obj(), + wrappers.MakeVolumeBundle("vb1", metav1.NamespaceDefault). + WithVolume("v3", "config3"). + WithVolumeMount(corev1.VolumeMount{Name: "vm3", MountPath: "/etc/config3"}). + WithEnvVar(corev1.EnvVar{Name: "e3", Value: "value3"}). + Obj(), + wrappers.MakeVolumeBundle("vb2", metav1.NamespaceDefault).Obj(), + }, + wantRootObj: wrappers.MakeJob("", metav1.NamespaceDefault).GenerateName("profile-job-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Label(kueueconstants.QueueLabel, "lq1"). + Spec( + testJobTemplateWrapper.Clone(). + Command([]string{"sleep"}). + Parallelism(2). + Completions(3). + WithRequest(corev1.ResourceCPU, resource.MustParse("3")). + WithVolume("v3", "config3"). + WithVolumeMount(corev1.VolumeMount{Name: "vm3", MountPath: "/etc/config3"}). + WithEnvVar(corev1.EnvVar{Name: "e3", Value: "value3"}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj().Template.Spec, + ). + Obj(), + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tcg := cmdtesting.NewTestClientGetter(). + WithKjobctlClientset(kjobctlfake.NewSimpleClientset(tc.kjobctlObjs...)) + + gotRootObj, gotChildObjs, gotErr := NewBuilder(tcg, testStartTime). + WithNamespace(tc.namespace). + WithProfileName(tc.profile). + WithModeName(tc.mode). + WithCommand(tc.command). + WithParallelism(tc.parallelism). + WithCompletions(tc.completions). + WithRequests(tc.requests). + WithLocalQueue(tc.localQueue). + WithSkipLocalQueueValidation(true). + Do(ctx) + + var opts []cmp.Option + var statusError *apierrors.StatusError + if !errors.As(tc.wantErr, &statusError) { + opts = append(opts, cmpopts.EquateErrors()) + } + if diff := cmp.Diff(tc.wantErr, gotErr, opts...); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + return + } + + if diff := cmp.Diff(tc.wantRootObj, gotRootObj, opts...); diff != "" { + t.Errorf("Root object after build (-want,+got):\n%s", diff) + } + + if diff := cmp.Diff([]runtime.Object(nil), gotChildObjs, opts...); diff != "" { + t.Errorf("Child objects after build (-want,+got):\n%s", diff) + } + }) + } +} diff --git a/pkg/builder/ray_cluster_builder.go b/pkg/builder/ray_cluster_builder.go new file mode 100644 index 0000000..f84b871 --- /dev/null +++ b/pkg/builder/ray_cluster_builder.go @@ -0,0 +1,61 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +type rayClusterBuilder struct { + *Builder +} + +var _ builder = (*rayClusterBuilder)(nil) + +func (b *rayClusterBuilder) build(ctx context.Context) (runtime.Object, []runtime.Object, error) { + template, err := b.kjobctlClientset.KjobctlV1alpha1().RayClusterTemplates(b.profile.Namespace). + Get(ctx, string(b.mode.Template), metav1.GetOptions{}) + if err != nil { + return nil, nil, err + } + + objectMeta, err := b.buildObjectMeta(template.Template.ObjectMeta, false) + if err != nil { + return nil, nil, err + } + + rayCluster := &rayv1.RayCluster{ + TypeMeta: metav1.TypeMeta{ + Kind: "RayCluster", + APIVersion: "ray.io/v1", + }, + ObjectMeta: objectMeta, + Spec: template.Template.Spec, + } + + b.buildRayClusterSpec(&rayCluster.Spec) + + return rayCluster, nil, nil +} + +func newRayClusterBuilder(b *Builder) *rayClusterBuilder { + return &rayClusterBuilder{Builder: b} +} diff --git a/pkg/builder/ray_cluster_builder_test.go b/pkg/builder/ray_cluster_builder_test.go new file mode 100644 index 0000000..30ca75e --- /dev/null +++ b/pkg/builder/ray_cluster_builder_test.go @@ -0,0 +1,267 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + kjobctlfake "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/fake" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestRayClusterBuilder(t *testing.T) { + testStartTime := time.Now() + userID := os.Getenv(constants.SystemEnvVarNameUser) + + testRayClusterTemplateWrapper := wrappers.MakeRayClusterTemplate("ray-cluster-template", metav1.NamespaceDefault). + Label("foo", "bar"). + Annotation("foo", "baz"). + Spec(*wrappers.MakeRayClusterSpec(). + WithWorkerGroupSpec( + *wrappers.MakeWorkerGroupSpec("g1"). + WithInitContainer( + *wrappers.MakeContainer("ic1", ""). + WithEnvVar(corev1.EnvVar{Name: "e0", Value: "default-value0"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm0", MountPath: "/etc/default-config0"}). + Obj(), + ). + WithContainer( + *wrappers.MakeContainer("c1", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("1")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithContainer(*wrappers.MakeContainer("c2", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithVolume("v1", "default-config1"). + WithVolume("v2", "default-config2"). + Obj(), + ). + WithWorkerGroupSpec( + *wrappers.MakeWorkerGroupSpec("g2"). + WithInitContainer( + *wrappers.MakeContainer("ic1", ""). + WithEnvVar(corev1.EnvVar{Name: "e0", Value: "default-value0"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm0", MountPath: "/etc/default-config0"}). + Obj(), + ). + WithContainer( + *wrappers.MakeContainer("c1", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("1")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithContainer(*wrappers.MakeContainer("c2", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithVolume("v1", "default-config1"). + WithVolume("v2", "default-config2"). + Obj(), + ). + Obj(), + ) + + testCases := map[string]struct { + namespace string + profile string + mode v1alpha1.ApplicationProfileMode + command []string + replicas map[string]int + minReplicas map[string]int + maxReplicas map[string]int + requests corev1.ResourceList + localQueue string + kjobctlObjs []runtime.Object + wantRootObj runtime.Object + wantErr error + }{ + "shouldn't build ray cluster because template not found": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.RayClusterMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.RayClusterMode, Template: "ray-cluster-template"}). + Obj(), + }, + wantErr: apierrors.NewNotFound(schema.GroupResource{Group: "kjobctl.x-k8s.io", Resource: "rayclustertemplates"}, "ray-cluster-template"), + }, + "should build ray cluster without replacements": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.RayClusterMode, + kjobctlObjs: []runtime.Object{ + testRayClusterTemplateWrapper.Clone().Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.RayClusterMode, Template: "ray-cluster-template"}). + Obj(), + }, + wantRootObj: wrappers.MakeRayCluster("", metav1.NamespaceDefault).GenerateName("profile-raycluster-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.RayClusterMode). + Spec( + testRayClusterTemplateWrapper.Clone(). + Spec( + *wrappers.FromRayClusterSpec(testRayClusterTemplateWrapper.Clone().Template.Spec). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj(), + ). + Template.Spec, + ). + Obj(), + }, + "should build ray cluster with replacements": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.RayClusterMode, + command: []string{"sleep"}, + replicas: map[string]int{"g1": 10, "g2": 20}, + minReplicas: map[string]int{"g1": 10, "g2": 20}, + maxReplicas: map[string]int{"g1": 15, "g2": 25}, + requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("3")}, + localQueue: "lq1", + kjobctlObjs: []runtime.Object{ + testRayClusterTemplateWrapper.Clone().Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.RayClusterMode, Template: "ray-cluster-template"}). + WithVolumeBundleReferences("vb1", "vb2"). + Obj(), + wrappers.MakeVolumeBundle("vb1", metav1.NamespaceDefault). + WithVolume("v3", "config3"). + WithVolumeMount(corev1.VolumeMount{Name: "vm3", MountPath: "/etc/config3"}). + WithEnvVar(corev1.EnvVar{Name: "e3", Value: "value3"}). + Obj(), + wrappers.MakeVolumeBundle("vb2", metav1.NamespaceDefault).Obj(), + }, + wantRootObj: wrappers.MakeRayCluster("", metav1.NamespaceDefault).GenerateName("profile-raycluster-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.RayClusterMode). + Label(kueueconstants.QueueLabel, "lq1"). + Spec( + testRayClusterTemplateWrapper.Clone(). + Spec( + *wrappers.FromRayClusterSpec(testRayClusterTemplateWrapper.Clone().Template.Spec). + Replicas("g1", 10). + Replicas("g2", 20). + MinReplicas("g1", 10). + MinReplicas("g2", 20). + MaxReplicas("g1", 15). + MaxReplicas("g2", 25). + WithVolume("v3", "config3"). + WithVolumeMount(corev1.VolumeMount{Name: "vm3", MountPath: "/etc/config3"}). + WithEnvVar(corev1.EnvVar{Name: "e3", Value: "value3"}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj(), + ). + Obj().Template.Spec, + ). + Obj(), + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tcg := cmdtesting.NewTestClientGetter(). + WithKjobctlClientset(kjobctlfake.NewSimpleClientset(tc.kjobctlObjs...)) + + gotRootObj, gotChildObjs, gotErr := NewBuilder(tcg, testStartTime). + WithNamespace(tc.namespace). + WithProfileName(tc.profile). + WithModeName(tc.mode). + WithCommand(tc.command). + WithReplicas(tc.replicas). + WithMinReplicas(tc.minReplicas). + WithMaxReplicas(tc.maxReplicas). + WithLocalQueue(tc.localQueue). + WithSkipLocalQueueValidation(true). + Do(ctx) + + var opts []cmp.Option + var statusError *apierrors.StatusError + if !errors.As(tc.wantErr, &statusError) { + opts = append(opts, cmpopts.EquateErrors()) + } + if diff := cmp.Diff(tc.wantErr, gotErr, opts...); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + return + } + + if diff := cmp.Diff(tc.wantRootObj, gotRootObj, opts...); diff != "" { + t.Errorf("Root object after build (-want,+got):\n%s", diff) + } + + if diff := cmp.Diff([]runtime.Object(nil), gotChildObjs, opts...); diff != "" { + t.Errorf("Child objects after build (-want,+got):\n%s", diff) + } + }) + } +} diff --git a/pkg/builder/ray_job_builder.go b/pkg/builder/ray_job_builder.go new file mode 100644 index 0000000..1969e13 --- /dev/null +++ b/pkg/builder/ray_job_builder.go @@ -0,0 +1,77 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + "strings" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + rayutil "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" +) + +type rayJobBuilder struct { + *Builder +} + +var _ builder = (*rayJobBuilder)(nil) + +func (b *rayJobBuilder) build(ctx context.Context) (runtime.Object, []runtime.Object, error) { + template, err := b.kjobctlClientset.KjobctlV1alpha1().RayJobTemplates(b.profile.Namespace). + Get(ctx, string(b.mode.Template), metav1.GetOptions{}) + if err != nil { + return nil, nil, err + } + + objectMeta, err := b.buildObjectMeta(template.Template.ObjectMeta, false) + if err != nil { + return nil, nil, err + } + + rayJob := &rayv1.RayJob{ + TypeMeta: metav1.TypeMeta{ + Kind: "RayJob", + APIVersion: "ray.io/v1", + }, + ObjectMeta: objectMeta, + Spec: template.Template.Spec, + } + + if b.command != nil { + rayJob.Spec.Entrypoint = strings.Join(b.command, " ") + } + + if len(b.rayCluster) != 0 { + delete(rayJob.ObjectMeta.Labels, kueueconstants.QueueLabel) + rayJob.Spec.RayClusterSpec = nil + if rayJob.Spec.ClusterSelector == nil { + rayJob.Spec.ClusterSelector = map[string]string{} + } + rayJob.Spec.ClusterSelector[rayutil.RayClusterLabelKey] = b.rayCluster + } else if rayJob.Spec.RayClusterSpec != nil { + b.buildRayClusterSpec(rayJob.Spec.RayClusterSpec) + } + + return rayJob, nil, nil +} + +func newRayJobBuilder(b *Builder) *rayJobBuilder { + return &rayJobBuilder{Builder: b} +} diff --git a/pkg/builder/ray_job_builder_test.go b/pkg/builder/ray_job_builder_test.go new file mode 100644 index 0000000..272bee0 --- /dev/null +++ b/pkg/builder/ray_job_builder_test.go @@ -0,0 +1,300 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + kjobctlfake "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/fake" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestRayJobBuilder(t *testing.T) { + testStartTime := time.Now() + userID := os.Getenv(constants.SystemEnvVarNameUser) + + testRayJobTemplateWrapper := wrappers.MakeRayJobTemplate("ray-job-template", metav1.NamespaceDefault). + Label("foo", "bar"). + Annotation("foo", "baz"). + WithRayClusterSpec(wrappers.MakeRayClusterSpec(). + WithWorkerGroupSpec( + *wrappers.MakeWorkerGroupSpec("g1"). + WithInitContainer( + *wrappers.MakeContainer("ic1", ""). + WithEnvVar(corev1.EnvVar{Name: "e0", Value: "default-value0"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm0", MountPath: "/etc/default-config0"}). + Obj(), + ). + WithContainer( + *wrappers.MakeContainer("c1", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("1")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithContainer(*wrappers.MakeContainer("c2", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithVolume("v1", "default-config1"). + WithVolume("v2", "default-config2"). + Obj(), + ). + WithWorkerGroupSpec( + *wrappers.MakeWorkerGroupSpec("g2"). + WithInitContainer( + *wrappers.MakeContainer("ic1", ""). + WithEnvVar(corev1.EnvVar{Name: "e0", Value: "default-value0"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm0", MountPath: "/etc/default-config0"}). + Obj(), + ). + WithContainer( + *wrappers.MakeContainer("c1", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("1")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithContainer(*wrappers.MakeContainer("c2", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithVolume("v1", "default-config1"). + WithVolume("v2", "default-config2"). + Obj(), + ). + Obj(), + ) + + testCases := map[string]struct { + namespace string + profile string + mode v1alpha1.ApplicationProfileMode + command []string + replicas map[string]int + minReplicas map[string]int + maxReplicas map[string]int + requests corev1.ResourceList + localQueue string + rayCluster string + kjobctlObjs []runtime.Object + wantRootObj runtime.Object + wantErr error + }{ + "shouldn't build ray job because template not found": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.RayJobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.RayJobMode, Template: "ray-job-template"}). + Obj(), + }, + wantErr: apierrors.NewNotFound(schema.GroupResource{Group: "kjobctl.x-k8s.io", Resource: "rayjobtemplates"}, "ray-job-template"), + }, + "should build ray job without replacements": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.RayJobMode, + kjobctlObjs: []runtime.Object{ + testRayJobTemplateWrapper.Clone().Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.RayJobMode, Template: "ray-job-template"}). + Obj(), + }, + wantRootObj: wrappers.MakeRayJob("", metav1.NamespaceDefault).GenerateName("profile-rayjob-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + Spec( + testRayJobTemplateWrapper.Clone(). + WithRayClusterSpec( + wrappers.FromRayClusterSpec(*testRayJobTemplateWrapper.Clone().Template.Spec.RayClusterSpec). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj(), + ). + Template.Spec, + ). + Obj(), + }, + "should build ray job with replacements": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.RayJobMode, + command: []string{"sleep"}, + replicas: map[string]int{"g1": 10, "g2": 20}, + minReplicas: map[string]int{"g1": 10, "g2": 20}, + maxReplicas: map[string]int{"g1": 15, "g2": 25}, + requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("3")}, + localQueue: "lq1", + kjobctlObjs: []runtime.Object{ + testRayJobTemplateWrapper.Clone().Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.RayJobMode, Template: "ray-job-template"}). + WithVolumeBundleReferences("vb1", "vb2"). + Obj(), + wrappers.MakeVolumeBundle("vb1", metav1.NamespaceDefault). + WithVolume("v3", "config3"). + WithVolumeMount(corev1.VolumeMount{Name: "vm3", MountPath: "/etc/config3"}). + WithEnvVar(corev1.EnvVar{Name: "e3", Value: "value3"}). + Obj(), + wrappers.MakeVolumeBundle("vb2", metav1.NamespaceDefault).Obj(), + }, + wantRootObj: wrappers.MakeRayJob("", metav1.NamespaceDefault).GenerateName("profile-rayjob-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + Label(kueueconstants.QueueLabel, "lq1"). + Spec( + testRayJobTemplateWrapper.Clone(). + Entrypoint("sleep"). + WithRayClusterSpec( + wrappers.FromRayClusterSpec(*testRayJobTemplateWrapper.Clone().Template.Spec.RayClusterSpec). + Replicas("g1", 10). + Replicas("g2", 20). + MinReplicas("g1", 10). + MinReplicas("g2", 20). + MaxReplicas("g1", 15). + MaxReplicas("g2", 25). + WithVolume("v3", "config3"). + WithVolumeMount(corev1.VolumeMount{Name: "vm3", MountPath: "/etc/config3"}). + WithEnvVar(corev1.EnvVar{Name: "e3", Value: "value3"}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj(), + ). + Obj().Template.Spec, + ). + Obj(), + }, + "should build ray job with raycluster replacement": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.RayJobMode, + command: []string{"python /home/ray/samples/sample_code.py"}, + requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("3")}, + localQueue: "lq1", + rayCluster: "rc1", + kjobctlObjs: []runtime.Object{ + testRayJobTemplateWrapper.Clone().Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.RayJobMode, Template: "ray-job-template"}). + WithVolumeBundleReferences("vb1", "vb2"). + Obj(), + wrappers.MakeVolumeBundle("vb1", metav1.NamespaceDefault). + WithVolume("v3", "config3"). + WithVolumeMount(corev1.VolumeMount{Name: "vm3", MountPath: "/etc/config3"}). + WithEnvVar(corev1.EnvVar{Name: "e3", Value: "value3"}). + Obj(), + wrappers.MakeVolumeBundle("vb2", metav1.NamespaceDefault).Obj(), + }, + wantRootObj: wrappers.MakeRayJob("", metav1.NamespaceDefault).GenerateName("profile-rayjob-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + WithRayClusterLabelSelector("rc1"). + Entrypoint("python /home/ray/samples/sample_code.py"). + Obj(), + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tcg := cmdtesting.NewTestClientGetter(). + WithKjobctlClientset(kjobctlfake.NewSimpleClientset(tc.kjobctlObjs...)) + + gotRootObj, gotChildObjs, gotErr := NewBuilder(tcg, testStartTime). + WithNamespace(tc.namespace). + WithProfileName(tc.profile). + WithModeName(tc.mode). + WithCommand(tc.command). + WithReplicas(tc.replicas). + WithMinReplicas(tc.minReplicas). + WithMaxReplicas(tc.maxReplicas). + WithLocalQueue(tc.localQueue). + WithSkipLocalQueueValidation(true). + WithRayCluster(tc.rayCluster). + Do(ctx) + + var opts []cmp.Option + var statusError *apierrors.StatusError + if !errors.As(tc.wantErr, &statusError) { + opts = append(opts, cmpopts.EquateErrors()) + } + if diff := cmp.Diff(tc.wantErr, gotErr, opts...); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + return + } + + if diff := cmp.Diff(tc.wantRootObj, gotRootObj, opts...); diff != "" { + t.Errorf("Root object after build (-want,+got):\n%s", diff) + } + + if diff := cmp.Diff([]runtime.Object(nil), gotChildObjs, opts...); diff != "" { + t.Errorf("Child objects after build (-want,+got):\n%s", diff) + } + }) + } +} diff --git a/pkg/builder/slurm_builder.go b/pkg/builder/slurm_builder.go new file mode 100644 index 0000000..fd1b643 --- /dev/null +++ b/pkg/builder/slurm_builder.go @@ -0,0 +1,775 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "bytes" + "context" + "embed" + "errors" + "fmt" + "math" + "os" + "strconv" + "strings" + "text/template" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + kjobctlconstants "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/parser" +) + +//go:embed templates/slurm_* +var slurmTemplates embed.FS + +const ( + // Note that the first job ID will always be 1. + slurmArrayJobID = 1 + + slurmScriptsPath = "/slurm/scripts" + slurmInitEntrypointFilename = "init-entrypoint.sh" + slurmEntrypointFilename = "entrypoint.sh" + slurmScriptFilename = "script" + + slurmEnvsPath = "/slurm/env" + slurmSlurmEnvFilename = "slurm.env" +) + +var ( + noScriptSpecifiedErr = errors.New("no script specified") +) + +var ( + slurmInitEntrypointFilenamePath = fmt.Sprintf("%s/%s", slurmScriptsPath, slurmInitEntrypointFilename) + slurmEntrypointFilenamePath = fmt.Sprintf("%s/%s", slurmScriptsPath, slurmEntrypointFilename) + slurmScriptFilenamePath = fmt.Sprintf("%s/%s", slurmScriptsPath, slurmScriptFilename) + + unmaskReplacer = strings.NewReplacer( + "%%", "%", + "%A", "${SLURM_ARRAY_JOB_ID}", + "%a", "${SLURM_ARRAY_TASK_ID}", + "%j", "${SLURM_JOB_ID}", + "%N", "${HOSTNAME}", + "%n", "${JOB_COMPLETION_INDEX}", + "%t", "${SLURM_ARRAY_TASK_ID}", + "%u", "${USER_ID}", + "%x", "${SBATCH_JOB_NAME}", + ) +) + +type slurmBuilder struct { + *Builder + + scriptContent string + template *template.Template + arrayIndexes parser.ArrayIndexes + cpusOnNode *resource.Quantity + cpusPerGpu *resource.Quantity + totalMemPerNode *resource.Quantity + totalGpus *resource.Quantity +} + +var _ builder = (*slurmBuilder)(nil) + +func (b *slurmBuilder) validateGeneral() error { + if len(b.script) == 0 { + return noScriptSpecifiedErr + } + + if b.memPerCPU != nil && b.cpusPerTask == nil { + return noCpusPerTaskSpecifiedErr + } + + if b.memPerGPU != nil && b.gpusPerTask == nil { + return noGpusPerTaskSpecifiedErr + } + + return nil +} + +func (b *slurmBuilder) complete() error { + content, err := os.ReadFile(b.script) + if err != nil { + return err + } + b.scriptContent = string(content) + + t, err := template.ParseFS(slurmTemplates, "templates/*") + if err != nil { + return err + } + b.template = t + + if err := b.getSbatchEnvs(); err != nil { + return err + } + + if err := b.replaceScriptFlags(); err != nil { + return err + } + + if err := b.validateMutuallyExclusiveFlags(); err != nil { + return err + } + + if b.array == "" { + b.arrayIndexes = parser.GenerateArrayIndexes(ptr.Deref(b.nodes, 1) * ptr.Deref(b.nTasks, 1)) + } else { + b.arrayIndexes, err = parser.ParseArrayIndexes(b.array) + if err != nil { + return err + } + if b.arrayIndexes.Parallelism != nil { + b.nodes = b.arrayIndexes.Parallelism + } + } + + return nil +} + +func (b *slurmBuilder) validateMutuallyExclusiveFlags() error { + flags := map[string]bool{ + string(v1alpha1.MemPerNodeFlag): b.memPerNode != nil, + string(v1alpha1.MemPerTaskFlag): b.memPerTask != nil, + string(v1alpha1.MemPerCPUFlag): b.memPerCPU != nil, + string(v1alpha1.MemPerGPUFlag): b.memPerGPU != nil, + } + + var setFlagsCount int + setFlags := make([]string, 0) + for f, isSet := range flags { + if isSet { + setFlagsCount++ + setFlags = append(setFlags, f) + } + } + + if setFlagsCount > 1 { + return fmt.Errorf( + "if any flags in the group [%s %s %s] are set none of the others can be; [%s] were all set", + v1alpha1.MemPerTaskFlag, + v1alpha1.MemPerGPUFlag, + v1alpha1.MemPerGPUFlag, + strings.Join(setFlags, " "), + ) + } + + return nil +} + +func (b *slurmBuilder) build(ctx context.Context) (runtime.Object, []runtime.Object, error) { + if err := b.validateGeneral(); err != nil { + return nil, nil, err + } + + if err := b.complete(); err != nil { + return nil, nil, err + } + + template, err := b.kjobctlClientset.KjobctlV1alpha1().JobTemplates(b.profile.Namespace). + Get(ctx, string(b.mode.Template), metav1.GetOptions{}) + if err != nil { + return nil, nil, err + } + + objectMeta, err := b.buildObjectMeta(template.Template.ObjectMeta, true) + if err != nil { + return nil, nil, err + } + + if b.script != "" { + if objectMeta.Annotations == nil { + objectMeta.Annotations = make(map[string]string, 1) + } + objectMeta.Annotations[kjobctlconstants.ScriptAnnotation] = b.script + } + + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{Kind: "Job", APIVersion: "batch/v1"}, + ObjectMeta: objectMeta, + Spec: template.Template.Spec, + } + + job.Spec.CompletionMode = ptr.To(batchv1.IndexedCompletion) + job.Spec.Template.Spec.Subdomain = job.Name + + b.buildPodSpecVolumesAndEnv(&job.Spec.Template.Spec) + job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, + corev1.Volume{ + Name: "slurm-scripts", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: job.Name, + }, + Items: []corev1.KeyToPath{ + { + Key: slurmInitEntrypointFilename, + Path: slurmInitEntrypointFilename, + }, + { + Key: slurmEntrypointFilename, + Path: slurmEntrypointFilename, + }, + { + Key: slurmScriptFilename, + Path: slurmScriptFilename, + Mode: ptr.To[int32](0755), + }, + }, + }, + }, + }, + corev1.Volume{ + Name: "slurm-env", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + ) + + job.Spec.Template.Spec.InitContainers = append(job.Spec.Template.Spec.InitContainers, corev1.Container{ + Name: "slurm-init-env", + Image: b.initImage, + Command: []string{"sh", slurmInitEntrypointFilenamePath}, + Env: []corev1.EnvVar{ + { + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"}, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "slurm-scripts", + MountPath: slurmScriptsPath, + }, + { + Name: "slurm-env", + MountPath: slurmEnvsPath, + }, + }, + }) + + var totalGPUsPerTask resource.Quantity + for _, number := range b.gpusPerTask { + totalGPUsPerTask.Add(*number) + } + + var memPerCPU, memPerGPU, memPerContainer resource.Quantity + if b.memPerCPU != nil && b.cpusPerTask != nil { + memPerCPU = *b.memPerCPU + memPerCPU.Mul(b.cpusPerTask.Value()) + } + + if b.memPerGPU != nil && b.gpusPerTask != nil { + memPerGPU = *b.memPerGPU + memPerGPU.Mul(totalGPUsPerTask.Value()) + } + + if b.memPerNode != nil { + mem := b.memPerNode.MilliValue() / int64(len(job.Spec.Template.Spec.Containers)) + memPerContainer = *resource.NewMilliQuantity(mem, b.memPerNode.Format) + } + + var totalCpus, totalGpus, totalMem resource.Quantity + for i := range job.Spec.Template.Spec.Containers { + container := &job.Spec.Template.Spec.Containers[i] + + container.Command = []string{"bash", slurmEntrypointFilenamePath} + + var requests corev1.ResourceList + if b.requests != nil { + requests = b.requests + } else { + requests = corev1.ResourceList{} + } + + if b.cpusPerTask != nil { + requests[corev1.ResourceCPU] = *b.cpusPerTask + totalCpus.Add(*b.cpusPerTask) + } + + if b.gpusPerTask != nil { + for name, number := range b.gpusPerTask { + requests[corev1.ResourceName(name)] = *number + } + totalGpus.Add(totalGPUsPerTask) + } + + if b.memPerTask != nil { + requests[corev1.ResourceMemory] = *b.memPerTask + totalMem.Add(*b.memPerTask) + } + + if !memPerCPU.IsZero() { + requests[corev1.ResourceMemory] = memPerCPU + totalMem.Add(memPerCPU) + } + + if !memPerGPU.IsZero() { + requests[corev1.ResourceMemory] = memPerGPU + totalMem.Add(memPerGPU) + } + + if len(requests) > 0 { + container.Resources.Requests = requests + } + + limits := corev1.ResourceList{} + if !memPerContainer.IsZero() { + limits[corev1.ResourceMemory] = memPerContainer + } + + if len(limits) > 0 { + container.Resources.Limits = limits + } + + container.VolumeMounts = append(container.VolumeMounts, + corev1.VolumeMount{ + Name: "slurm-scripts", + MountPath: slurmScriptsPath, + }, + corev1.VolumeMount{ + Name: "slurm-env", + MountPath: slurmEnvsPath, + }, + ) + } + + nTasks := ptr.Deref(b.nTasks, 1) + completions := int32(math.Ceil(float64(b.arrayIndexes.Count()) / float64(nTasks))) + + job.Spec.Completions = ptr.To(completions) + job.Spec.Parallelism = b.nodes + + if nTasks > 1 { + for i := 1; i < int(nTasks); i++ { + replica := job.Spec.Template.Spec.Containers[0].DeepCopy() + replica.Name = fmt.Sprintf("%s-%d", job.Spec.Template.Spec.Containers[0].Name, i) + job.Spec.Template.Spec.Containers = append(job.Spec.Template.Spec.Containers, *replica) + + if b.cpusPerTask != nil { + totalCpus.Add(*b.cpusPerTask) + } + + if !memPerCPU.IsZero() { + totalMem.Add(memPerCPU) + } + + if !memPerGPU.IsZero() { + totalMem.Add(memPerGPU) + } + } + + job.Spec.Template.Spec.Containers[0].Name = fmt.Sprintf("%s-0", job.Spec.Template.Spec.Containers[0].Name) + } + + for i := range job.Spec.Template.Spec.Containers { + job.Spec.Template.Spec.Containers[i].Env = append(job.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{ + Name: "JOB_CONTAINER_INDEX", + Value: strconv.FormatInt(int64(i), 10), + }) + } + + if b.nodes != nil { + job.Spec.Parallelism = b.nodes + } + + if !totalCpus.IsZero() { + b.cpusOnNode = &totalCpus + } + + if !totalGpus.IsZero() { + b.totalGpus = &totalGpus + } + + if b.memPerNode != nil { + b.totalMemPerNode = b.memPerNode + } else if !totalMem.IsZero() { + b.totalMemPerNode = &totalMem + } + + if b.cpusOnNode != nil && b.totalGpus != nil && !totalGpus.IsZero() { + cpusPerGpu := totalCpus.MilliValue() / totalGpus.MilliValue() + b.cpusPerGpu = resource.NewQuantity(cpusPerGpu, b.cpusOnNode.Format) + } + + initEntrypointScript, err := b.buildInitEntrypointScript(job.Name) + if err != nil { + return nil, nil, err + } + + entrypointScript, err := b.buildEntrypointScript() + if err != nil { + return nil, nil, err + } + + configMap := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: b.buildChildObjectMeta(job.Name), + Data: map[string]string{ + slurmInitEntrypointFilename: initEntrypointScript, + slurmEntrypointFilename: entrypointScript, + slurmScriptFilename: b.scriptContent, + }, + } + + service := &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: b.buildChildObjectMeta(job.Name), + Spec: corev1.ServiceSpec{ + ClusterIP: "None", + Selector: map[string]string{ + "job-name": job.Name, + }, + }, + } + + return job, []runtime.Object{configMap, service}, nil +} + +func (b *slurmBuilder) buildArrayIndexes() string { + nTasks := ptr.Deref(b.nTasks, 1) + length := int64(math.Ceil(float64(len(b.arrayIndexes.Indexes)) / float64(nTasks))) + containerIndexes := make([][]string, length) + + var ( + completionIndex int32 + containerIndex int32 + ) + for _, index := range b.arrayIndexes.Indexes { + containerIndexes[completionIndex] = append(containerIndexes[completionIndex], fmt.Sprint(index)) + containerIndex++ + if containerIndex >= nTasks { + containerIndex = 0 + completionIndex++ + } + } + + completionIndexes := make([]string, length) + for completionIndex, containerIndexes := range containerIndexes { + completionIndexes[completionIndex] = strings.Join(containerIndexes, ",") + } + + return strings.Join(completionIndexes, ";") +} + +type slurmInitEntrypointScript struct { + ArrayIndexes string + + JobName string + Namespace string + + EnvsPath string + SlurmEnvFilename string + + SlurmArrayJobID int32 + SlurmArrayTaskCount int32 + SlurmArrayTaskMax int32 + SlurmArrayTaskMin int32 + SlurmTasksPerNode int32 + SlurmCPUsPerTask string + SlurmCPUsOnNode string + SlurmJobCPUsPerNode string + SlurmCPUsPerGPU string + SlurmMemPerCPU string + SlurmMemPerGPU string + SlurmMemPerNode string + SlurmGPUs string + SlurmNTasks int32 + SlurmNTasksPerNode int32 + SlurmNProcs int32 + SlurmNNodes int32 + SlurmSubmitDir string + SlurmJobNodeList string + SlurmJobFirstNode string + + FirstNodeIP bool + FirstNodeIPTimeoutSeconds int32 +} + +func (b *slurmBuilder) buildInitEntrypointScript(jobName string) (string, error) { + nTasks := ptr.Deref(b.nTasks, 1) + nodes := ptr.Deref(b.nodes, 1) + + nodeList := make([]string, nodes) + for i := int32(0); i < nodes; i++ { + nodeList[i] = fmt.Sprintf("%s-%d.%s", jobName, i, jobName) + } + + scriptValues := slurmInitEntrypointScript{ + ArrayIndexes: b.buildArrayIndexes(), + + JobName: jobName, + Namespace: b.namespace, + + EnvsPath: slurmEnvsPath, + SlurmEnvFilename: slurmSlurmEnvFilename, + + SlurmArrayJobID: slurmArrayJobID, + SlurmArrayTaskCount: int32(b.arrayIndexes.Count()), + SlurmArrayTaskMax: b.arrayIndexes.Max(), + SlurmArrayTaskMin: b.arrayIndexes.Min(), + SlurmTasksPerNode: nTasks, + SlurmCPUsPerTask: getValueOrEmpty(b.cpusPerTask), + SlurmCPUsOnNode: getValueOrEmpty(b.cpusOnNode), + SlurmJobCPUsPerNode: getValueOrEmpty(b.cpusOnNode), + SlurmCPUsPerGPU: getValueOrEmpty(b.cpusPerGpu), + SlurmMemPerCPU: getValueOrEmpty(b.memPerCPU), + SlurmMemPerGPU: getValueOrEmpty(b.memPerGPU), + SlurmMemPerNode: getValueOrEmpty(b.totalMemPerNode), + SlurmGPUs: getValueOrEmpty(b.totalGpus), + SlurmNTasks: nTasks, + SlurmNTasksPerNode: nTasks, + SlurmNProcs: nTasks, + SlurmNNodes: nodes, + SlurmSubmitDir: slurmScriptsPath, + SlurmJobNodeList: strings.Join(nodeList, ","), + SlurmJobFirstNode: nodeList[0], + + FirstNodeIP: b.firstNodeIP, + FirstNodeIPTimeoutSeconds: int32(b.firstNodeIPTimeout.Seconds()), + } + + var script bytes.Buffer + + if err := b.template.ExecuteTemplate(&script, "slurm_init_entrypoint_script.sh.tmpl", scriptValues); err != nil { + return "", err + } + + return script.String(), nil +} + +type slurmEntrypointScript struct { + EnvsPath string + SbatchJobName string + SlurmEnvFilename string + ChangeDir string + BuildEntrypointCommand string +} + +func (b *slurmBuilder) buildEntrypointScript() (string, error) { + scriptValues := slurmEntrypointScript{ + EnvsPath: slurmEnvsPath, + SbatchJobName: b.jobName, + SlurmEnvFilename: slurmSlurmEnvFilename, + ChangeDir: b.changeDir, + BuildEntrypointCommand: b.buildEntrypointCommand(), + } + + var script bytes.Buffer + + if err := b.template.ExecuteTemplate(&script, "slurm_entrypoint_script.sh.tmpl", scriptValues); err != nil { + return "", err + } + + return script.String(), nil +} + +func getValueOrEmpty(ptr *resource.Quantity) string { + if ptr != nil { + return ptr.String() + } + + return "" +} + +// unmaskFilename unmasks a filename based on the filename pattern. +// For more details, see https://slurm.schedmd.com/sbatch.html#SECTION_FILENAME-PATTERN. +func unmaskFilename(filename string) string { + if strings.Contains(filename, "\\\\") { + return strings.ReplaceAll(filename, "\\\\", "") + } + return unmaskReplacer.Replace(filename) +} + +func (b *slurmBuilder) buildEntrypointCommand() string { + strBuilder := strings.Builder{} + + strBuilder.WriteString(slurmScriptFilenamePath) + + if b.input != "" { + strBuilder.WriteString(" <") + strBuilder.WriteString(unmaskFilename(b.input)) + } + + if b.output != "" { + strBuilder.WriteString(" 1>") + strBuilder.WriteString(unmaskFilename(b.output)) + } + + if b.error != "" { + strBuilder.WriteString(" 2>") + strBuilder.WriteString(unmaskFilename(b.error)) + } + + return strBuilder.String() +} + +func (b *slurmBuilder) getSbatchEnvs() error { + if len(b.array) == 0 { + b.array = os.Getenv("SBATCH_ARRAY_INX") + } + + if b.gpusPerTask == nil { + if env, ok := os.LookupEnv("SBATCH_GPUS_PER_TASK"); ok { + val, err := parser.GpusFlag(env) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", env, err) + } + b.gpusPerTask = val + } + } + + if b.memPerNode == nil { + if env, ok := os.LookupEnv("SBATCH_MEM_PER_NODE"); ok { + val, err := resource.ParseQuantity(env) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", env, err) + } + b.memPerNode = ptr.To(val) + } + } + + if b.memPerTask == nil { + if env, ok := os.LookupEnv("SBATCH_MEM_PER_CPU"); ok { + val, err := resource.ParseQuantity(env) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", env, err) + } + b.memPerTask = ptr.To(val) + } + } + + if b.memPerGPU == nil { + if env, ok := os.LookupEnv("SBATCH_MEM_PER_GPU"); ok { + val, err := resource.ParseQuantity(env) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", env, err) + } + b.memPerGPU = ptr.To(val) + } + } + + if len(b.output) == 0 { + b.output = os.Getenv("SBATCH_OUTPUT") + } + + if len(b.error) == 0 { + b.error = os.Getenv("SBATCH_ERROR") + } + + if len(b.input) == 0 { + b.input = os.Getenv("SBATCH_INPUT") + } + + if len(b.jobName) == 0 { + b.jobName = os.Getenv("SBATCH_JOB_NAME") + } + + if len(b.partition) == 0 { + b.partition = os.Getenv("SBATCH_PARTITION") + } + + if b.timeLimit == "" { + b.timeLimit = os.Getenv("SBATCH_TIMELIMIT") + } + + return nil +} + +func (b *slurmBuilder) replaceScriptFlags() error { + scriptFlags, err := parser.SlurmFlags(b.scriptContent, b.ignoreUnknown) + if err != nil { + return err + } + + if len(b.array) == 0 { + b.array = scriptFlags.Array + } + + if b.cpusPerTask == nil { + b.cpusPerTask = scriptFlags.CpusPerTask + } + + if b.gpusPerTask == nil { + b.gpusPerTask = scriptFlags.GpusPerTask + } + + if b.memPerNode == nil { + b.memPerNode = scriptFlags.MemPerNode + } + + if b.memPerTask == nil { + b.memPerTask = scriptFlags.MemPerTask + } + + if b.memPerCPU == nil { + b.memPerCPU = scriptFlags.MemPerCPU + } + + if b.memPerGPU == nil { + b.memPerGPU = scriptFlags.MemPerGPU + } + + if b.nodes == nil { + b.nodes = scriptFlags.Nodes + } + + if b.nTasks == nil { + b.nTasks = scriptFlags.NTasks + } + + if len(b.output) == 0 { + b.output = scriptFlags.Output + } + + if len(b.error) == 0 { + b.error = scriptFlags.Error + } + + if len(b.input) == 0 { + b.input = scriptFlags.Input + } + + if len(b.jobName) == 0 { + b.jobName = scriptFlags.JobName + } + + if len(b.partition) == 0 { + b.partition = scriptFlags.Partition + } + + if b.timeLimit == "" { + b.timeLimit = scriptFlags.TimeLimit + } + + return nil +} + +func newSlurmBuilder(b *Builder) *slurmBuilder { + return &slurmBuilder{Builder: b} +} diff --git a/pkg/builder/slurm_builder_test.go b/pkg/builder/slurm_builder_test.go new file mode 100644 index 0000000..78fe933 --- /dev/null +++ b/pkg/builder/slurm_builder_test.go @@ -0,0 +1,413 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + "errors" + "fmt" + "os" + "regexp" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/ptr" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + kjobctlfake "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/fake" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +type slurmBuilderTestCase struct { + beforeTest func(t *testing.T, tc *slurmBuilderTestCase) + tempFile string + namespace string + profile string + mode v1alpha1.ApplicationProfileMode + array string + cpusPerTask *resource.Quantity + gpusPerTask map[string]*resource.Quantity + memPerTask *resource.Quantity + memPerCPU *resource.Quantity + memPerGPU *resource.Quantity + nodes *int32 + nTasks *int32 + output string + err string + input string + jobName string + partition string + initImage string + firstNodeTimeout time.Duration + kjobctlObjs []runtime.Object + wantRootObj runtime.Object + wantChildObjs []runtime.Object + wantErr error + cmpopts []cmp.Option +} + +func beforeSlurmTest(t *testing.T, tc *slurmBuilderTestCase) { + file, err := os.CreateTemp("", "slurm") + if err != nil { + t.Fatal(err) + } + defer file.Close() + t.Cleanup(func() { + if err := os.Remove(file.Name()); err != nil { + t.Fatal(err) + } + }) + + if _, err := file.WriteString("#!/bin/bash\nsleep 300'"); err != nil { + t.Fatal(err) + } + + tc.tempFile = file.Name() +} + +func TestSlurmBuilderDo(t *testing.T) { + testStartTime := time.Now() + userID := os.Getenv(constants.SystemEnvVarNameUser) + + testCases := map[string]slurmBuilderTestCase{ + "shouldn't build slurm job because script not specified": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.SlurmMode, Template: "slurm-template"}). + Obj(), + }, + wantErr: noScriptSpecifiedErr, + }, + "shouldn't build slurm job because template not found": { + beforeTest: beforeSlurmTest, + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.SlurmMode, Template: "slurm-template"}). + Obj(), + }, + wantErr: apierrors.NewNotFound(schema.GroupResource{Group: "kjobctl.x-k8s.io", Resource: "jobtemplates"}, "slurm-template"), + }, + "should build slurm job": { + beforeTest: beforeSlurmTest, + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + array: "1-5%2", + initImage: "bash:latest", + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + wantRootObj: wrappers.MakeJob("", metav1.NamespaceDefault). + Parallelism(2). + Completions(5). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Subdomain("profile-slurm"). + WithInitContainer(*wrappers.MakeContainer("slurm-init-env", "bash:latest"). + Command("sh", "/slurm/scripts/init-entrypoint.sh"). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-scripts", MountPath: "/slurm/scripts"}). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-env", MountPath: "/slurm/env"}). + WithEnvVar(corev1.EnvVar{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"}, + }}). + Obj()). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-scripts", MountPath: "/slurm/scripts"}). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-env", MountPath: "/slurm/env"}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + WithEnvVar(corev1.EnvVar{Name: "JOB_CONTAINER_INDEX", Value: "0"}). + Obj()). + WithVolume(corev1.Volume{ + Name: "slurm-scripts", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "profile-slurm"}, + Items: []corev1.KeyToPath{ + {Key: "init-entrypoint.sh", Path: "init-entrypoint.sh"}, + {Key: "entrypoint.sh", Path: "entrypoint.sh"}, + {Key: "script", Path: "script", Mode: ptr.To[int32](0755)}, + }, + }, + }, + }). + WithVolume(corev1.Volume{ + Name: "slurm-env", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }). + Obj(), + wantChildObjs: []runtime.Object{ + wrappers.MakeConfigMap("", metav1.NamespaceDefault). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Data(map[string]string{ + "script": "#!/bin/bash\nsleep 300'", + "init-entrypoint.sh": `#!/bin/sh + +set -o errexit +set -o nounset +set -o pipefail +set -x + +# External variables +# JOB_COMPLETION_INDEX - completion index of the job. +# POD_IP - current pod IP + +array_indexes="1;2;3;4;5" +container_indexes=$(echo "$array_indexes" | awk -F';' -v idx="$JOB_COMPLETION_INDEX" '{print $((idx + 1))}') + +for i in $(seq 0 1) +do + container_index=$(echo "$container_indexes" | awk -F',' -v idx="$i" '{print $((idx + 1))}') + + if [ -z "$container_index" ]; then + break + fi + + mkdir -p /slurm/env/$i + + + cat << EOF > /slurm/env/$i/slurm.env +SLURM_ARRAY_JOB_ID=1 +SLURM_ARRAY_TASK_COUNT=5 +SLURM_ARRAY_TASK_MAX=5 +SLURM_ARRAY_TASK_MIN=1 +SLURM_TASKS_PER_NODE=1 +SLURM_CPUS_PER_TASK= +SLURM_CPUS_ON_NODE= +SLURM_JOB_CPUS_PER_NODE= +SLURM_CPUS_PER_GPU= +SLURM_MEM_PER_CPU= +SLURM_MEM_PER_GPU= +SLURM_MEM_PER_NODE= +SLURM_GPUS= +SLURM_NTASKS=1 +SLURM_NTASKS_PER_NODE=1 +SLURM_NPROCS=1 +SLURM_NNODES=2 +SLURM_SUBMIT_DIR=/slurm/scripts +SLURM_SUBMIT_HOST=$HOSTNAME +SLURM_JOB_NODELIST=profile-slurm-0.profile-slurm,profile-slurm-1.profile-slurm +SLURM_JOB_FIRST_NODE=profile-slurm-0.profile-slurm +SLURM_JOB_ID=$(expr $JOB_COMPLETION_INDEX \* 1 + $i + 1) +SLURM_JOBID=$(expr $JOB_COMPLETION_INDEX \* 1 + $i + 1) +SLURM_ARRAY_TASK_ID=$container_index +SLURM_JOB_FIRST_NODE_IP=${SLURM_JOB_FIRST_NODE_IP:-""} +EOF + +done +`, + "entrypoint.sh": `#!/usr/local/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +# External variables +# JOB_CONTAINER_INDEX - container index in the container template. + +if [ ! -d "/slurm/env/$JOB_CONTAINER_INDEX" ]; then + exit 0 +fi + +SBATCH_JOB_NAME= + +export $(cat /slurm/env/$JOB_CONTAINER_INDEX/slurm.env | xargs) + +/slurm/scripts/script +`, + }). + Obj(), + wrappers.MakeService("profile-slurm", metav1.NamespaceDefault). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + ClusterIP("None"). + Selector("job-name", "profile-slurm"). + Obj(), + }, + cmpopts: []cmp.Option{ + cmpopts.AcyclicTransformer("RemoveGeneratedNameSuffixInString", func(val string) string { + return regexp.MustCompile("(profile-slurm)(-.{5})").ReplaceAllString(val, "$1") + }), + cmpopts.AcyclicTransformer("RemoveGeneratedNameSuffixInMap", func(m map[string]string) map[string]string { + for key, val := range m { + m[key] = regexp.MustCompile("(profile-slurm)(-.{5})").ReplaceAllString(val, "$1") + } + return m + }), + }, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + if tc.beforeTest != nil { + tc.beforeTest(t, &tc) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tcg := cmdtesting.NewTestClientGetter(). + WithKjobctlClientset(kjobctlfake.NewSimpleClientset(tc.kjobctlObjs...)) + + gotRootObj, gotChildObjs, gotErr := NewBuilder(tcg, testStartTime). + WithNamespace(tc.namespace). + WithProfileName(tc.profile). + WithModeName(tc.mode). + WithScript(tc.tempFile). + WithArray(tc.array). + WithCpusPerTask(tc.cpusPerTask). + WithGpusPerTask(tc.gpusPerTask). + WithMemPerTask(tc.memPerTask). + WithMemPerCPU(tc.memPerCPU). + WithMemPerGPU(tc.memPerGPU). + WithNodes(tc.nodes). + WithNTasks(tc.nTasks). + WithOutput(tc.output). + WithError(tc.err). + WithInput(tc.input). + WithJobName(tc.jobName). + WithPartition(tc.partition). + WithInitImage(tc.initImage). + WithFirstNodeIPTimeout(tc.firstNodeTimeout). + Do(ctx) + + var opts []cmp.Option + var statusError *apierrors.StatusError + if !errors.As(tc.wantErr, &statusError) { + opts = append(opts, cmpopts.EquateErrors()) + } + if diff := cmp.Diff(tc.wantErr, gotErr, opts...); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + return + } + + defaultCmpOpts := []cmp.Option{cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name")} + opts = append(defaultCmpOpts, tc.cmpopts...) + + if job, ok := tc.wantRootObj.(*batchv1.Job); ok { + if job.Annotations == nil { + job.Annotations = make(map[string]string, 1) + } + job.Annotations[constants.ScriptAnnotation] = tc.tempFile + } + + if diff := cmp.Diff(tc.wantRootObj, gotRootObj, opts...); diff != "" { + t.Errorf("Root object after build (-want,+got):\n%s", diff) + } + + if diff := cmp.Diff(tc.wantChildObjs, gotChildObjs, opts...); diff != "" { + t.Errorf("Child objects after build (-want,+got):\n%s", diff) + } + }) + } +} + +func TestSlurmBuilderBuildEntrypointCommand(t *testing.T) { + testStartTime := time.Now() + + testCases := map[string]struct { + input string + output string + error string + wantEntrypointCommand string + }{ + "should build entrypoint command": { + wantEntrypointCommand: "/slurm/scripts/script", + }, + "should build entrypoint command with output": { + output: "/home/test/stdout.out", + wantEntrypointCommand: "/slurm/scripts/script 1>/home/test/stdout.out", + }, + "should build entrypoint command with error": { + error: "/home/test/stderr.out", + wantEntrypointCommand: "/slurm/scripts/script 2>/home/test/stderr.out", + }, + "should build entrypoint command with output and error": { + output: "/home/test/stdout.out", + error: "/home/test/stderr.out", + wantEntrypointCommand: "/slurm/scripts/script 1>/home/test/stdout.out 2>/home/test/stderr.out", + }, + "should build entrypoint command with input": { + input: "/home/test/script.sh", + wantEntrypointCommand: "/slurm/scripts/script /home/test/stdout.out", + }, + "should build entrypoint command with input and error": { + input: "/home/test/script.sh", + error: "/home/test/stderr.out", + wantEntrypointCommand: "/slurm/scripts/script /home/test/stderr.out", + }, + "should build entrypoint command with input, output and error": { + input: "/home/test/script.sh", + output: "/home/test/stdout.out", + error: "/home/test/stderr.out", + wantEntrypointCommand: "/slurm/scripts/script /home/test/stdout.out 2>/home/test/stderr.out", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + tcg := cmdtesting.NewTestClientGetter() + + newBuilder := NewBuilder(tcg, testStartTime). + WithInput(tc.input). + WithOutput(tc.output). + WithError(tc.error) + gotEntrypointCommand := newSlurmBuilder(newBuilder).buildEntrypointCommand() + if diff := cmp.Diff(tc.wantEntrypointCommand, gotEntrypointCommand); diff != "" { + t.Errorf("Unexpected entrypoint command (-want/+got)\n%s", diff) + return + } + }) + } +} diff --git a/pkg/builder/templates/slurm_entrypoint_script.sh.tmpl b/pkg/builder/templates/slurm_entrypoint_script.sh.tmpl new file mode 100644 index 0000000..b4a47f7 --- /dev/null +++ b/pkg/builder/templates/slurm_entrypoint_script.sh.tmpl @@ -0,0 +1,20 @@ +#!/usr/local/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +# External variables +# JOB_CONTAINER_INDEX - container index in the container template. + +if [ ! -d "{{.EnvsPath}}/$JOB_CONTAINER_INDEX" ]; then + exit 0 +fi + +SBATCH_JOB_NAME={{.SbatchJobName}} + +export $(cat {{.EnvsPath}}/$JOB_CONTAINER_INDEX/{{.SlurmEnvFilename}} | xargs) + +{{- if .ChangeDir }}cd {{.ChangeDir}}{{end}} + +{{.BuildEntrypointCommand}} diff --git a/pkg/builder/templates/slurm_init_entrypoint_script.sh.tmpl b/pkg/builder/templates/slurm_init_entrypoint_script.sh.tmpl new file mode 100644 index 0000000..12c2be4 --- /dev/null +++ b/pkg/builder/templates/slurm_init_entrypoint_script.sh.tmpl @@ -0,0 +1,77 @@ +#!/bin/sh + +set -o errexit +set -o nounset +set -o pipefail +set -x + +# External variables +# JOB_COMPLETION_INDEX - completion index of the job. +# POD_IP - current pod IP + +array_indexes="{{.ArrayIndexes}}" +container_indexes=$(echo "$array_indexes" | awk -F';' -v idx="$JOB_COMPLETION_INDEX" '{print $((idx + 1))}') + +for i in $(seq 0 {{.SlurmNTasksPerNode}}) +do + container_index=$(echo "$container_indexes" | awk -F',' -v idx="$i" '{print $((idx + 1))}') + + if [ -z "$container_index" ]; then + break + fi + + mkdir -p {{.EnvsPath}}/$i + +{{if .FirstNodeIP}} + if [[ "$JOB_COMPLETION_INDEX" -eq 0 ]]; then + SLURM_JOB_FIRST_NODE_IP=${POD_IP} + else + timeout={{.FirstNodeIPTimeoutSeconds}} + start_time=$(date +%s) + while true; do + ip=$(nslookup "{{.SlurmJobFirstNode}}" | grep "Address 1" | awk 'NR==2 {print $3}') || true + if [[ -n "$ip" ]]; then + SLURM_JOB_FIRST_NODE_IP=$ip + break + else + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + if [ "$elapsed_time" -ge "$timeout" ]; then + echo "Timeout reached, IP address for the first node ({{.SlurmJobFirstNode}}) not found." + break + fi + echo "IP Address for the first node ({{.SlurmJobFirstNode}}) not found, retrying..." + sleep 1 + fi + done + fi +{{end}} + cat << EOF > {{.EnvsPath}}/$i/{{.SlurmEnvFilename}} +SLURM_ARRAY_JOB_ID={{.SlurmArrayJobID}} +SLURM_ARRAY_TASK_COUNT={{.SlurmArrayTaskCount}} +SLURM_ARRAY_TASK_MAX={{.SlurmArrayTaskMax}} +SLURM_ARRAY_TASK_MIN={{.SlurmArrayTaskMin}} +SLURM_TASKS_PER_NODE={{.SlurmTasksPerNode}} +SLURM_CPUS_PER_TASK={{.SlurmCPUsPerTask}} +SLURM_CPUS_ON_NODE={{.SlurmCPUsOnNode}} +SLURM_JOB_CPUS_PER_NODE={{.SlurmJobCPUsPerNode}} +SLURM_CPUS_PER_GPU={{.SlurmCPUsPerGPU}} +SLURM_MEM_PER_CPU={{.SlurmMemPerCPU}} +SLURM_MEM_PER_GPU={{.SlurmMemPerGPU}} +SLURM_MEM_PER_NODE={{.SlurmMemPerNode}} +SLURM_GPUS={{.SlurmGPUs}} +SLURM_NTASKS={{.SlurmNTasks}} +SLURM_NTASKS_PER_NODE={{.SlurmNTasksPerNode}} +SLURM_NPROCS={{.SlurmNProcs}} +SLURM_NNODES={{.SlurmNNodes}} +SLURM_SUBMIT_DIR={{.SlurmSubmitDir}} +SLURM_SUBMIT_HOST=$HOSTNAME +SLURM_JOB_NODELIST={{.SlurmJobNodeList}} +SLURM_JOB_FIRST_NODE={{.SlurmJobFirstNode}} +SLURM_JOB_ID=$(expr $JOB_COMPLETION_INDEX \* {{.SlurmNTasksPerNode}} + $i + 1) +SLURM_JOBID=$(expr $JOB_COMPLETION_INDEX \* {{.SlurmNTasksPerNode}} + $i + 1) +SLURM_ARRAY_TASK_ID=$container_index +SLURM_JOB_FIRST_NODE_IP=${SLURM_JOB_FIRST_NODE_IP:-""} +EOF + +done diff --git a/pkg/cmd/cmd.go b/pkg/cmd/cmd.go new file mode 100644 index 0000000..c16e429 --- /dev/null +++ b/pkg/cmd/cmd.go @@ -0,0 +1,87 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "os" + + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/utils/clock" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/create" + deletecmd "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/delete" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/describe" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/list" + crds "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/printcrds" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" +) + +type KjobctlOptions struct { + Clock clock.Clock + ConfigFlags *genericclioptions.ConfigFlags + + genericiooptions.IOStreams +} + +func defaultConfigFlags() *genericclioptions.ConfigFlags { + return genericclioptions.NewConfigFlags(true).WithDiscoveryQPS(50.0) +} + +func NewDefaultKjobctlCmd() *cobra.Command { + ioStreams := genericiooptions.IOStreams{In: os.Stdin, Out: os.Stdout, ErrOut: os.Stderr} + return NewKjobctlCmd(KjobctlOptions{ + ConfigFlags: defaultConfigFlags().WithWarningPrinter(ioStreams), + IOStreams: ioStreams, + }) +} + +func NewKjobctlCmd(o KjobctlOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "kjobctl", + Short: "ML/AI/Batch Jobs Made Easy", + } + + if o.Clock == nil { + o.Clock = clock.RealClock{} + } + + flags := cmd.PersistentFlags() + + configFlags := o.ConfigFlags + if configFlags == nil { + configFlags = defaultConfigFlags().WithWarningPrinter(o.IOStreams) + } + configFlags.AddFlags(flags) + + clientGetter := util.NewClientGetter(configFlags) + + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("namespace", completion.NamespaceNameFunc(clientGetter))) + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("context", completion.ContextsFunc(clientGetter))) + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("cluster", completion.ClustersFunc(clientGetter))) + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("user", completion.UsersFunc(clientGetter))) + + cmd.AddCommand(create.NewCreateCmd(clientGetter, o.IOStreams, o.Clock)) + cmd.AddCommand(describe.NewDescribeCmd(clientGetter, o.IOStreams)) + cmd.AddCommand(list.NewListCmd(clientGetter, o.IOStreams, o.Clock)) + cmd.AddCommand(deletecmd.NewDeleteCmd(clientGetter, o.IOStreams)) + cmd.AddCommand(crds.NewCmd()) + + return cmd +} diff --git a/pkg/cmd/completion/completion.go b/pkg/cmd/completion/completion.go new file mode 100644 index 0000000..91feaa3 --- /dev/null +++ b/pkg/cmd/completion/completion.go @@ -0,0 +1,276 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package completion + +import ( + "fmt" + "slices" + "strings" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +const completionLimit = 100 + +func NamespaceNameFunc(clientGetter util.ClientGetter) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + clientSet, err := clientGetter.K8sClientset() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + list, err := clientSet.CoreV1().Namespaces().List(cmd.Context(), metav1.ListOptions{Limit: completionLimit}) + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + var validArgs []string + for _, ns := range list.Items { + validArgs = append(validArgs, ns.Name) + } + + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} + +func ContextsFunc(clientGetter util.ClientGetter) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + config, err := clientGetter.ToRawKubeConfigLoader().RawConfig() + if err != nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + var validArgs []string + for name := range config.Contexts { + if strings.HasPrefix(name, toComplete) { + validArgs = append(validArgs, name) + } + } + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} + +func ClustersFunc(clientGetter util.ClientGetter) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + config, err := clientGetter.ToRawKubeConfigLoader().RawConfig() + if err != nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + var validArgs []string + for name := range config.Clusters { + if strings.HasPrefix(name, toComplete) { + validArgs = append(validArgs, name) + } + } + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} + +func UsersFunc(clientGetter util.ClientGetter) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + config, err := clientGetter.ToRawKubeConfigLoader().RawConfig() + if err != nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + var validArgs []string + for name := range config.AuthInfos { + if strings.HasPrefix(name, toComplete) { + validArgs = append(validArgs, name) + } + } + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} + +func ApplicationProfileNameFunc(clientGetter util.ClientGetter) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + clientSet, err := clientGetter.KjobctlClientset() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + namespace, _, err := clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + list, err := clientSet.KjobctlV1alpha1().ApplicationProfiles(namespace).List(cmd.Context(), metav1.ListOptions{Limit: completionLimit}) + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + var validArgs []string + for _, ap := range list.Items { + if !slices.Contains(args, ap.Name) { + validArgs = append(validArgs, ap.Name) + } + } + + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} + +func LocalQueueNameFunc(clientGetter util.ClientGetter) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + clientSet, err := clientGetter.KueueClientset() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + namespace, _, err := clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + list, err := clientSet.KueueV1beta1().LocalQueues(namespace).List(cmd.Context(), metav1.ListOptions{Limit: completionLimit}) + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + var validArgs []string + for _, lq := range list.Items { + if !slices.Contains(args, lq.Name) { + validArgs = append(validArgs, lq.Name) + } + } + + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} + +func JobNameFunc(clientGetter util.ClientGetter, mode v1alpha1.ApplicationProfileMode) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + clientset, err := clientGetter.K8sClientset() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + namespace, _, err := clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + opts := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s,%s=%s", constants.ProfileLabel, constants.ModeLabel, mode), + Limit: completionLimit, + } + list, err := clientset.BatchV1().Jobs(namespace).List(cmd.Context(), opts) + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + var validArgs []string + for _, job := range list.Items { + if !slices.Contains(args, job.Name) { + validArgs = append(validArgs, job.Name) + } + } + + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} + +func RayJobNameFunc(clientGetter util.ClientGetter) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + clientset, err := clientGetter.RayClientset() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + namespace, _, err := clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + opts := metav1.ListOptions{LabelSelector: constants.ProfileLabel, Limit: completionLimit} + list, err := clientset.RayV1().RayJobs(namespace).List(cmd.Context(), opts) + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + var validArgs []string + for _, rayJob := range list.Items { + if !slices.Contains(args, rayJob.Name) { + validArgs = append(validArgs, rayJob.Name) + } + } + + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} + +func RayClusterNameFunc(clientGetter util.ClientGetter) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + clientset, err := clientGetter.RayClientset() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + namespace, _, err := clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + opts := metav1.ListOptions{LabelSelector: constants.ProfileLabel, Limit: completionLimit} + list, err := clientset.RayV1().RayClusters(namespace).List(cmd.Context(), opts) + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + var validArgs []string + for _, rayCluster := range list.Items { + if !slices.Contains(args, rayCluster.Name) { + validArgs = append(validArgs, rayCluster.Name) + } + } + + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} + +func PodNameFunc(clientGetter util.ClientGetter) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + clientset, err := clientGetter.K8sClientset() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + namespace, _, err := clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + opts := metav1.ListOptions{LabelSelector: constants.ProfileLabel, Limit: completionLimit} + list, err := clientset.CoreV1().Pods(namespace).List(cmd.Context(), opts) + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + var validArgs []string + for _, pod := range list.Items { + if !slices.Contains(args, pod.Name) { + validArgs = append(validArgs, pod.Name) + } + } + + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} diff --git a/pkg/cmd/completion/completion_test.go b/pkg/cmd/completion/completion_test.go new file mode 100644 index 0000000..5251923 --- /dev/null +++ b/pkg/cmd/completion/completion_test.go @@ -0,0 +1,216 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package completion + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + rayfake "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/fake" + "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + k8sfake "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueuefake "sigs.k8s.io/kueue/client-go/clientset/versioned/fake" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/fake" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestNamespaceNameFunc(t *testing.T) { + objs := []runtime.Object{ + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns1"}}, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns2"}}, + } + + wantNames := []string{"ns1", "ns2"} + wantDirective := cobra.ShellCompDirectiveNoFileComp + + tcg := cmdtesting.NewTestClientGetter() + tcg.WithK8sClientset(k8sfake.NewSimpleClientset(objs...)) + + complFn := NamespaceNameFunc(tcg) + names, directive := complFn(&cobra.Command{}, []string{}, "") + if diff := cmp.Diff(wantNames, names); diff != "" { + t.Errorf("Unexpected names (-want/+got)\n%s", diff) + } + + if diff := cmp.Diff(wantDirective, directive); diff != "" { + t.Errorf("Unexpected directive (-want/+got)\n%s", diff) + } +} + +func TestApplicationProfileNameFunc(t *testing.T) { + args := []string{"ap1"} + objs := []runtime.Object{ + wrappers.MakeApplicationProfile("ap1", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("ap2", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("ap3", "test").Obj(), + } + + wantNames := []string{"ap2"} + wantDirective := cobra.ShellCompDirectiveNoFileComp + + tcg := cmdtesting.NewTestClientGetter() + tcg.WithKjobctlClientset(fake.NewSimpleClientset(objs...)) + + complFn := ApplicationProfileNameFunc(tcg) + names, directive := complFn(&cobra.Command{}, args, "") + if diff := cmp.Diff(wantNames, names); diff != "" { + t.Errorf("Unexpected names (-want/+got)\n%s", diff) + } + + if diff := cmp.Diff(wantDirective, directive); diff != "" { + t.Errorf("Unexpected directive (-want/+got)\n%s", diff) + } +} + +func TestLocalQueueNameFunc(t *testing.T) { + args := []string{"lq1"} + objs := []runtime.Object{ + &v1beta1.LocalQueue{ObjectMeta: metav1.ObjectMeta{Name: "lq1", Namespace: metav1.NamespaceDefault}}, + &v1beta1.LocalQueue{ObjectMeta: metav1.ObjectMeta{Name: "lq2", Namespace: metav1.NamespaceDefault}}, + &v1beta1.LocalQueue{ObjectMeta: metav1.ObjectMeta{Name: "lq3", Namespace: "test"}}, + } + + wantNames := []string{"lq2"} + wantDirective := cobra.ShellCompDirectiveNoFileComp + + tcg := cmdtesting.NewTestClientGetter() + tcg.WithKueueClientset(kueuefake.NewSimpleClientset(objs...)) + + complFn := LocalQueueNameFunc(tcg) + names, directive := complFn(&cobra.Command{}, args, "") + if diff := cmp.Diff(wantNames, names); diff != "" { + t.Errorf("Unexpected names (-want/+got)\n%s", diff) + } + + if diff := cmp.Diff(wantDirective, directive); diff != "" { + t.Errorf("Unexpected directive (-want/+got)\n%s", diff) + } +} + +func TestJobNameCompletionFunc(t *testing.T) { + args := []string{"job1"} + objs := []runtime.Object{ + wrappers.MakeJob("job1", metav1.NamespaceDefault).Profile("p1").Mode("jJb").Obj(), + wrappers.MakeJob("job2", metav1.NamespaceDefault).Profile("p1").Mode("Job").Obj(), + wrappers.MakeJob("job3", metav1.NamespaceDefault).Profile("p1").Mode("Slurm").Obj(), + wrappers.MakeJob("job4", "test").Profile("p1").Mode("Job").Obj(), + wrappers.MakeJob("job5", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeJob("job6", metav1.NamespaceDefault).Mode("Job").Obj(), + } + mode := v1alpha1.JobMode + + wantNames := []string{"job2"} + wantDirective := cobra.ShellCompDirectiveNoFileComp + + tcg := cmdtesting.NewTestClientGetter() + tcg.WithK8sClientset(k8sfake.NewSimpleClientset(objs...)) + + complFn := JobNameFunc(tcg, mode) + names, directive := complFn(&cobra.Command{}, args, "") + if diff := cmp.Diff(wantNames, names); diff != "" { + t.Errorf("Unexpected names (-want/+got)\n%s", diff) + } + + if diff := cmp.Diff(wantDirective, directive); diff != "" { + t.Errorf("Unexpected directive (-want/+got)\n%s", diff) + } +} + +func TestRayJobNameCompletionFunc(t *testing.T) { + args := []string{"ray-job1"} + objs := []runtime.Object{ + wrappers.MakeRayJob("ray-job1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayJob("ray-job2", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayJob("ray-job3", "test").Profile("p1").Obj(), + wrappers.MakeRayJob("ray-job4", metav1.NamespaceDefault).Obj(), + } + + wantNames := []string{"ray-job2"} + wantDirective := cobra.ShellCompDirectiveNoFileComp + + tcg := cmdtesting.NewTestClientGetter() + tcg.WithRayClientset(rayfake.NewSimpleClientset(objs...)) + + complFn := RayJobNameFunc(tcg) + names, directive := complFn(&cobra.Command{}, args, "") + if diff := cmp.Diff(wantNames, names); diff != "" { + t.Errorf("Unexpected names (-want/+got)\n%s", diff) + } + + if diff := cmp.Diff(wantDirective, directive); diff != "" { + t.Errorf("Unexpected directive (-want/+got)\n%s", diff) + } +} + +func TestRayClusterNameCompletionFunc(t *testing.T) { + args := []string{"ray-cluster1"} + objs := []runtime.Object{ + wrappers.MakeRayCluster("ray-cluster1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayCluster("ray-cluster2", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayCluster("ray-cluster3", "test").Profile("p1").Obj(), + wrappers.MakeRayCluster("ray-cluster4", metav1.NamespaceDefault).Obj(), + } + + wantNames := []string{"ray-cluster2"} + wantDirective := cobra.ShellCompDirectiveNoFileComp + + tcg := cmdtesting.NewTestClientGetter() + tcg.WithRayClientset(rayfake.NewSimpleClientset(objs...)) + + complFn := RayClusterNameFunc(tcg) + names, directive := complFn(&cobra.Command{}, args, "") + if diff := cmp.Diff(wantNames, names); diff != "" { + t.Errorf("Unexpected names (-want/+got)\n%s", diff) + } + + if diff := cmp.Diff(wantDirective, directive); diff != "" { + t.Errorf("Unexpected directive (-want/+got)\n%s", diff) + } +} + +func TestPodNameCompletionFunc(t *testing.T) { + args := []string{"pod1"} + objs := []runtime.Object{ + wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakePod("pod3", "test").Profile("p1").Obj(), + wrappers.MakePod("pod4", metav1.NamespaceDefault).Obj(), + } + + wantNames := []string{"pod2"} + wantDirective := cobra.ShellCompDirectiveNoFileComp + + tcg := cmdtesting.NewTestClientGetter() + tcg.WithK8sClientset(k8sfake.NewSimpleClientset(objs...)) + + complFn := PodNameFunc(tcg) + names, directive := complFn(&cobra.Command{}, args, "") + if diff := cmp.Diff(wantNames, names); diff != "" { + t.Errorf("Unexpected names (-want/+got)\n%s", diff) + } + + if diff := cmp.Diff(wantDirective, directive); diff != "" { + t.Errorf("Unexpected directive (-want/+got)\n%s", diff) + } +} diff --git a/pkg/cmd/create/create.go b/pkg/cmd/create/create.go new file mode 100644 index 0000000..7d55670 --- /dev/null +++ b/pkg/cmd/create/create.go @@ -0,0 +1,861 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package create + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + corev1 "k8s.io/api/core/v1" + apiresource "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + k8sscheme "k8s.io/client-go/kubernetes/scheme" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/kubectl/pkg/cmd/attach" + "k8s.io/kubectl/pkg/cmd/exec" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/clock" + "k8s.io/utils/ptr" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/builder" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/parser" +) + +const ( + profileFlagName = "profile" + podRunningTimeoutFlagName = "pod-running-timeout" + removeFlagName = "rm" + ignoreUnknownFlagName = "ignore-unknown-flags" + initImageFlagName = "init-image" + skipLocalQueueValidationFlagName = "skip-localqueue-validation" + skipPriorityValidationFlagName = "skip-priority-validation" + changeDirFlagName = "chdir" + firstNodeIPFlagName = "first-node-ip" + firstNodeIPTimeoutFlagName = "first-node-ip-timeout" + + commandFlagName = string(v1alpha1.CmdFlag) + parallelismFlagName = string(v1alpha1.ParallelismFlag) + completionsFlagName = string(v1alpha1.CompletionsFlag) + replicasFlagName = string(v1alpha1.ReplicasFlag) + minReplicasFlagName = string(v1alpha1.MinReplicasFlag) + maxReplicasFlagName = string(v1alpha1.MaxReplicasFlag) + requestFlagName = string(v1alpha1.RequestFlag) + localQueueFlagName = string(v1alpha1.LocalQueueFlag) + rayClusterFlagName = string(v1alpha1.RayClusterFlag) + arrayFlagName = string(v1alpha1.ArrayFlag) + cpusPerTaskFlagName = string(v1alpha1.CpusPerTaskFlag) + gpusPerTaskFlagName = string(v1alpha1.GpusPerTaskFlag) + memPerNodeFlagName = string(v1alpha1.MemPerNodeFlag) + memPerTaskFlagName = string(v1alpha1.MemPerTaskFlag) + memPerCPUFlagName = string(v1alpha1.MemPerCPUFlag) + memPerGPUFlagName = string(v1alpha1.MemPerGPUFlag) + nodesFlagName = string(v1alpha1.NodesFlag) + nTasksFlagName = string(v1alpha1.NTasksFlag) + outputFlagName = string(v1alpha1.OutputFlag) + errorFlagName = string(v1alpha1.ErrorFlag) + inputFlagName = string(v1alpha1.InputFlag) + jobNameFlagName = string(v1alpha1.JobNameFlag) + partitionFlagName = string(v1alpha1.PartitionFlag) + priorityFlagName = string(v1alpha1.PriorityFlag) + timeFlagName = string(v1alpha1.TimeFlag) +) + +func withTimeFlag(f *pflag.FlagSet, p *string) { + f.StringVarP(p, timeFlagName, "t", "", + `Set a limit on the total run time of the job. +A time limit of zero requests that no time limit be imposed. +Acceptable time formats include "minutes", "minutes:seconds", +"hours:minutes:seconds", "days-hours", "days-hours:minutes" +and "days-hours:minutes:seconds".`) +} + +var ( + createJobExample = templates.Examples(` + # Create job + kjobctl create job \ + --profile my-application-profile \ + --cmd "sleep 5" \ + --parallelism 4 \ + --completions 4 \ + --request cpu=500m,memory=4Gi \ + --localqueue my-local-queue-name + `) + createInteractiveExample = templates.Examples(` + # Create interactive + kjobctl create interactive \ + --profile my-application-profile \ + --pod-running-timeout 30s \ + --rm + `) + createRayJobLong = templates.LongDesc(` + Create a rayjob. + + KubeRay operator is required for RayJob. + How to install KubeRay operator you can find here https://ray-project.github.io/kuberay/deploy/installation/. + `) + createRayJobExample = templates.Examples(` + # Create rayjob + kjobctl create rayjob \ + --profile my-application-profile \ + --cmd "python /home/ray/samples/sample_code.py" \ + --replicas small-group=1 \ + --min-replicas small-group=1 \ + --max-replicas small-group=5 \ + --localqueue my-local-queue-name + `) + createRayClusterLong = templates.LongDesc(` + Create a raycluster. + + KubeRay operator is required for RayCluster. + How to install KubeRay operator you can find here https://ray-project.github.io/kuberay/deploy/installation/. + `) + createRayClusterExample = templates.Examples(` + # Create raycluster + kjobctl create raycluster \ + --profile my-application-profile \ + --replicas small-group=1 \ + --min-replicas small-group=1 \ + --max-replicas small-group=5 \ + --localqueue my-local-queue-name + `) + createSlurmExample = templates.Examples(` + # Create slurm + kjobctl create slurm --profile my-application-profile -- \ + --array 0-5 --nodes 3 --ntasks 1 ./script.sh + `) +) + +var ( + podRunningTimeoutDefault = 1 * time.Minute +) + +type CreateOptions struct { + exec.StreamOptions + + PrintFlags *genericclioptions.PrintFlags + Config *restclient.Config + Attach attach.RemoteAttach + AttachFunc func(*CreateOptions, *corev1.Container, remotecommand.TerminalSizeQueue, *corev1.Pod) func() error + + DryRunStrategy util.DryRunStrategy + + Namespace string + ProfileName string + ModeName v1alpha1.ApplicationProfileMode + Script string + InitImage string + PodRunningTimeout time.Duration + FirstNodeIPTimeout time.Duration + FirstNodeIP bool + RemoveInteractivePod bool + ChangeDir string + + SlurmFlagSet *pflag.FlagSet + + Command []string + Parallelism *int32 + Completions *int32 + Replicas map[string]int + MinReplicas map[string]int + MaxReplicas map[string]int + Requests corev1.ResourceList + LocalQueue string + RayCluster string + Array string + CpusPerTask *apiresource.Quantity + GpusPerTask map[string]*apiresource.Quantity + MemPerNode *apiresource.Quantity + MemPerTask *apiresource.Quantity + MemPerCPU *apiresource.Quantity + MemPerGPU *apiresource.Quantity + Nodes *int32 + NTasks *int32 + Output string + Error string + Input string + JobName string + Partition string + Priority string + TimeLimit string + IgnoreUnknown bool + SkipLocalQueueValidation bool + SkipPriorityValidation bool + + UserSpecifiedCommand string + UserSpecifiedParallelism int32 + UserSpecifiedCompletions int32 + UserSpecifiedRequest map[string]string + UserSpecifiedCpusPerTask string + UserSpecifiedGpusPerTask string + UserSpecifiedMemPerNode string + UserSpecifiedMemPerTask string + UserSpecifiedMemPerCPU string + UserSpecifiedMemPerGPU string + UserSpecifiedNodes int32 + UserSpecifiedNTasks int32 + + PrintObj printers.ResourcePrinterFunc + + genericiooptions.IOStreams +} + +func NewCreateOptions(streams genericiooptions.IOStreams) *CreateOptions { + scheme := runtime.NewScheme() + utilruntime.Must(k8sscheme.AddToScheme(scheme)) + utilruntime.Must(rayv1.AddToScheme(scheme)) + + return &CreateOptions{ + PrintFlags: genericclioptions.NewPrintFlags("created").WithTypeSetter(scheme), + IOStreams: streams, + StreamOptions: exec.StreamOptions{ + IOStreams: streams, + }, + Attach: &attach.DefaultRemoteAttach{}, + AttachFunc: defaultAttachFunc, + } +} + +type modeSubcommand struct { + ModeName v1alpha1.ApplicationProfileMode + Setup func(clientGetter util.ClientGetter, subcmd *cobra.Command, o *CreateOptions) +} + +var createModeSubcommands = map[string]modeSubcommand{ + "job": { + ModeName: v1alpha1.JobMode, + Setup: func(clientGetter util.ClientGetter, subcmd *cobra.Command, o *CreateOptions) { + subcmd.Use += " [--cmd COMMAND]" + + " [--request RESOURCE_NAME=QUANTITY]" + + " [--parallelism PARALLELISM]" + + " [--completions COMPLETIONS]" + + " [--time TIME_LIMIT]" + subcmd.Short = "Create a job" + subcmd.Example = createJobExample + + subcmd.Flags().StringVar(&o.UserSpecifiedCommand, commandFlagName, "", + "Command which is associated with the resource.") + subcmd.Flags().StringToStringVar(&o.UserSpecifiedRequest, requestFlagName, nil, + "Request is a set of (resource name, quantity) pairs.") + subcmd.Flags().Int32Var(&o.UserSpecifiedParallelism, parallelismFlagName, 0, + "Parallelism specifies the maximum desired number of pods the job should run at any given time.") + subcmd.Flags().Int32Var(&o.UserSpecifiedCompletions, completionsFlagName, 0, + "Completions specifies the desired number of successfully finished pods.") + + withTimeFlag(subcmd.Flags(), &o.TimeLimit) + }, + }, + "interactive": { + ModeName: v1alpha1.InteractiveMode, + Setup: func(clientGetter util.ClientGetter, subcmd *cobra.Command, o *CreateOptions) { + subcmd.Use += " [--cmd COMMAND]" + + " [--request RESOURCE_NAME=QUANTITY]" + + " [--pod-running-timeout DURATION]" + + " [--time TIME_LIMIT]" + + " [--rm]" + subcmd.Short = "Create an interactive shell" + subcmd.Example = createInteractiveExample + + subcmd.Flags().StringVar(&o.UserSpecifiedCommand, commandFlagName, "", + "Command which is associated with the resource.") + subcmd.Flags().StringToStringVar(&o.UserSpecifiedRequest, requestFlagName, nil, + "Request is a set of (resource name, quantity) pairs.") + subcmd.Flags().DurationVar(&o.PodRunningTimeout, podRunningTimeoutFlagName, podRunningTimeoutDefault, + "The length of time (like 5s, 2m, or 3h, higher than zero) to wait until at least one pod is running.") + subcmd.Flags().BoolVar(&o.RemoveInteractivePod, removeFlagName, false, + "Remove pod when interactive session exits.") + + withTimeFlag(subcmd.Flags(), &o.TimeLimit) + }, + }, + "rayjob": { + ModeName: v1alpha1.RayJobMode, + Setup: func(clientGetter util.ClientGetter, subcmd *cobra.Command, o *CreateOptions) { + subcmd.Use += " [--cmd COMMAND]" + + " [--replicas [WORKER_GROUP]=REPLICAS]" + + " [--min-replicas [WORKER_GROUP]=MIN_REPLICAS]" + + " [--max-replicas [WORKER_GROUP]=MAX_REPLICAS]" + + " [--time TIME_LIMIT]" + subcmd.Short = "Create a rayjob" + subcmd.Long = createRayJobLong + subcmd.Example = createRayJobExample + + subcmd.Flags().StringVar(&o.UserSpecifiedCommand, commandFlagName, "", + "Command which is associated with the resource.") + subcmd.Flags().StringToIntVar(&o.Replicas, replicasFlagName, nil, + "Replicas is the number of desired Pods for this worker group.") + subcmd.Flags().StringToIntVar(&o.MinReplicas, minReplicasFlagName, nil, + "MinReplicas denotes the minimum number of desired Pods for this worker group.") + subcmd.Flags().StringToIntVar(&o.MaxReplicas, maxReplicasFlagName, nil, + "MaxReplicas denotes the maximum number of desired Pods for this worker group, and the default value is maxInt32.") + subcmd.Flags().StringVar(&o.RayCluster, rayClusterFlagName, "", + "Existing ray cluster on which the job will be created.") + + withTimeFlag(subcmd.Flags(), &o.TimeLimit) + + subcmd.MarkFlagsMutuallyExclusive(rayClusterFlagName, replicasFlagName) + subcmd.MarkFlagsMutuallyExclusive(rayClusterFlagName, minReplicasFlagName) + subcmd.MarkFlagsMutuallyExclusive(rayClusterFlagName, maxReplicasFlagName) + subcmd.MarkFlagsMutuallyExclusive(rayClusterFlagName, localQueueFlagName) + }, + }, + "raycluster": { + ModeName: v1alpha1.RayClusterMode, + Setup: func(clientGetter util.ClientGetter, subcmd *cobra.Command, o *CreateOptions) { + subcmd.Use += " [--replicas [WORKER_GROUP]=REPLICAS]" + + " [--min-replicas [WORKER_GROUP]=MIN_REPLICAS]" + + " [--max-replicas [WORKER_GROUP]=MAX_REPLICAS]" + + " [--time TIME_LIMIT]" + subcmd.Short = "Create a raycluster" + subcmd.Long = createRayClusterLong + subcmd.Example = createRayClusterExample + + subcmd.Flags().StringToIntVar(&o.Replicas, replicasFlagName, nil, + "Replicas is the number of desired Pods for this worker group.") + subcmd.Flags().StringToIntVar(&o.MinReplicas, minReplicasFlagName, nil, + "MinReplicas denotes the minimum number of desired Pods for this worker group.") + subcmd.Flags().StringToIntVar(&o.MaxReplicas, maxReplicasFlagName, nil, + "MaxReplicas denotes the maximum number of desired Pods for this worker group, and the default value is maxInt32.") + + withTimeFlag(subcmd.Flags(), &o.TimeLimit) + }, + }, + "slurm": { + ModeName: v1alpha1.SlurmMode, + Setup: func(clientGetter util.ClientGetter, subcmd *cobra.Command, o *CreateOptions) { + subcmd.Use += " [--ignore-unknown-flags]" + + " [--init-image IMAGE]" + + " [--first-node-ip]" + + " [--first-node-ip-timeout DURATION]" + + " -- " + + " [--array ARRAY]" + + " [--cpus-per-task QUANTITY]" + + " [--gpus-per-task QUANTITY]" + + " [--mem QUANTITY]" + + " [--mem-per-task QUANTITY]" + + " [--mem-per-cpu QUANTITY]" + + " [--mem-per-gpu QUANTITY]" + + " [--nodes COUNT]" + + " [--ntasks COUNT]" + + " [--output FILENAME_PATTERN]" + + " [--error FILENAME_PATTERN]" + + " [--input FILENAME_PATTERN]" + + " [--job-name NAME]" + + " [--partition NAME]" + + " SCRIPT" + + subcmd.Short = "Create a slurm job" + subcmd.Example = createSlurmExample + subcmd.Args = cobra.MinimumNArgs(1) + + subcmd.Flags().BoolVar(&o.IgnoreUnknown, ignoreUnknownFlagName, false, + "Ignore all the unsupported flags in the bash script.") + subcmd.Flags().StringVar(&o.InitImage, initImageFlagName, "registry.k8s.io/busybox:1.27.2", + "The image used for the init container.") + subcmd.Flags().BoolVar(&o.FirstNodeIP, firstNodeIPFlagName, false, + "Enable the retrieval of the first node's IP address.") + subcmd.Flags().DurationVar(&o.FirstNodeIPTimeout, firstNodeIPTimeoutFlagName, time.Minute, + "The timeout for the retrieval of the first node's IP address.") + + o.SlurmFlagSet = pflag.NewFlagSet("slurm", pflag.ExitOnError) + o.SlurmFlagSet.StringVarP(&o.Array, arrayFlagName, "a", "", + `Submit a job array, multiple jobs to be executed with identical parameters. +The indexes specification identifies what array index values should be used. +Multiple values may be specified using a comma separated list and/or a range of values with a "-" separator. For example, "--array=0-15" or "--array=0,6,16-32". +A maximum number of simultaneously running tasks from the job array may be specified using a "%" separator. For example "--array=0-15%4" will limit the number of simultaneously running tasks from this job array to 4. +The minimum index value is 0. The maximum index value is 2147483647.`) + o.SlurmFlagSet.StringVarP(&o.UserSpecifiedCpusPerTask, cpusPerTaskFlagName, "c", "", + "How much cpus a container inside a pod requires.") + o.SlurmFlagSet.StringVar(&o.UserSpecifiedGpusPerTask, gpusPerTaskFlagName, "", + "How much gpus a container inside a pod requires.") + o.SlurmFlagSet.StringVar(&o.UserSpecifiedMemPerNode, memPerNodeFlagName, "", + "How much memory a pod requires.") + o.SlurmFlagSet.StringVar(&o.UserSpecifiedMemPerTask, memPerTaskFlagName, "", + "How much memory a container requires.") + o.SlurmFlagSet.StringVar(&o.UserSpecifiedMemPerCPU, memPerCPUFlagName, "", + "How much memory a container requires, it multiplies the number of requested cpus per task by mem-per-cpu.") + o.SlurmFlagSet.StringVar(&o.UserSpecifiedMemPerGPU, memPerGPUFlagName, "", + "How much memory a container requires, it multiplies the number of requested gpus per task by mem-per-gpu.") + o.SlurmFlagSet.Int32VarP(&o.UserSpecifiedNodes, nodesFlagName, "N", 0, + "Number of pods to be used at a time.") + o.SlurmFlagSet.Int32VarP(&o.UserSpecifiedNTasks, nTasksFlagName, "n", 1, + "Number of identical containers inside of a pod, usually 1.") + o.SlurmFlagSet.StringVarP(&o.Output, outputFlagName, "o", "", + "Where to redirect the standard output stream of a task. If not passed it proceeds to stdout, and is available via kubectl logs.") + o.SlurmFlagSet.StringVarP(&o.Error, errorFlagName, "e", "", + "Where to redirect std error stream of a task. If not passed it proceeds to stdout, and is available via kubectl logs.") + o.SlurmFlagSet.StringVar(&o.Input, inputFlagName, "", + "What to pipe into the script.") + o.SlurmFlagSet.StringVarP(&o.JobName, jobNameFlagName, "J", "", + "What is the job name.") + o.SlurmFlagSet.StringVar(&o.Partition, partitionFlagName, "", + "Local queue name.") + o.SlurmFlagSet.StringVarP(&o.ChangeDir, changeDirFlagName, "D", "", + "Change directory before executing the script.") + + withTimeFlag(o.SlurmFlagSet, &o.TimeLimit) + }, + }, +} + +func NewCreateCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams, clock clock.Clock) *cobra.Command { + o := NewCreateOptions(streams) + + cmd := &cobra.Command{ + Use: "create", + Short: "Create a task", + Example: fmt.Sprintf("%s\n\n%s\n\n%s\n\n%s\n\n%s", + createJobExample, + createInteractiveExample, + createRayJobExample, + createRayClusterExample, + createSlurmExample, + ), + } + + for modeName, modeSubcommand := range createModeSubcommands { + subcmd := &cobra.Command{ + Use: modeName + + " --profile APPLICATION_PROFILE_NAME" + + " [--localqueue LOCAL_QUEUE_NAME]" + + " [--skip-localqueue-validation]" + + " [--priority NAME]" + + " [--skip-priority-validation]", + DisableFlagsInUseLine: true, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + + err := o.Complete(clientGetter, cmd, args) + if err != nil { + return err + } + + return o.Run(cmd.Context(), clientGetter, clock.Now()) + }, + } + + o.PrintFlags.AddFlags(subcmd) + + subcmd.Flags().StringVarP(&o.ProfileName, profileFlagName, "p", "", + "Application profile contains a template (with defaults set) for running a specific type of application.") + subcmd.Flags().StringVar(&o.LocalQueue, localQueueFlagName, "", + "Kueue localqueue name which is associated with the resource.") + subcmd.Flags().BoolVar(&o.SkipLocalQueueValidation, skipLocalQueueValidationFlagName, false, + "Skip local queue validation. Add local queue even if the queue does not exist.") + subcmd.Flags().StringVar(&o.Priority, priorityFlagName, "", + "Apply priority for the entire workload.") + subcmd.Flags().BoolVar(&o.SkipPriorityValidation, skipPriorityValidationFlagName, false, + "Skip workload priority class validation. Add priority class label even if the class does not exist.") + + modeSubcommand.Setup(clientGetter, subcmd, o) + + util.AddDryRunFlag(subcmd) + + _ = subcmd.MarkFlagRequired(profileFlagName) + + cobra.CheckErr(subcmd.RegisterFlagCompletionFunc(profileFlagName, completion.ApplicationProfileNameFunc(clientGetter))) + cobra.CheckErr(subcmd.RegisterFlagCompletionFunc(localQueueFlagName, completion.LocalQueueNameFunc(clientGetter))) + + cmd.AddCommand(subcmd) + } + + return cmd +} + +func (o *CreateOptions) Complete(clientGetter util.ClientGetter, cmd *cobra.Command, args []string) error { + currentSubcommand := createModeSubcommands[cmd.Name()] + o.ModeName = currentSubcommand.ModeName + + var err error + + if o.ModeName == v1alpha1.SlurmMode { + argsLenAtDash := cmd.ArgsLenAtDash() + + if argsLenAtDash == -1 || argsLenAtDash > 0 { + return fmt.Errorf("unknown command \"%s\" for \"%s\"", args[0], cmd.CommandPath()) + } + + if err := o.SlurmFlagSet.Parse(args[argsLenAtDash:]); err != nil { + return err + } + + slurmArgs := o.SlurmFlagSet.Args() + + if len(slurmArgs) == 0 { + return errors.New("must specify script") + } + + if len(slurmArgs) > 1 { + return errors.New("must specify only one script") + } + + o.Script = slurmArgs[0] + } + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + if o.UserSpecifiedCommand != "" { + o.Command = strings.Fields(o.UserSpecifiedCommand) + } + + if cmd.Flags().Changed(parallelismFlagName) { + o.Parallelism = ptr.To(o.UserSpecifiedParallelism) + } + + if cmd.Flags().Changed(completionsFlagName) { + o.Completions = ptr.To(o.UserSpecifiedCompletions) + } + + if len(o.UserSpecifiedRequest) > 0 { + o.Requests = make(corev1.ResourceList) + for key, value := range o.UserSpecifiedRequest { + quantity, err := apiresource.ParseQuantity(value) + if err != nil { + return err + } + o.Requests[corev1.ResourceName(key)] = quantity + } + } + + if cmd.Flags().Changed(podRunningTimeoutFlagName) && o.PodRunningTimeout <= 0 { + return errors.New("--pod-running-timeout must be higher than zero") + } + + if o.SlurmFlagSet.Changed(cpusPerTaskFlagName) { + quantity, err := apiresource.ParseQuantity(o.UserSpecifiedCpusPerTask) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", o.UserSpecifiedCpusPerTask, err) + } + o.CpusPerTask = &quantity + } + + if o.SlurmFlagSet.Changed(gpusPerTaskFlagName) { + gpusPerTask, err := parser.GpusFlag(o.UserSpecifiedGpusPerTask) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", o.UserSpecifiedGpusPerTask, err) + } + o.GpusPerTask = gpusPerTask + } + + if o.SlurmFlagSet.Changed(memPerNodeFlagName) { + quantity, err := apiresource.ParseQuantity(o.UserSpecifiedMemPerNode) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", o.UserSpecifiedMemPerNode, err) + } + o.MemPerNode = &quantity + } + + if o.SlurmFlagSet.Changed(memPerTaskFlagName) { + quantity, err := apiresource.ParseQuantity(o.UserSpecifiedMemPerTask) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", o.UserSpecifiedMemPerTask, err) + } + o.MemPerTask = &quantity + } + + if o.SlurmFlagSet.Changed(memPerCPUFlagName) { + quantity, err := apiresource.ParseQuantity(o.UserSpecifiedMemPerCPU) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", o.UserSpecifiedMemPerCPU, err) + } + o.MemPerCPU = &quantity + } + + if o.SlurmFlagSet.Changed(memPerGPUFlagName) { + quantity, err := apiresource.ParseQuantity(o.UserSpecifiedMemPerGPU) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", o.UserSpecifiedMemPerGPU, err) + } + o.MemPerGPU = &quantity + } + + if o.SlurmFlagSet.Changed(nodesFlagName) { + o.Nodes = &o.UserSpecifiedNodes + } + + if o.SlurmFlagSet.Changed(nTasksFlagName) { + if o.UserSpecifiedNTasks <= 0 { + return errors.New("--nTasks must be greater than 0") + } + + o.NTasks = &o.UserSpecifiedNTasks + } + + o.DryRunStrategy, err = util.GetDryRunStrategy(cmd) + if err != nil { + return err + } + + err = util.PrintFlagsWithDryRunStrategy(o.PrintFlags, o.DryRunStrategy) + if err != nil { + return err + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return err + } + + o.PrintObj = printer.PrintObj + + o.Config, err = clientGetter.ToRESTConfig() + if err != nil { + return err + } + + err = setKubernetesDefaults(o.Config) + if err != nil { + return err + } + + return nil +} + +func (o *CreateOptions) Run(ctx context.Context, clientGetter util.ClientGetter, runTime time.Time) error { + rootObj, childObjs, err := builder.NewBuilder(clientGetter, runTime). + WithNamespace(o.Namespace). + WithProfileName(o.ProfileName). + WithModeName(o.ModeName). + WithCommand(o.Command). + WithParallelism(o.Parallelism). + WithCompletions(o.Completions). + WithReplicas(o.Replicas). + WithMinReplicas(o.MinReplicas). + WithMaxReplicas(o.MaxReplicas). + WithRequests(o.Requests). + WithLocalQueue(o.LocalQueue). + WithRayCluster(o.RayCluster). + WithScript(o.Script). + WithArray(o.Array). + WithCpusPerTask(o.CpusPerTask). + WithGpusPerTask(o.GpusPerTask). + WithMemPerNode(o.MemPerNode). + WithMemPerTask(o.MemPerTask). + WithMemPerCPU(o.MemPerCPU). + WithMemPerGPU(o.MemPerGPU). + WithNodes(o.Nodes). + WithNTasks(o.NTasks). + WithOutput(o.Output). + WithError(o.Error). + WithInput(o.Input). + WithJobName(o.JobName). + WithPartition(o.Partition). + WithPriority(o.Priority). + WithInitImage(o.InitImage). + WithIgnoreUnknown(o.IgnoreUnknown). + WithSkipLocalQueueValidation(o.SkipLocalQueueValidation). + WithSkipPriorityValidation(o.SkipPriorityValidation). + WithChangeDir(o.ChangeDir). + WithFirstNodeIP(o.FirstNodeIP). + WithFirstNodeIPTimeout(o.FirstNodeIPTimeout). + WithTimeLimit(o.TimeLimit). + Do(ctx) + if err != nil { + return err + } + + if o.DryRunStrategy != util.DryRunClient { + rootObj, err = o.createObject(ctx, clientGetter, rootObj, nil) + if err != nil { + return err + } + } + + err = o.PrintObj(rootObj, o.Out) + if err != nil { + return err + } + + for i := range childObjs { + if o.DryRunStrategy != util.DryRunClient { + childObjs[i], err = o.createObject(ctx, clientGetter, childObjs[i], rootObj) + if err != nil { + return err + } + } + + err = o.PrintObj(childObjs[i], o.Out) + if err != nil { + return err + } + } + + if o.DryRunStrategy == util.DryRunNone && o.ModeName == v1alpha1.InteractiveMode { + pod := rootObj.(*corev1.Pod) + return o.RunInteractivePod(ctx, clientGetter, pod.Name) + } + + return nil +} + +func (o *CreateOptions) createObject(ctx context.Context, clientGetter util.ClientGetter, obj runtime.Object, owner runtime.Object) (runtime.Object, error) { + options := metav1.CreateOptions{} + if o.DryRunStrategy == util.DryRunServer { + options.DryRun = []string{metav1.DryRunAll} + } + + dynamicClient, err := clientGetter.DynamicClient() + if err != nil { + return nil, err + } + + restMapper, err := clientGetter.ToRESTMapper() + if err != nil { + return nil, err + } + + gvk := obj.GetObjectKind().GroupVersionKind() + mapping, err := restMapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return nil, err + } + gvr := mapping.Resource + + unstructuredObj := &unstructured.Unstructured{} + unstructuredObj.Object, err = runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + return nil, err + } + + if owner != nil { + unstructuredOwner := &unstructured.Unstructured{} + unstructuredOwner.Object, err = runtime.DefaultUnstructuredConverter.ToUnstructured(owner) + if err != nil { + return nil, err + } + unstructuredObj.SetOwnerReferences(append(unstructuredOwner.GetOwnerReferences(), metav1.OwnerReference{ + APIVersion: unstructuredOwner.GetAPIVersion(), + Kind: unstructuredOwner.GetKind(), + Name: unstructuredOwner.GetName(), + UID: unstructuredOwner.GetUID(), + })) + } + + unstructuredObj, err = dynamicClient.Resource(gvr).Namespace(o.Namespace).Create(ctx, unstructuredObj, options) + if err != nil { + return nil, err + } + + createdObj := obj.DeepCopyObject() + + err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredObj.UnstructuredContent(), createdObj) + if err != nil { + return nil, err + } + + return createdObj, nil +} + +func (o *CreateOptions) RunInteractivePod(ctx context.Context, clientGetter util.ClientGetter, podName string) error { + k8sclient, err := clientGetter.K8sClientset() + if err != nil { + return err + } + + if o.RemoveInteractivePod { + defer func() { + err = k8sclient.CoreV1().Pods(o.Namespace).Delete(ctx, podName, metav1.DeleteOptions{}) + if err != nil { + fmt.Fprintln(o.ErrOut, err.Error()) + } + fmt.Fprintf(o.Out, "pod \"%s\" deleted\n", podName) + }() + } + + fmt.Fprintf(o.Out, "waiting for pod \"%s\" to be running...\n", podName) + err = waitForPodRunning(ctx, k8sclient, o.Namespace, podName, o.PodRunningTimeout) + if err != nil { + return err + } + + pod, err := k8sclient.CoreV1().Pods(o.Namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return err + } + + err = attachTTY(o, pod) + if err != nil { + return err + } + + return nil +} + +func attachTTY(o *CreateOptions, pod *corev1.Pod) error { + o.Stdin = true + o.TTY = true + containerToAttach := &pod.Spec.Containers[0] + if !containerToAttach.TTY { + return fmt.Errorf("error: Unable to use a TTY - container %s did not allocate one", containerToAttach.Name) + } + + tty := o.SetupTTY() + + var sizeQueue remotecommand.TerminalSizeQueue + if tty.Raw { + // this call spawns a goroutine to monitor/update the terminal size + sizeQueue = tty.MonitorSize(tty.GetSize()) + + // unset p.Err if it was previously set because both stdout and stderr go over p.Out when tty is true + o.ErrOut = nil + } + + return tty.Safe(o.AttachFunc(o, containerToAttach, sizeQueue, pod)) +} + +func defaultAttachFunc(o *CreateOptions, containerToAttach *corev1.Container, sizeQueue remotecommand.TerminalSizeQueue, pod *corev1.Pod) func() error { + return func() error { + restClient, err := restclient.RESTClientFor(o.Config) + if err != nil { + return err + } + + req := restClient.Post(). + Resource("pods"). + Name(pod.Name). + Namespace(pod.Namespace). + SubResource("attach") + req.VersionedParams(&corev1.PodAttachOptions{ + Container: containerToAttach.Name, + Stdin: o.Stdin, + Stdout: o.Out != nil, + Stderr: o.ErrOut != nil, + TTY: o.TTY, + }, k8sscheme.ParameterCodec) + + return o.Attach.Attach(req.URL(), o.Config, o.In, o.Out, o.ErrOut, o.TTY, sizeQueue) + } +} diff --git a/pkg/cmd/create/create_test.go b/pkg/cmd/create/create_test.go new file mode 100644 index 0000000..fd93421 --- /dev/null +++ b/pkg/cmd/create/create_test.go @@ -0,0 +1,2340 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package create + +import ( + "context" + "fmt" + "io" + "net/url" + "os" + "regexp" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + utilrand "k8s.io/apimachinery/pkg/util/rand" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/client-go/dynamic/fake" + k8sfake "k8s.io/client-go/kubernetes/fake" + k8sscheme "k8s.io/client-go/kubernetes/scheme" + restclient "k8s.io/client-go/rest" + kubetesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/remotecommand" + clocktesting "k8s.io/utils/clock/testing" + "k8s.io/utils/ptr" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueuefake "sigs.k8s.io/kueue/client-go/clientset/versioned/fake" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + kjobctlfake "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/fake" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + cmdutil "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestCreateOptions_Complete(t *testing.T) { + testStartTime := time.Now() + + testCases := map[string]struct { + args []string + options *CreateOptions + wantOptions *CreateOptions + wantErr string + }{ + "invalid request": { + args: []string{"job"}, + options: &CreateOptions{ + Namespace: metav1.NamespaceDefault, + ModeName: v1alpha1.JobMode, + UserSpecifiedRequest: map[string]string{"cpu": "invalid"}, + }, + wantOptions: &CreateOptions{}, + wantErr: "quantities must match the regular expression '^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + tcg := cmdtesting.NewTestClientGetter() + + cmd := NewCreateCmd(tcg, streams, clocktesting.NewFakeClock(testStartTime)) + cmd.SetOut(out) + cmd.SetErr(outErr) + cmd.SetArgs(tc.args) + + gotErr := tc.options.Complete(tcg, cmd.Commands()[0], nil) + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + if gotErr != nil { + return + } + }) + } +} + +type createCmdTestCase struct { + beforeTest func(t *testing.T, tc *createCmdTestCase) + tempFile string + ns string + args func(tc *createCmdTestCase) []string + kjobctlObjs []runtime.Object + kueueObjs []runtime.Object + gvks []schema.GroupVersionKind + wantLists []runtime.Object + cmpopts []cmp.Option + wantOut string + wantOutPattern string + wantOutErr string + wantErr string +} + +func beforeSlurmTest(t *testing.T, tc *createCmdTestCase) { + file, err := os.CreateTemp("", "slurm") + if err != nil { + t.Fatal(err) + } + defer file.Close() + t.Cleanup(func() { + if err := os.Remove(tc.tempFile); err != nil { + t.Fatal(err) + } + }) + + if _, err := file.WriteString("#!/bin/bash\nsleep 300'"); err != nil { + t.Fatal(err) + } + + tc.tempFile = file.Name() +} + +func TestCreateCmd(t *testing.T) { + testStartTime := time.Now() + userID := os.Getenv(constants.SystemEnvVarNameUser) + + testCases := map[string]createCmdTestCase{ + "should create job": { + args: func(tc *createCmdTestCase) []string { return []string{"job", "--profile", "profile"} }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create rayjob": { + args: func(tc *createCmdTestCase) []string { return []string{"rayjob", "--profile", "profile"} }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayJobTemplate("ray-job-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayJobMode, "ray-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayJob"}}, + wantLists: []runtime.Object{ + &rayv1.RayJobList{ + TypeMeta: metav1.TypeMeta{Kind: "RayJobList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayJob{ + *wrappers.MakeRayJob("", metav1.NamespaceDefault). + GenerateName("profile-rayjob-"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "rayjob.ray.io/ created\n", + }, + "should create raycluster": { + args: func(tc *createCmdTestCase) []string { return []string{"raycluster", "--profile", "profile"} }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayClusterTemplate("ray-cluster-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayClusterMode, "ray-cluster-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayCluster"}}, + wantLists: []runtime.Object{ + &rayv1.RayClusterList{ + TypeMeta: metav1.TypeMeta{Kind: "RayClusterList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("", metav1.NamespaceDefault). + GenerateName("profile-raycluster-"). + Profile("profile"). + Mode(v1alpha1.RayClusterMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "raycluster.ray.io/ created\n", + }, + "should create job with short profile flag": { + args: func(tc *createCmdTestCase) []string { return []string{"job", "-p", "profile"} }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create job with localqueue replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"job", "--profile", "profile", "--localqueue", "lq1"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + kueueObjs: []runtime.Object{ + &kueue.LocalQueue{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "lq1", + }, + }, + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create job with localqueue and skip local queue validation": { + args: func(tc *createCmdTestCase) []string { + return []string{"job", "--profile", "profile", "--localqueue", "lq1", "--skip-localqueue-validation"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create job with parallelism replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"job", "--profile", "profile", "--parallelism", "5"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault). + Parallelism(1). + Completions(1). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Parallelism(5). + Completions(1). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create job with --priority flag": { + args: func(tc *createCmdTestCase) []string { + return []string{"job", "--profile", "profile", "--priority", "sample-priority"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + kueueObjs: []runtime.Object{ + &kueue.WorkloadPriorityClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample-priority", + }, + }, + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + Priority("sample-priority"). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create job with --priority flag and skip workload priority class validation": { + args: func(tc *createCmdTestCase) []string { + return []string{"job", "--profile", "profile", "--skip-priority-validation", "--priority", "sample-priority"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + Priority("sample-priority"). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create job with completions replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"job", "--profile", "profile", "--completions", "5"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault). + Parallelism(1). + Completions(1). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Parallelism(1). + Completions(5). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create job with command replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"job", "--profile", "profile", "--cmd", "sleep 15s"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault). + Parallelism(1). + Completions(1). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "sleep").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Parallelism(1). + Completions(1). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Command("sleep", "15s").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "sleep").Obj()). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create job with request replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"job", "--profile", "profile", "--request", "cpu=100m,ram=3Gi"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault). + Parallelism(1). + Completions(1). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "sleep").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Parallelism(1). + Completions(1). + WithContainer( + *wrappers.MakeContainer("c1", "sleep"). + WithRequest("cpu", resource.MustParse("100m")). + WithRequest("ram", resource.MustParse("3Gi")). + Obj(), + ). + WithContainer(*wrappers.MakeContainer("c2", "sleep").Obj()). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create ray job with replicas replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--replicas", "g1=5"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayJobTemplate("ray-job-template", metav1.NamespaceDefault). + WithRayClusterSpec( + wrappers.MakeRayClusterSpec(). + WithWorkerGroupSpec(*wrappers.MakeWorkerGroupSpec("g1").Obj()). + Obj(), + ). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayJobMode, "ray-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayJob"}}, + wantLists: []runtime.Object{ + &rayv1.RayJobList{ + TypeMeta: metav1.TypeMeta{Kind: "RayJobList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayJob{ + *wrappers.MakeRayJob("", metav1.NamespaceDefault). + GenerateName("profile-rayjob-"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + WithWorkerGroupSpec(*wrappers.MakeWorkerGroupSpec("g1").Replicas(5).Obj()). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "rayjob.ray.io/ created\n", + }, + "should create ray job with cmd replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--cmd", "sleep 3s"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayJobTemplate("ray-job-template", metav1.NamespaceDefault). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayJobMode, "ray-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayJob"}}, + wantLists: []runtime.Object{ + &rayv1.RayJobList{ + TypeMeta: metav1.TypeMeta{Kind: "RayJobList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayJob{ + *wrappers.MakeRayJob("", metav1.NamespaceDefault). + GenerateName("profile-rayjob-"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + Entrypoint("sleep 3s"). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "rayjob.ray.io/ created\n", + }, + "should create ray job with min-replicas replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--min-replicas", "g1=5"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayJobTemplate("ray-job-template", metav1.NamespaceDefault). + WithRayClusterSpec( + wrappers.MakeRayClusterSpec(). + WithWorkerGroupSpec(*wrappers.MakeWorkerGroupSpec("g1").Obj()). + Obj(), + ). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayJobMode, "ray-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayJob"}}, + wantLists: []runtime.Object{ + &rayv1.RayJobList{ + TypeMeta: metav1.TypeMeta{Kind: "RayJobList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayJob{ + *wrappers.MakeRayJob("", metav1.NamespaceDefault). + GenerateName("profile-rayjob-"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + WithWorkerGroupSpec(*wrappers.MakeWorkerGroupSpec("g1").MinReplicas(5).Obj()). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "rayjob.ray.io/ created\n", + }, + "should create ray job with max-replicas replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--max-replicas", "g1=5"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayJobTemplate("ray-job-template", metav1.NamespaceDefault). + WithRayClusterSpec( + wrappers.MakeRayClusterSpec(). + WithWorkerGroupSpec(*wrappers.MakeWorkerGroupSpec("g1").Obj()). + Obj(), + ). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayJobMode, "ray-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayJob"}}, + wantLists: []runtime.Object{ + &rayv1.RayJobList{ + TypeMeta: metav1.TypeMeta{Kind: "RayJobList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayJob{ + *wrappers.MakeRayJob("", metav1.NamespaceDefault). + GenerateName("profile-rayjob-"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + WithWorkerGroupSpec(*wrappers.MakeWorkerGroupSpec("g1").MaxReplicas(5).Obj()). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "rayjob.ray.io/ created\n", + }, + "should create ray job with raycluster replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--raycluster", "rc1"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayJobTemplate("ray-job-template", metav1.NamespaceDefault). + WithRayClusterSpec( + wrappers.MakeRayClusterSpec(). + WithWorkerGroupSpec(*wrappers.MakeWorkerGroupSpec("g1").Obj()). + Obj(), + ). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayJobMode, "ray-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayJob"}}, + wantLists: []runtime.Object{ + &rayv1.RayJobList{ + TypeMeta: metav1.TypeMeta{Kind: "RayJobList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayJob{ + *wrappers.MakeRayJob("", metav1.NamespaceDefault). + GenerateName("profile-rayjob-"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + WithRayClusterLabelSelector("rc1"). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "rayjob.ray.io/ created\n", + }, + "should create ray job with --priority flag": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--priority", "sample-priority"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayJobTemplate("ray-job-template", metav1.NamespaceDefault). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayJobMode, "ray-job-template").Obj()). + Obj(), + }, + kueueObjs: []runtime.Object{ + &kueue.WorkloadPriorityClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample-priority", + }, + }, + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayJob"}}, + wantLists: []runtime.Object{ + &rayv1.RayJobList{ + TypeMeta: metav1.TypeMeta{Kind: "RayJobList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayJob{ + *wrappers.MakeRayJob("", metav1.NamespaceDefault). + Priority("sample-priority"). + GenerateName("profile-rayjob-"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "rayjob.ray.io/ created\n", + }, + "should create ray job with --priority flag and skip workload priority class validation": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--skip-priority-validation", "--priority", "sample-priority"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayJobTemplate("ray-job-template", metav1.NamespaceDefault). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayJobMode, "ray-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayJob"}}, + wantLists: []runtime.Object{ + &rayv1.RayJobList{ + TypeMeta: metav1.TypeMeta{Kind: "RayJobList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayJob{ + *wrappers.MakeRayJob("", metav1.NamespaceDefault). + Priority("sample-priority"). + GenerateName("profile-rayjob-"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "rayjob.ray.io/ created\n", + }, + "shouldn't create ray job with raycluster and localqueue replacements because mutually exclusive": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--raycluster", "rc1", "--localqueue", "lq1"} + }, + wantErr: "if any flags in the group [raycluster localqueue] are set none of the others can be; [localqueue raycluster] were all set", + }, + "shouldn't create ray job with raycluster and replicas replacements because mutually exclusive": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--raycluster", "rc1", "--replicas", "g1=5"} + }, + wantErr: "if any flags in the group [raycluster replicas] are set none of the others can be; [raycluster replicas] were all set", + }, + "shouldn't create ray job with raycluster and min-replicas replacements because mutually exclusive": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--raycluster", "rc1", "--min-replicas", "g1=5"} + }, + wantErr: "if any flags in the group [raycluster min-replicas] are set none of the others can be; [min-replicas raycluster] were all set", + }, + "shouldn't create ray job with raycluster and max-replicas replacements because mutually exclusive": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--raycluster", "rc1", "--max-replicas", "g1=5"} + }, + wantErr: "if any flags in the group [raycluster max-replicas] are set none of the others can be; [max-replicas raycluster] were all set", + }, + "should create raycluster with array ": { + args: func(tc *createCmdTestCase) []string { return []string{"raycluster", "--profile", "profile"} }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayClusterTemplate("ray-cluster-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayClusterMode, "ray-cluster-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayCluster"}}, + wantLists: []runtime.Object{ + &rayv1.RayClusterList{ + TypeMeta: metav1.TypeMeta{Kind: "RayClusterList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("", metav1.NamespaceDefault). + GenerateName("profile-raycluster-"). + Profile("profile"). + Mode(v1alpha1.RayClusterMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "raycluster.ray.io/ created\n", + }, + "should create raycluster with --priority flag": { + args: func(tc *createCmdTestCase) []string { + return []string{ + "raycluster", + "--profile", "profile", + "--priority", "sample-priority", + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayClusterTemplate("ray-cluster-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayClusterMode, "ray-cluster-template").Obj()). + Obj(), + }, + kueueObjs: []runtime.Object{ + &kueue.WorkloadPriorityClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample-priority", + }, + }, + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayCluster"}}, + wantLists: []runtime.Object{ + &rayv1.RayClusterList{ + TypeMeta: metav1.TypeMeta{Kind: "RayClusterList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("", metav1.NamespaceDefault). + Priority("sample-priority"). + GenerateName("profile-raycluster-"). + Profile("profile"). + Mode(v1alpha1.RayClusterMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "raycluster.ray.io/ created\n", + }, + "should create raycluster with --priority flag and skip workload priority class validation": { + args: func(tc *createCmdTestCase) []string { + return []string{ + "raycluster", + "--profile", "profile", + "--skip-priority-validation", + "--priority", "sample-priority", + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayClusterTemplate("ray-cluster-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayClusterMode, "ray-cluster-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayCluster"}}, + wantLists: []runtime.Object{ + &rayv1.RayClusterList{ + TypeMeta: metav1.TypeMeta{Kind: "RayClusterList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("", metav1.NamespaceDefault). + Priority("sample-priority"). + GenerateName("profile-raycluster-"). + Profile("profile"). + Mode(v1alpha1.RayClusterMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "raycluster.ray.io/ created\n", + }, + "shouldn't create slurm because slurm args must be specified": { + args: func(tc *createCmdTestCase) []string { + return []string{"slurm", "--profile", "profile"} + }, + wantErr: "requires at least 1 arg(s), only received 0", + }, + "shouldn't create slurm because script must be specified on slurm args": { + args: func(tc *createCmdTestCase) []string { + return []string{"slurm", "--profile", "profile", "./script.sh"} + }, + wantErr: "unknown command \"./script.sh\" for \"create slurm\"", + }, + "shouldn't create slurm because script must be specified": { + args: func(tc *createCmdTestCase) []string { + return []string{"slurm", "--profile", "profile", "--", "--array", "0-5"} + }, + wantErr: "must specify script", + }, + "shouldn't create slurm because script only one script must be specified": { + args: func(tc *createCmdTestCase) []string { + return []string{"slurm", "--profile", "profile", "--", "./script.sh", "./script.sh"} + }, + wantErr: "must specify only one script", + }, + "should create slurm": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{"slurm", "--profile", "profile", "--", tc.tempFile} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("profile-slurm", metav1.NamespaceDefault). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Subdomain("profile-slurm"). + WithInitContainer(*wrappers.MakeContainer("slurm-init-env", "registry.k8s.io/busybox:1.27.2"). + Command("sh", "/slurm/scripts/init-entrypoint.sh"). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-scripts", MountPath: "/slurm/scripts"}). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-env", MountPath: "/slurm/env"}). + WithEnvVar(corev1.EnvVar{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"}, + }}). + Obj()). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-scripts", MountPath: "/slurm/scripts"}). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-env", MountPath: "/slurm/env"}). + Obj()). + WithVolume(corev1.Volume{ + Name: "slurm-scripts", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "profile-slurm"}, + Items: []corev1.KeyToPath{ + {Key: "init-entrypoint.sh", Path: "init-entrypoint.sh"}, + {Key: "entrypoint.sh", Path: "entrypoint.sh"}, + {Key: "script", Path: "script", Mode: ptr.To[int32](0755)}, + }, + }, + }, + }). + WithVolume(corev1.Volume{ + Name: "slurm-env", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + WithEnvVar(corev1.EnvVar{Name: "JOB_CONTAINER_INDEX", Value: "0"}). + Obj(), + }, + }, + &corev1.ConfigMapList{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMapList", APIVersion: "v1"}, + Items: []corev1.ConfigMap{ + *wrappers.MakeConfigMap("profile-slurm", metav1.NamespaceDefault). + WithOwnerReference(metav1.OwnerReference{ + Name: "profile-slurm", + APIVersion: "batch/v1", + Kind: "Job", + }). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Data(map[string]string{ + "script": "#!/bin/bash\nsleep 300'", + "init-entrypoint.sh": `#!/bin/sh + +set -o errexit +set -o nounset +set -o pipefail +set -x + +# External variables +# JOB_COMPLETION_INDEX - completion index of the job. +# POD_IP - current pod IP + +array_indexes="0" +container_indexes=$(echo "$array_indexes" | awk -F';' -v idx="$JOB_COMPLETION_INDEX" '{print $((idx + 1))}') + +for i in $(seq 0 1) +do + container_index=$(echo "$container_indexes" | awk -F',' -v idx="$i" '{print $((idx + 1))}') + + if [ -z "$container_index" ]; then + break + fi + + mkdir -p /slurm/env/$i + + + cat << EOF > /slurm/env/$i/slurm.env +SLURM_ARRAY_JOB_ID=1 +SLURM_ARRAY_TASK_COUNT=1 +SLURM_ARRAY_TASK_MAX=0 +SLURM_ARRAY_TASK_MIN=0 +SLURM_TASKS_PER_NODE=1 +SLURM_CPUS_PER_TASK= +SLURM_CPUS_ON_NODE= +SLURM_JOB_CPUS_PER_NODE= +SLURM_CPUS_PER_GPU= +SLURM_MEM_PER_CPU= +SLURM_MEM_PER_GPU= +SLURM_MEM_PER_NODE= +SLURM_GPUS= +SLURM_NTASKS=1 +SLURM_NTASKS_PER_NODE=1 +SLURM_NPROCS=1 +SLURM_NNODES=1 +SLURM_SUBMIT_DIR=/slurm/scripts +SLURM_SUBMIT_HOST=$HOSTNAME +SLURM_JOB_NODELIST=profile-slurm-0.profile-slurm +SLURM_JOB_FIRST_NODE=profile-slurm-0.profile-slurm +SLURM_JOB_ID=$(expr $JOB_COMPLETION_INDEX \* 1 + $i + 1) +SLURM_JOBID=$(expr $JOB_COMPLETION_INDEX \* 1 + $i + 1) +SLURM_ARRAY_TASK_ID=$container_index +SLURM_JOB_FIRST_NODE_IP=${SLURM_JOB_FIRST_NODE_IP:-""} +EOF + +done +`, + "entrypoint.sh": `#!/usr/local/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +# External variables +# JOB_CONTAINER_INDEX - container index in the container template. + +if [ ! -d "/slurm/env/$JOB_CONTAINER_INDEX" ]; then + exit 0 +fi + +SBATCH_JOB_NAME= + +export $(cat /slurm/env/$JOB_CONTAINER_INDEX/slurm.env | xargs) + +/slurm/scripts/script +`, + }). + Obj(), + }, + }, + &corev1.ServiceList{ + TypeMeta: metav1.TypeMeta{Kind: "ServiceList", APIVersion: "v1"}, + Items: []corev1.Service{ + *wrappers.MakeService("profile-slurm", metav1.NamespaceDefault). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + ClusterIP("None"). + Selector("job-name", "profile-slurm"). + WithOwnerReference(metav1.OwnerReference{ + Name: "profile-slurm", + APIVersion: "batch/v1", + Kind: "Job", + }). + Obj(), + }, + }, + }, + cmpopts: []cmp.Option{ + cmpopts.AcyclicTransformer("RemoveGeneratedNameSuffixInString", func(val string) string { + return regexp.MustCompile("(profile-slurm)(-.{5})").ReplaceAllString(val, "$1") + }), + cmpopts.AcyclicTransformer("RemoveGeneratedNameSuffixInMap", func(m map[string]string) map[string]string { + for key, val := range m { + m[key] = regexp.MustCompile("(profile-slurm)(-.{5})").ReplaceAllString(val, "$1") + } + return m + }), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "should create slurm with flags": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--localqueue", "lq1", + "--init-image", "bash:latest", + "--first-node-ip", + "--first-node-ip-timeout", "29s", + "--", + "--array", "0-25", + "--nodes", "2", + "--ntasks", "3", + "--input", "\\\\/home/%u/%x/stderr%%-%A-%a-%j-%N-%n-%t.out", + "--output", "/home/%u/%x/stdout%%-%A-%a-%j-%N-%n-%t.out", + "--error", "/home/%u/%x/stderr%%-%A-%a-%j-%N-%n-%t.out", + "--job-name", "job-name", + "--partition", "lq1", + "--chdir", "/mydir", + "--cpus-per-task", "2", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + kueueObjs: []runtime.Object{ + &kueue.LocalQueue{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "lq1", + }, + }, + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("profile-slurm", metav1.NamespaceDefault). + Parallelism(2). + Completions(9). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Subdomain("profile-slurm"). + WithInitContainer(*wrappers.MakeContainer("slurm-init-env", "bash:latest"). + Command("sh", "/slurm/scripts/init-entrypoint.sh"). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-scripts", MountPath: "/slurm/scripts"}). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-env", MountPath: "/slurm/env"}). + WithEnvVar(corev1.EnvVar{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"}, + }}). + Obj()). + WithContainer(*wrappers.MakeContainer("c1-0", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-scripts", MountPath: "/slurm/scripts"}). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-env", MountPath: "/slurm/env"}). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-scripts", MountPath: "/slurm/scripts"}). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-env", MountPath: "/slurm/env"}). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + Obj()). + WithContainer(*wrappers.MakeContainer("c1-1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-scripts", MountPath: "/slurm/scripts"}). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-env", MountPath: "/slurm/env"}). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + Obj()). + WithContainer(*wrappers.MakeContainer("c1-2", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-scripts", MountPath: "/slurm/scripts"}). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-env", MountPath: "/slurm/env"}). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + Obj()). + WithVolume(corev1.Volume{ + Name: "slurm-scripts", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "profile-slurm"}, + Items: []corev1.KeyToPath{ + {Key: "init-entrypoint.sh", Path: "init-entrypoint.sh"}, + {Key: "entrypoint.sh", Path: "entrypoint.sh"}, + {Key: "script", Path: "script", Mode: ptr.To[int32](0755)}, + }, + }, + }, + }). + WithVolume(corev1.Volume{ + Name: "slurm-env", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + WithEnvVarIndexValue("JOB_CONTAINER_INDEX"). + Obj(), + }, + }, + &corev1.ConfigMapList{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMapList", APIVersion: "v1"}, + Items: []corev1.ConfigMap{ + *wrappers.MakeConfigMap("profile-slurm", metav1.NamespaceDefault). + WithOwnerReference(metav1.OwnerReference{ + Name: "profile-slurm", + APIVersion: "batch/v1", + Kind: "Job", + }). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Data(map[string]string{ + "script": "#!/bin/bash\nsleep 300'", + "init-entrypoint.sh": `#!/bin/sh + +set -o errexit +set -o nounset +set -o pipefail +set -x + +# External variables +# JOB_COMPLETION_INDEX - completion index of the job. +# POD_IP - current pod IP + +array_indexes="0,1,2;3,4,5;6,7,8;9,10,11;12,13,14;15,16,17;18,19,20;21,22,23;24,25" +container_indexes=$(echo "$array_indexes" | awk -F';' -v idx="$JOB_COMPLETION_INDEX" '{print $((idx + 1))}') + +for i in $(seq 0 3) +do + container_index=$(echo "$container_indexes" | awk -F',' -v idx="$i" '{print $((idx + 1))}') + + if [ -z "$container_index" ]; then + break + fi + + mkdir -p /slurm/env/$i + + + if [[ "$JOB_COMPLETION_INDEX" -eq 0 ]]; then + SLURM_JOB_FIRST_NODE_IP=${POD_IP} + else + timeout=29 + start_time=$(date +%s) + while true; do + ip=$(nslookup "profile-slurm-r8njg-0.profile-slurm-r8njg" | grep "Address 1" | awk 'NR==2 {print $3}') || true + if [[ -n "$ip" ]]; then + SLURM_JOB_FIRST_NODE_IP=$ip + break + else + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + if [ "$elapsed_time" -ge "$timeout" ]; then + echo "Timeout reached, IP address for the first node (profile-slurm-r8njg-0.profile-slurm-r8njg) not found." + break + fi + echo "IP Address for the first node (profile-slurm-r8njg-0.profile-slurm-r8njg) not found, retrying..." + sleep 1 + fi + done + fi + + cat << EOF > /slurm/env/$i/slurm.env +SLURM_ARRAY_JOB_ID=1 +SLURM_ARRAY_TASK_COUNT=26 +SLURM_ARRAY_TASK_MAX=25 +SLURM_ARRAY_TASK_MIN=0 +SLURM_TASKS_PER_NODE=3 +SLURM_CPUS_PER_TASK=2 +SLURM_CPUS_ON_NODE=8 +SLURM_JOB_CPUS_PER_NODE=8 +SLURM_CPUS_PER_GPU= +SLURM_MEM_PER_CPU= +SLURM_MEM_PER_GPU= +SLURM_MEM_PER_NODE= +SLURM_GPUS= +SLURM_NTASKS=3 +SLURM_NTASKS_PER_NODE=3 +SLURM_NPROCS=3 +SLURM_NNODES=2 +SLURM_SUBMIT_DIR=/slurm/scripts +SLURM_SUBMIT_HOST=$HOSTNAME +SLURM_JOB_NODELIST=profile-slurm-fpxnj-0.profile-slurm-fpxnj,profile-slurm-fpxnj-1.profile-slurm-fpxnj +SLURM_JOB_FIRST_NODE=profile-slurm-fpxnj-0.profile-slurm-fpxnj +SLURM_JOB_ID=$(expr $JOB_COMPLETION_INDEX \* 3 + $i + 1) +SLURM_JOBID=$(expr $JOB_COMPLETION_INDEX \* 3 + $i + 1) +SLURM_ARRAY_TASK_ID=$container_index +SLURM_JOB_FIRST_NODE_IP=${SLURM_JOB_FIRST_NODE_IP:-""} +EOF + +done +`, + "entrypoint.sh": `#!/usr/local/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +# External variables +# JOB_CONTAINER_INDEX - container index in the container template. + +if [ ! -d "/slurm/env/$JOB_CONTAINER_INDEX" ]; then + exit 0 +fi + +SBATCH_JOB_NAME=job-name + +export $(cat /slurm/env/$JOB_CONTAINER_INDEX/slurm.env | xargs)cd /mydir + +/slurm/scripts/script /home/${USER_ID}/${SBATCH_JOB_NAME}/stdout%-${SLURM_ARRAY_JOB_ID}-${SLURM_ARRAY_TASK_ID}-${SLURM_JOB_ID}-${HOSTNAME}-${JOB_COMPLETION_INDEX}-${SLURM_ARRAY_TASK_ID}.out 2>/home/${USER_ID}/${SBATCH_JOB_NAME}/stderr%-${SLURM_ARRAY_JOB_ID}-${SLURM_ARRAY_TASK_ID}-${SLURM_JOB_ID}-${HOSTNAME}-${JOB_COMPLETION_INDEX}-${SLURM_ARRAY_TASK_ID}.out +`, + }). + Obj(), + }, + }, + &corev1.ServiceList{ + TypeMeta: metav1.TypeMeta{Kind: "ServiceList", APIVersion: "v1"}, + Items: []corev1.Service{ + *wrappers.MakeService("profile-slurm", metav1.NamespaceDefault). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + ClusterIP("None"). + Selector("job-name", "profile-slurm"). + WithOwnerReference(metav1.OwnerReference{ + Name: "profile-slurm", + APIVersion: "batch/v1", + Kind: "Job", + }). + Obj(), + }, + }, + }, + cmpopts: []cmp.Option{ + cmpopts.AcyclicTransformer("RemoveGeneratedNameSuffixInString", func(val string) string { + return regexp.MustCompile("(profile-slurm)(-.{5})").ReplaceAllString(val, "$1") + }), + cmpopts.AcyclicTransformer("RemoveGeneratedNameSuffixInMap", func(m map[string]string) map[string]string { + for key, val := range m { + m[key] = regexp.MustCompile("(profile-slurm)(-.{5})").ReplaceAllString(val, "$1") + } + return m + }), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "should create slurm with --ntasks flag": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--", + "--ntasks", "3", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Subdomain("profile-slurm"). + WithContainer(*wrappers.MakeContainer("c1-0", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + Obj()). + WithContainer(*wrappers.MakeContainer("c1-1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + Obj()). + WithContainer(*wrappers.MakeContainer("c1-2", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + Obj()). + Obj(), + }, + }, + &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + }, + cmpopts: []cmp.Option{ + cmpopts.IgnoreFields(corev1.LocalObjectReference{}, "Name"), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "Name"), + cmpopts.IgnoreFields(corev1.PodSpec{}, "InitContainers", "Subdomain"), + cmpopts.IgnoreTypes([]corev1.EnvVar{}), + cmpopts.IgnoreTypes([]corev1.Volume{}), + cmpopts.IgnoreTypes([]corev1.VolumeMount{}), + cmpopts.IgnoreTypes(corev1.ConfigMapList{}), + cmpopts.IgnoreTypes(corev1.ServiceList{}), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "should divide --mem exactly across containers": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--", + "--mem", "2G", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithResources(corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("1G"), + }, + }). + Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithResources(corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("1G"), + }, + }). + Obj()). + Obj(), + }, + }, + &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + }, + cmpopts: []cmp.Option{ + cmpopts.IgnoreFields(corev1.LocalObjectReference{}, "Name"), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "Name"), + cmpopts.IgnoreFields(corev1.PodSpec{}, "InitContainers", "Subdomain"), + cmpopts.IgnoreTypes([]corev1.EnvVar{}), + cmpopts.IgnoreTypes([]corev1.Volume{}), + cmpopts.IgnoreTypes([]corev1.VolumeMount{}), + cmpopts.IgnoreTypes(corev1.ConfigMapList{}), + cmpopts.IgnoreTypes(corev1.ServiceList{}), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "should handle non-exact --mem division across containers": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--", + "--mem", "1G", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithResources(corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("500M"), + }, + }). + Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithResources(corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("500M"), + }, + }). + Obj()). + Obj(), + }, + }, + &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + }, + cmpopts: []cmp.Option{ + cmpopts.IgnoreFields(corev1.LocalObjectReference{}, "Name"), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "Name"), + cmpopts.IgnoreFields(corev1.PodSpec{}, "InitContainers", "Subdomain"), + cmpopts.IgnoreTypes([]corev1.EnvVar{}), + cmpopts.IgnoreTypes([]corev1.Volume{}), + cmpopts.IgnoreTypes([]corev1.VolumeMount{}), + cmpopts.IgnoreTypes(corev1.ConfigMapList{}), + cmpopts.IgnoreTypes(corev1.ServiceList{}), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "should create slurm with --mem-per-cpu flag": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--", + "--cpus-per-task", "2", + "--mem-per-cpu", "500M", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithResources(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("1G"), + }, + }). + Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithResources(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("1G"), + }, + }). + Obj()). + Obj(), + }, + }, + &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + }, + cmpopts: []cmp.Option{ + cmpopts.IgnoreFields(corev1.LocalObjectReference{}, "Name"), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "Name"), + cmpopts.IgnoreFields(corev1.PodSpec{}, "InitContainers", "Subdomain"), + cmpopts.IgnoreTypes([]corev1.EnvVar{}), + cmpopts.IgnoreTypes([]corev1.Volume{}), + cmpopts.IgnoreTypes([]corev1.VolumeMount{}), + cmpopts.IgnoreTypes(corev1.ConfigMapList{}), + cmpopts.IgnoreTypes(corev1.ServiceList{}), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "shouldn't create slurm with --mem-per-cpu flag because --cpus-per-task flag not specified": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--", + "--mem-per-cpu", "500M", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + wantErr: "no cpus-per-task specified", + }, + "should create slurm with --priority flag": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--priority", "sample-priority", + "--", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + kueueObjs: []runtime.Object{ + &kueue.WorkloadPriorityClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample-priority", + }, + }, + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + Priority("sample-priority"). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + Obj()). + Obj(), + }, + }, + &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + }, + cmpopts: []cmp.Option{ + cmpopts.IgnoreFields(corev1.LocalObjectReference{}, "Name"), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "Name"), + cmpopts.IgnoreFields(corev1.PodSpec{}, "InitContainers", "Subdomain"), + cmpopts.IgnoreTypes([]corev1.EnvVar{}), + cmpopts.IgnoreTypes([]corev1.Volume{}), + cmpopts.IgnoreTypes([]corev1.VolumeMount{}), + cmpopts.IgnoreTypes(corev1.ConfigMapList{}), + cmpopts.IgnoreTypes(corev1.ServiceList{}), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "should create slurm with --mem-per-gpu flag": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--", + "--gpus-per-task", "volta:3,kepler:1", + "--mem-per-gpu", "500M", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithResources(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("2G"), + "volta": resource.MustParse("3"), + "kepler": resource.MustParse("1"), + }, + }). + Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithResources(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("2G"), + "volta": resource.MustParse("3"), + "kepler": resource.MustParse("1"), + }, + }). + Obj()). + Obj(), + }, + }, + &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + }, + cmpopts: []cmp.Option{ + cmpopts.IgnoreFields(corev1.LocalObjectReference{}, "Name"), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "Name"), + cmpopts.IgnoreFields(corev1.PodSpec{}, "InitContainers", "Subdomain"), + cmpopts.IgnoreTypes([]corev1.EnvVar{}), + cmpopts.IgnoreTypes([]corev1.Volume{}), + cmpopts.IgnoreTypes([]corev1.VolumeMount{}), + cmpopts.IgnoreTypes(corev1.ConfigMapList{}), + cmpopts.IgnoreTypes(corev1.ServiceList{}), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "shouldn't create slurm with --mem-per-gpu flag because --gpus-per-task flag not specified": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--", + "--mem-per-gpu", "500M", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + wantErr: "no gpus-per-task specified", + }, + "should create slurm with --priority flag and skip workload priority class validation": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--priority", "sample-priority", + "--skip-priority-validation", + "--", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + Priority("sample-priority"). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Subdomain("profile-slurm"). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + Obj()). + Obj(), + }, + }, + &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + }, + cmpopts: []cmp.Option{ + cmpopts.IgnoreFields(corev1.LocalObjectReference{}, "Name"), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "Name"), + cmpopts.IgnoreFields(corev1.PodSpec{}, "InitContainers", "Subdomain"), + cmpopts.IgnoreTypes([]corev1.EnvVar{}), + cmpopts.IgnoreTypes([]corev1.Volume{}), + cmpopts.IgnoreTypes([]corev1.VolumeMount{}), + cmpopts.IgnoreTypes(corev1.ConfigMapList{}), + cmpopts.IgnoreTypes(corev1.ServiceList{}), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "should create slurm with --time flag": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--", + tc.tempFile, + "--time", "1", + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + MaxExecTimeSecondsLabel("60"). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Subdomain("profile-slurm"). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + Obj()). + Obj(), + }, + }, + &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + }, + cmpopts: []cmp.Option{ + cmpopts.IgnoreFields(corev1.LocalObjectReference{}, "Name"), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "Name"), + cmpopts.IgnoreFields(corev1.PodSpec{}, "InitContainers", "Subdomain"), + cmpopts.IgnoreTypes([]corev1.EnvVar{}), + cmpopts.IgnoreTypes([]corev1.Volume{}), + cmpopts.IgnoreTypes([]corev1.VolumeMount{}), + cmpopts.IgnoreTypes(corev1.ConfigMapList{}), + cmpopts.IgnoreTypes(corev1.ServiceList{}), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "should create slurm with -t flag": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--", + tc.tempFile, + "-t", "2-12:05:23", + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + MaxExecTimeSecondsLabel("216323"). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Subdomain("profile-slurm"). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + Obj()). + Obj(), + }, + }, + &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + }, + cmpopts: []cmp.Option{ + cmpopts.IgnoreFields(corev1.LocalObjectReference{}, "Name"), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "Name"), + cmpopts.IgnoreFields(corev1.PodSpec{}, "InitContainers", "Subdomain"), + cmpopts.IgnoreTypes([]corev1.EnvVar{}), + cmpopts.IgnoreTypes([]corev1.Volume{}), + cmpopts.IgnoreTypes([]corev1.VolumeMount{}), + cmpopts.IgnoreTypes(corev1.ConfigMapList{}), + cmpopts.IgnoreTypes(corev1.ServiceList{}), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "shouldn't create job with client dry run": { + args: func(tc *createCmdTestCase) []string { + return []string{"job", "--profile", "profile", "--dry-run", "client"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{}, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created (client dry run)\n", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + if tc.beforeTest != nil { + tc.beforeTest(t, &tc) + } + + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + scheme := runtime.NewScheme() + utilruntime.Must(k8sscheme.AddToScheme(scheme)) + utilruntime.Must(rayv1.AddToScheme(scheme)) + + clientset := kjobctlfake.NewSimpleClientset(tc.kjobctlObjs...) + dynamicClient := fake.NewSimpleDynamicClient(scheme) + kueueClientset := kueuefake.NewSimpleClientset(tc.kueueObjs...) + restMapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{}) + + for _, gvk := range tc.gvks { + restMapper.Add(gvk, meta.RESTScopeNamespace) + } + + tcg := cmdtesting.NewTestClientGetter(). + WithKjobctlClientset(clientset). + WithDynamicClient(dynamicClient). + WithKueueClientset(kueueClientset). + WithRESTMapper(restMapper) + if tc.ns != "" { + tcg.WithNamespace(tc.ns) + } + + cmd := NewCreateCmd(tcg, streams, clocktesting.NewFakeClock(testStartTime)) + cmd.SetOut(out) + cmd.SetErr(outErr) + cmd.SetArgs(tc.args(&tc)) + + gotErr := cmd.Execute() + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + if gotErr != nil { + return + } + + gotOut := out.String() + if tc.wantOutPattern != "" { + gotOut = strings.ReplaceAll(gotOut, "\n", "\\n") + match, err := regexp.MatchString(tc.wantOutPattern, gotOut) + if err != nil { + t.Error(err) + return + } + if !match { + t.Errorf("Unexpected output. Not match pattern \"%s\":\n%s", tc.wantOutPattern, gotOut) + } + } else if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + for index, gvk := range tc.gvks { + mapping, err := restMapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + t.Error(err) + return + } + + unstructured, err := dynamicClient.Resource(mapping.Resource).Namespace(metav1.NamespaceDefault). + List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Error(err) + return + } + + gotList := tc.wantLists[index].DeepCopyObject() + + err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructured.UnstructuredContent(), gotList) + if err != nil { + t.Error(err) + return + } + + if job, ok := tc.wantLists[index].(*batchv1.JobList); ok && len(job.Items) > 0 { + if tc.tempFile != "" { + if job.Items[0].Annotations == nil { + job.Items[0].Annotations = make(map[string]string) + } + job.Items[0].Annotations[constants.ScriptAnnotation] = tc.tempFile + } + } + + if diff := cmp.Diff(tc.wantLists[index], gotList, tc.cmpopts...); diff != "" { + t.Errorf("Unexpected list for %s (-want/+got)\n%s", gvk.String(), diff) + } + } + }) + } +} + +func TestCreateOptionsRunInteractive(t *testing.T) { + testStartTime := time.Now() + userID := os.Getenv(constants.SystemEnvVarNameUser) + + testCases := map[string]struct { + options *CreateOptions + k8sObjs []runtime.Object + kjobctlObjs []runtime.Object + createMutation func(pod *corev1.Pod) + wantPodList *corev1.PodList + wantErr string + }{ + "success": { + options: &CreateOptions{ + Namespace: metav1.NamespaceDefault, + ProfileName: "profile", + ModeName: v1alpha1.InteractiveMode, + Attach: &fakeRemoteAttach{}, + AttachFunc: testAttachFunc, + }, + k8sObjs: []runtime.Object{ + wrappers.MakePodTemplate("pod-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + Obj(), + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.InteractiveMode, "pod-template").Obj()). + Obj(), + }, + createMutation: func(pod *corev1.Pod) { + pod.Status.Phase = corev1.PodRunning + }, + wantPodList: &corev1.PodList{ + Items: []corev1.Pod{ + *wrappers.MakePod("", metav1.NamespaceDefault). + GenerateName("profile-interactive-"). + Profile("profile"). + Mode(v1alpha1.InteractiveMode). + WithContainer(*wrappers.MakeContainer("c1", "sleep"). + TTY(). + Stdin(). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj()). + Phase(corev1.PodRunning). + Obj(), + }, + }, + }, + "success with remove interactive pod": { + options: &CreateOptions{ + Namespace: metav1.NamespaceDefault, + ProfileName: "profile", + ModeName: v1alpha1.InteractiveMode, + RemoveInteractivePod: true, + Attach: &fakeRemoteAttach{}, + AttachFunc: testAttachFunc, + }, + k8sObjs: []runtime.Object{ + wrappers.MakePodTemplate("pod-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + Obj(), + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.InteractiveMode, "pod-template").Obj()). + Obj(), + }, + createMutation: func(pod *corev1.Pod) { + pod.Status.Phase = corev1.PodRunning + }, + wantPodList: &corev1.PodList{}, + }, + "success with dry-run client": { + options: &CreateOptions{ + Namespace: metav1.NamespaceDefault, + ProfileName: "profile", + ModeName: v1alpha1.InteractiveMode, + DryRunStrategy: cmdutil.DryRunClient, + Attach: &fakeRemoteAttach{}, + AttachFunc: testAttachFunc, + }, + k8sObjs: []runtime.Object{ + wrappers.MakePodTemplate("pod-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + Obj(), + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.InteractiveMode, "pod-template").Obj()). + Obj(), + }, + wantPodList: &corev1.PodList{}, + }, + "timeout waiting for pod": { + options: &CreateOptions{ + Namespace: metav1.NamespaceDefault, + ProfileName: "profile", + ModeName: v1alpha1.InteractiveMode, + Attach: &fakeRemoteAttach{}, + AttachFunc: testAttachFunc, + }, + k8sObjs: []runtime.Object{ + wrappers.MakePodTemplate("pod-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + Obj(), + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.InteractiveMode, "pod-template").Obj()). + Obj(), + }, + wantPodList: &corev1.PodList{ + Items: []corev1.Pod{ + *wrappers.MakePod("", metav1.NamespaceDefault). + GenerateName("profile-interactive-"). + Profile("profile"). + Mode(v1alpha1.InteractiveMode). + WithContainer(*wrappers.MakeContainer("c1", "sleep"). + TTY(). + Stdin(). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj()). + Obj(), + }, + }, + wantErr: "context deadline exceeded", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + tc.options.IOStreams = streams + tc.options.Out = out + tc.options.ErrOut = outErr + tc.options.PrintFlags = genericclioptions.NewPrintFlags("created").WithTypeSetter(k8sscheme.Scheme) + printer, err := tc.options.PrintFlags.ToPrinter() + if err != nil { + t.Fatal(err) + } + tc.options.PrintObj = printer.PrintObj + + k8sClientset := k8sfake.NewSimpleClientset(tc.k8sObjs...) + kjobctlClientset := kjobctlfake.NewSimpleClientset(tc.kjobctlObjs...) + dynamicClient := fake.NewSimpleDynamicClient(k8sscheme.Scheme) + restMapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{}) + restMapper.Add(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}, meta.RESTScopeNamespace) + + dynamicClient.PrependReactor("create", "pods", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + createAction := action.(kubetesting.CreateAction) + + unstructuredObj := createAction.GetObject().(*unstructured.Unstructured) + unstructuredObj.SetName(unstructuredObj.GetGenerateName() + utilrand.String(5)) + + pod := &corev1.Pod{} + + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredObj.UnstructuredContent(), pod); err != nil { + return true, nil, err + } + + if tc.createMutation != nil { + tc.createMutation(pod) + } + + _, err = k8sClientset.CoreV1().Pods(pod.GetNamespace()).Create(context.Background(), pod, metav1.CreateOptions{}) + if err != nil { + return true, nil, err + } + + return true, unstructuredObj, err + }) + + tcg := cmdtesting.NewTestClientGetter(). + WithK8sClientset(k8sClientset). + WithKjobctlClientset(kjobctlClientset). + WithDynamicClient(dynamicClient). + WithRESTMapper(restMapper) + + gotErr := tc.options.Run(context.Background(), tcg, testStartTime) + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + gotPodList, err := k8sClientset.CoreV1().Pods(metav1.NamespaceDefault).List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + + defaultCmpOpts := []cmp.Option{cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name")} + if diff := cmp.Diff(tc.wantPodList, gotPodList, defaultCmpOpts...); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + }) + } +} + +type fakeRemoteAttach struct { + url *url.URL + err error +} + +func (f *fakeRemoteAttach) Attach(url *url.URL, _ *restclient.Config, _ io.Reader, _, _ io.Writer, _ bool, _ remotecommand.TerminalSizeQueue) error { + f.url = url + return f.err +} + +func testAttachFunc(o *CreateOptions, _ *corev1.Container, sizeQueue remotecommand.TerminalSizeQueue, _ *corev1.Pod) func() error { + return func() error { + u, err := url.Parse("http://kjobctl.test") + if err != nil { + return err + } + + return o.Attach.Attach(u, nil, nil, nil, nil, o.TTY, sizeQueue) + } +} diff --git a/pkg/cmd/create/helpers.go b/pkg/cmd/create/helpers.go new file mode 100644 index 0000000..c59b582 --- /dev/null +++ b/pkg/cmd/create/helpers.go @@ -0,0 +1,68 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package create + +import ( + "context" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/scheme" +) + +// setKubernetesDefaults sets default values on the provided client config for accessing the +// Kubernetes API or returns an error if any of the defaults are impossible or invalid. +// copied from https://github.com/kubernetes/kubectl/blob/a6de79e3d40575480caa71a8e5efffeea6587251/pkg/cmd/util/kubectl_match_version.go#L115 +func setKubernetesDefaults(config *rest.Config) error { + // This is allowing the GetOptions to be serialized. + config.GroupVersion = &schema.GroupVersion{Group: "", Version: "v1"} + + if config.APIPath == "" { + config.APIPath = "/api" + } + if config.NegotiatedSerializer == nil { + // This codec factory ensures the resources are not converted. Therefore, resources + // will not be round-tripped through internal versions. Defaulting does not happen + // on the client. + config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + } + return rest.SetKubernetesDefaults(config) +} + +// return a condition function that indicates whether the given pod is currently running +func isPodRunning(ctx context.Context, k8sClient kubernetes.Interface, podName, namespace string) wait.ConditionWithContextFunc { + return func(ct context.Context) (bool, error) { + pod, err := k8sClient.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return false, err + } + + return pod.Status.Phase == corev1.PodRunning, nil + } +} + +// Poll up to timeout seconds for pod to enter running state. +// Returns an error if the pod never enters the running state. +func waitForPodRunning(ctx context.Context, k8sClient kubernetes.Interface, namespace, podName string, timeout time.Duration) error { + return wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, isPodRunning(ctx, k8sClient, podName, namespace)) +} diff --git a/pkg/cmd/delete/delete.go b/pkg/cmd/delete/delete.go new file mode 100644 index 0000000..d6aba48 --- /dev/null +++ b/pkg/cmd/delete/delete.go @@ -0,0 +1,49 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "fmt" + + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/kubectl/pkg/util/templates" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" +) + +var ( + deleteExample = templates.Examples( + fmt.Sprintf("%s\n\n%s\n\n%s\n\n%s", interactiveExample, jobExample, rayJobExample, rayClusterExample), + ) +) + +func NewDeleteCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "delete", + Short: "Delete resources", + Example: deleteExample, + } + + cmd.AddCommand(NewInteractiveCmd(clientGetter, streams)) + cmd.AddCommand(NewJobCmd(clientGetter, streams)) + cmd.AddCommand(NewRayJobCmd(clientGetter, streams)) + cmd.AddCommand(NewRayClusterCmd(clientGetter, streams)) + cmd.AddCommand(NewSlurmCmd(clientGetter, streams)) + + return cmd +} diff --git a/pkg/cmd/delete/delete_interactive.go b/pkg/cmd/delete/delete_interactive.go new file mode 100644 index 0000000..83704d4 --- /dev/null +++ b/pkg/cmd/delete/delete_interactive.go @@ -0,0 +1,176 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/client-go/kubernetes/scheme" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + interactiveExample = templates.Examples(` + # Delete interactive shell + kjobctl delete interactive my-application-profile-interactive-k2wzd + `) +) + +type InteractiveOptions struct { + PrintFlags *genericclioptions.PrintFlags + + InteractiveNames []string + Namespace string + + CascadeStrategy metav1.DeletionPropagation + DryRunStrategy util.DryRunStrategy + + Client corev1.CoreV1Interface + + PrintObj printers.ResourcePrinterFunc + + genericiooptions.IOStreams +} + +func NewInteractiveOptions(streams genericiooptions.IOStreams) *InteractiveOptions { + return &InteractiveOptions{ + PrintFlags: genericclioptions.NewPrintFlags("deleted").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + } +} + +func NewInteractiveCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams) *cobra.Command { + o := NewInteractiveOptions(streams) + + cmd := &cobra.Command{ + Use: "interactive NAME [--cascade STRATEGY] [--dry-run STRATEGY]", + DisableFlagsInUseLine: true, + Short: "Delete interactive shell", + Example: interactiveExample, + Args: cobra.MinimumNArgs(1), + ValidArgsFunction: completion.PodNameFunc(clientGetter), + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + + err := o.Complete(clientGetter, cmd, args) + if err != nil { + return err + } + + return o.Run(cmd.Context()) + }, + } + + addCascadingFlag(cmd) + util.AddDryRunFlag(cmd) + + o.PrintFlags.AddFlags(cmd) + + return cmd +} + +func (o *InteractiveOptions) Complete(clientGetter util.ClientGetter, cmd *cobra.Command, args []string) error { + o.InteractiveNames = args + + var err error + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + o.DryRunStrategy, err = util.GetDryRunStrategy(cmd) + if err != nil { + return err + } + + err = util.PrintFlagsWithDryRunStrategy(o.PrintFlags, o.DryRunStrategy) + if err != nil { + return err + } + + o.CascadeStrategy, err = getCascadingStrategy(cmd) + if err != nil { + return err + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return err + } + + o.PrintObj = printer.PrintObj + + clientset, err := clientGetter.K8sClientset() + if err != nil { + return err + } + + o.Client = clientset.CoreV1() + + return nil +} + +func (o *InteractiveOptions) Run(ctx context.Context) error { + for _, podName := range o.InteractiveNames { + pod, err := o.Client.Pods(o.Namespace).Get(ctx, podName, metav1.GetOptions{}) + if client.IgnoreNotFound(err) != nil { + return err + } + if err != nil { + fmt.Fprintln(o.ErrOut, err) + continue + } + if _, ok := pod.Labels[constants.ProfileLabel]; !ok { + fmt.Fprintf(o.ErrOut, "pods \"%s\" not created via kjob\n", pod.Name) + continue + } + + if o.DryRunStrategy != util.DryRunClient { + deleteOptions := metav1.DeleteOptions{ + PropagationPolicy: ptr.To(o.CascadeStrategy), + } + + if o.DryRunStrategy == util.DryRunServer { + deleteOptions.DryRun = []string{metav1.DryRunAll} + } + + if err := o.Client.Pods(o.Namespace).Delete(ctx, podName, deleteOptions); err != nil { + return err + } + } + + if err := o.PrintObj(pod, o.Out); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/cmd/delete/delete_interactive_test.go b/pkg/cmd/delete/delete_interactive_test.go new file mode 100644 index 0000000..2899a16 --- /dev/null +++ b/pkg/cmd/delete/delete_interactive_test.go @@ -0,0 +1,190 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "slices" + "testing" + + "github.com/google/go-cmp/cmp" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + k8sfake "k8s.io/client-go/kubernetes/fake" + kubetesting "k8s.io/client-go/testing" + + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestInteractiveCmd(t *testing.T) { + testCases := map[string]struct { + ns string + args []string + objs []runtime.Object + wantPods []corev1.Pod + wantOut string + wantOutErr string + wantErr string + }{ + "shouldn't delete an interactive shell because it is not found": { + args: []string{"pod"}, + objs: []runtime.Object{ + wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantPods: []corev1.Pod{ + *wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOutErr: "pods \"pod\" not found\n", + }, + "shouldn't delete an interactive shell because it is not created via kjob": { + args: []string{"pod1"}, + objs: []runtime.Object{ + wrappers.MakePod("pod1", metav1.NamespaceDefault).Obj(), + }, + wantPods: []corev1.Pod{ + *wrappers.MakePod("pod1", metav1.NamespaceDefault).Obj(), + }, + wantOutErr: "pods \"pod1\" not created via kjob\n", + }, + "should delete an interactive shell": { + args: []string{"pod1"}, + objs: []runtime.Object{ + wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantPods: []corev1.Pod{ + *wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "pod/pod1 deleted\n", + }, + "should delete interactive shells": { + args: []string{"pod1", "pod2"}, + objs: []runtime.Object{ + wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "pod/pod1 deleted\npod/pod2 deleted\n", + }, + "should delete only one interactive shell": { + args: []string{"pod1", "pod"}, + objs: []runtime.Object{ + wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantPods: []corev1.Pod{ + *wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "pod/pod1 deleted\n", + wantOutErr: "pods \"pod\" not found\n", + }, + "shouldn't delete an interactive shell with client dry run": { + args: []string{"pod1", "--dry-run", "client"}, + objs: []runtime.Object{ + wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantPods: []corev1.Pod{ + *wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "pod/pod1 deleted (client dry run)\n", + }, + "shouldn't delete an interactive shell with server dry run": { + args: []string{"pod1", "--dry-run", "server"}, + objs: []runtime.Object{ + wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantPods: []corev1.Pod{ + *wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "pod/pod1 deleted (server dry run)\n", + }, + "no args": { + args: []string{}, + wantErr: "requires at least 1 arg(s), only received 0", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ns := metav1.NamespaceDefault + if tc.ns != "" { + ns = tc.ns + } + + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := k8sfake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("delete", "pods", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + if slices.Contains(action.(kubetesting.DeleteAction).GetDeleteOptions().DryRun, metav1.DryRunAll) { + handled = true + } + return handled, ret, err + }) + + tcg := cmdtesting.NewTestClientGetter(). + WithK8sClientset(clientset). + WithNamespace(ns) + + cmd := NewInteractiveCmd(tcg, streams) + cmd.SetOut(out) + cmd.SetErr(outErr) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + if gotErr != nil { + return + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected error output (-want/+got)\n%s", diff) + } + + gotInteractiveList, err := clientset.CoreV1().Pods(tc.ns).List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Error(err) + return + } + + if diff := cmp.Diff(tc.wantPods, gotInteractiveList.Items); diff != "" { + t.Errorf("Unexpected pods (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/delete/delete_job.go b/pkg/cmd/delete/delete_job.go new file mode 100644 index 0000000..5f0ca09 --- /dev/null +++ b/pkg/cmd/delete/delete_job.go @@ -0,0 +1,182 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/client-go/kubernetes/scheme" + batchv1 "k8s.io/client-go/kubernetes/typed/batch/v1" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + jobExample = templates.Examples(` + # Delete Job + kjobctl delete job my-application-profile-job-k2wzd + `) +) + +type JobOptions struct { + PrintFlags *genericclioptions.PrintFlags + + JobNames []string + Namespace string + + CascadeStrategy metav1.DeletionPropagation + DryRunStrategy util.DryRunStrategy + + Client batchv1.BatchV1Interface + + PrintObj printers.ResourcePrinterFunc + + genericiooptions.IOStreams +} + +func NewJobOptions(streams genericiooptions.IOStreams) *JobOptions { + return &JobOptions{ + PrintFlags: genericclioptions.NewPrintFlags("deleted").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + } +} + +func NewJobCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams) *cobra.Command { + o := NewJobOptions(streams) + + cmd := &cobra.Command{ + Use: "job NAME [--cascade STRATEGY] [--dry-run STRATEGY]", + DisableFlagsInUseLine: true, + Short: "Delete Job", + Example: jobExample, + Args: cobra.MinimumNArgs(1), + ValidArgsFunction: completion.JobNameFunc(clientGetter, v1alpha1.JobMode), + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + + err := o.Complete(clientGetter, cmd, args) + if err != nil { + return err + } + + return o.Run(cmd.Context()) + }, + } + + addCascadingFlag(cmd) + util.AddDryRunFlag(cmd) + + o.PrintFlags.AddFlags(cmd) + + return cmd +} + +func (o *JobOptions) Complete(clientGetter util.ClientGetter, cmd *cobra.Command, args []string) error { + o.JobNames = args + + var err error + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + o.DryRunStrategy, err = util.GetDryRunStrategy(cmd) + if err != nil { + return err + } + + err = util.PrintFlagsWithDryRunStrategy(o.PrintFlags, o.DryRunStrategy) + if err != nil { + return err + } + + o.CascadeStrategy, err = getCascadingStrategy(cmd) + if err != nil { + return err + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return err + } + + o.PrintObj = printer.PrintObj + + clientset, err := clientGetter.K8sClientset() + if err != nil { + return err + } + + o.Client = clientset.BatchV1() + + return nil +} + +func (o *JobOptions) Run(ctx context.Context) error { + for _, jobName := range o.JobNames { + job, err := o.Client.Jobs(o.Namespace).Get(ctx, jobName, metav1.GetOptions{}) + if client.IgnoreNotFound(err) != nil { + return err + } + if err != nil { + fmt.Fprintln(o.ErrOut, err) + continue + } + if _, ok := job.Labels[constants.ProfileLabel]; !ok { + fmt.Fprintf(o.ErrOut, "jobs.batch \"%s\" not created via kjob\n", job.Name) + continue + } + if job.Labels[constants.ModeLabel] != string(v1alpha1.JobMode) { + fmt.Fprintf(o.ErrOut, "jobs.batch \"%s\" created in \"%s\" mode. Switch to the correct mode to delete it\n", + job.Name, job.Labels[constants.ModeLabel]) + continue + } + + if o.DryRunStrategy != util.DryRunClient { + deleteOptions := metav1.DeleteOptions{ + PropagationPolicy: ptr.To(o.CascadeStrategy), + } + + if o.DryRunStrategy == util.DryRunServer { + deleteOptions.DryRun = []string{metav1.DryRunAll} + } + + if err := o.Client.Jobs(o.Namespace).Delete(ctx, jobName, deleteOptions); err != nil { + return err + } + } + + if err := o.PrintObj(job, o.Out); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/cmd/delete/delete_job_test.go b/pkg/cmd/delete/delete_job_test.go new file mode 100644 index 0000000..531dabb --- /dev/null +++ b/pkg/cmd/delete/delete_job_test.go @@ -0,0 +1,205 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "slices" + "testing" + + "github.com/google/go-cmp/cmp" + batchv1 "k8s.io/api/batch/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + k8sfake "k8s.io/client-go/kubernetes/fake" + kubetesting "k8s.io/client-go/testing" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestJobCmd(t *testing.T) { + testCases := map[string]struct { + ns string + args []string + objs []runtime.Object + wantJobs []batchv1.Job + wantOut string + wantOutErr string + wantErr string + }{ + "shouldn't delete job because it is not found": { + args: []string{"j"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantOutErr: "jobs.batch \"j\" not found\n", + }, + "shouldn't delete job because it is not created via kjob": { + args: []string{"j1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Obj(), + }, + wantOutErr: "jobs.batch \"j1\" not created via kjob\n", + }, + "shouldn't delete job because it is not used for Job mode": { + args: []string{"j1", "j2"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantOutErr: `jobs.batch "j1" created in "" mode. Switch to the correct mode to delete it +jobs.batch "j2" created in "Slurm" mode. Switch to the correct mode to delete it +`, + }, + "should delete job": { + args: []string{"j1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantOut: "job.batch/j1 deleted\n", + }, + "should delete jobs": { + args: []string{"j1", "j2"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantOut: "job.batch/j1 deleted\njob.batch/j2 deleted\n", + }, + "should delete only one job": { + args: []string{"j1", "j"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantOut: "job.batch/j1 deleted\n", + wantOutErr: "jobs.batch \"j\" not found\n", + }, + "shouldn't delete job with client dry run": { + args: []string{"j1", "--dry-run", "client"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantOut: "job.batch/j1 deleted (client dry run)\n", + }, + "shouldn't delete job with server dry run": { + args: []string{"j1", "--dry-run", "server"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantOut: "job.batch/j1 deleted (server dry run)\n", + }, + "no args": { + args: []string{}, + wantErr: "requires at least 1 arg(s), only received 0", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ns := metav1.NamespaceDefault + if tc.ns != "" { + ns = tc.ns + } + + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := k8sfake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("delete", "jobs", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + if slices.Contains(action.(kubetesting.DeleteAction).GetDeleteOptions().DryRun, metav1.DryRunAll) { + handled = true + } + return handled, ret, err + }) + + tcg := cmdtesting.NewTestClientGetter(). + WithK8sClientset(clientset). + WithNamespace(ns) + + cmd := NewJobCmd(tcg, streams) + cmd.SetOut(out) + cmd.SetErr(outErr) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + if gotErr != nil { + return + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected error output (-want/+got)\n%s", diff) + } + + gotJobList, err := clientset.BatchV1().Jobs(tc.ns).List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Error(err) + return + } + + if diff := cmp.Diff(tc.wantJobs, gotJobList.Items); diff != "" { + t.Errorf("Unexpected jobs (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/delete/delete_ray_cluster.go b/pkg/cmd/delete/delete_ray_cluster.go new file mode 100644 index 0000000..7d963ef --- /dev/null +++ b/pkg/cmd/delete/delete_ray_cluster.go @@ -0,0 +1,176 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "fmt" + + "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/scheme" + rayv1 "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/typed/ray/v1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + rayClusterExample = templates.Examples(` + # Delete RayCluster + kjobctl delete raycluster my-application-profile-raycluster-k2wzd + `) +) + +type RayClusterOptions struct { + PrintFlags *genericclioptions.PrintFlags + + RayClusterNames []string + Namespace string + + CascadeStrategy metav1.DeletionPropagation + DryRunStrategy util.DryRunStrategy + + Client rayv1.RayV1Interface + + PrintObj printers.ResourcePrinterFunc + + genericiooptions.IOStreams +} + +func NewRayClusterOptions(streams genericiooptions.IOStreams) *RayClusterOptions { + return &RayClusterOptions{ + PrintFlags: genericclioptions.NewPrintFlags("deleted").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + } +} + +func NewRayClusterCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams) *cobra.Command { + o := NewRayClusterOptions(streams) + + cmd := &cobra.Command{ + Use: "raycluster NAME [--cascade STRATEGY] [--dry-run STRATEGY]", + DisableFlagsInUseLine: true, + Short: "Delete RayCluster", + Example: rayClusterExample, + Args: cobra.MinimumNArgs(1), + ValidArgsFunction: completion.RayClusterNameFunc(clientGetter), + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + + err := o.Complete(clientGetter, cmd, args) + if err != nil { + return err + } + + return o.Run(cmd.Context()) + }, + } + + addCascadingFlag(cmd) + util.AddDryRunFlag(cmd) + + o.PrintFlags.AddFlags(cmd) + + return cmd +} + +func (o *RayClusterOptions) Complete(clientGetter util.ClientGetter, cmd *cobra.Command, args []string) error { + o.RayClusterNames = args + + var err error + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + o.DryRunStrategy, err = util.GetDryRunStrategy(cmd) + if err != nil { + return err + } + + err = util.PrintFlagsWithDryRunStrategy(o.PrintFlags, o.DryRunStrategy) + if err != nil { + return err + } + + o.CascadeStrategy, err = getCascadingStrategy(cmd) + if err != nil { + return err + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return err + } + + o.PrintObj = printer.PrintObj + + clientset, err := clientGetter.RayClientset() + if err != nil { + return err + } + + o.Client = clientset.RayV1() + + return nil +} + +func (o *RayClusterOptions) Run(ctx context.Context) error { + for _, rayClusterName := range o.RayClusterNames { + rayCluster, err := o.Client.RayClusters(o.Namespace).Get(ctx, rayClusterName, metav1.GetOptions{}) + if client.IgnoreNotFound(err) != nil { + return err + } + if err != nil { + fmt.Fprintln(o.ErrOut, err) + continue + } + if _, ok := rayCluster.Labels[constants.ProfileLabel]; !ok { + fmt.Fprintf(o.ErrOut, "rayclusters.ray.io \"%s\" not created via kjob\n", rayCluster.Name) + continue + } + + if o.DryRunStrategy != util.DryRunClient { + deleteOptions := metav1.DeleteOptions{ + PropagationPolicy: ptr.To(o.CascadeStrategy), + } + + if o.DryRunStrategy == util.DryRunServer { + deleteOptions.DryRun = []string{metav1.DryRunAll} + } + + if err := o.Client.RayClusters(o.Namespace).Delete(ctx, rayClusterName, deleteOptions); err != nil { + return err + } + } + + if err := o.PrintObj(rayCluster, o.Out); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/cmd/delete/delete_ray_cluster_test.go b/pkg/cmd/delete/delete_ray_cluster_test.go new file mode 100644 index 0000000..0dce881 --- /dev/null +++ b/pkg/cmd/delete/delete_ray_cluster_test.go @@ -0,0 +1,190 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "slices" + "testing" + + "github.com/google/go-cmp/cmp" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/fake" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + kubetesting "k8s.io/client-go/testing" + + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestRayClusterCmd(t *testing.T) { + testCases := map[string]struct { + ns string + args []string + objs []runtime.Object + wantRayClusters []rayv1.RayCluster + wantOut string + wantOutErr string + wantErr string + }{ + "shouldn't delete ray cluster because it is not found": { + args: []string{"rc"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayClusters: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOutErr: "rayclusters.ray.io \"rc\" not found\n", + }, + "shouldn't delete ray cluster because it is not created via kjob": { + args: []string{"rc1"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Obj(), + }, + wantRayClusters: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Obj(), + }, + wantOutErr: "rayclusters.ray.io \"rc1\" not created via kjob\n", + }, + "should delete ray cluster": { + args: []string{"rc1"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayClusters: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "raycluster.ray.io/rc1 deleted\n", + }, + "should delete ray clusters": { + args: []string{"rc1", "rc2"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "raycluster.ray.io/rc1 deleted\nraycluster.ray.io/rc2 deleted\n", + }, + "should delete only one ray cluster": { + args: []string{"rc1", "rc"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayClusters: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "raycluster.ray.io/rc1 deleted\n", + wantOutErr: "rayclusters.ray.io \"rc\" not found\n", + }, + "shouldn't delete ray cluster with client dry run": { + args: []string{"rc1", "--dry-run", "client"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayClusters: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "raycluster.ray.io/rc1 deleted (client dry run)\n", + }, + "shouldn't delete ray cluster with server dry run": { + args: []string{"rc1", "--dry-run", "server"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayClusters: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "raycluster.ray.io/rc1 deleted (server dry run)\n", + }, + "no args": { + args: []string{}, + wantErr: "requires at least 1 arg(s), only received 0", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ns := metav1.NamespaceDefault + if tc.ns != "" { + ns = tc.ns + } + + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := fake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("delete", "rayclusters", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + if slices.Contains(action.(kubetesting.DeleteAction).GetDeleteOptions().DryRun, metav1.DryRunAll) { + handled = true + } + return handled, ret, err + }) + + tcg := cmdtesting.NewTestClientGetter(). + WithRayClientset(clientset). + WithNamespace(ns) + + cmd := NewRayClusterCmd(tcg, streams) + cmd.SetOut(out) + cmd.SetErr(outErr) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + if gotErr != nil { + return + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected error output (-want/+got)\n%s", diff) + } + + gotRayClusterList, err := clientset.RayV1().RayClusters(tc.ns).List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Error(err) + return + } + + if diff := cmp.Diff(tc.wantRayClusters, gotRayClusterList.Items); diff != "" { + t.Errorf("Unexpected ray clusters (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/delete/delete_ray_job.go b/pkg/cmd/delete/delete_ray_job.go new file mode 100644 index 0000000..6dee8a7 --- /dev/null +++ b/pkg/cmd/delete/delete_ray_job.go @@ -0,0 +1,176 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "fmt" + + "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/scheme" + rayv1 "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/typed/ray/v1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + rayJobExample = templates.Examples(` + # Delete RayJob + kjobctl delete rayjob my-application-profile-rayjob-k2wzd + `) +) + +type RayJobOptions struct { + PrintFlags *genericclioptions.PrintFlags + + RayJobNames []string + Namespace string + + CascadeStrategy metav1.DeletionPropagation + DryRunStrategy util.DryRunStrategy + + Client rayv1.RayV1Interface + + PrintObj printers.ResourcePrinterFunc + + genericiooptions.IOStreams +} + +func NewRayJobOptions(streams genericiooptions.IOStreams) *RayJobOptions { + return &RayJobOptions{ + PrintFlags: genericclioptions.NewPrintFlags("deleted").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + } +} + +func NewRayJobCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams) *cobra.Command { + o := NewRayJobOptions(streams) + + cmd := &cobra.Command{ + Use: "rayjob NAME [--cascade STRATEGY] [--dry-run STRATEGY]", + DisableFlagsInUseLine: true, + Short: "Delete RayJob", + Example: rayJobExample, + Args: cobra.MinimumNArgs(1), + ValidArgsFunction: completion.RayJobNameFunc(clientGetter), + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + + err := o.Complete(clientGetter, cmd, args) + if err != nil { + return err + } + + return o.Run(cmd.Context()) + }, + } + + addCascadingFlag(cmd) + util.AddDryRunFlag(cmd) + + o.PrintFlags.AddFlags(cmd) + + return cmd +} + +func (o *RayJobOptions) Complete(clientGetter util.ClientGetter, cmd *cobra.Command, args []string) error { + o.RayJobNames = args + + var err error + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + o.DryRunStrategy, err = util.GetDryRunStrategy(cmd) + if err != nil { + return err + } + + err = util.PrintFlagsWithDryRunStrategy(o.PrintFlags, o.DryRunStrategy) + if err != nil { + return err + } + + o.CascadeStrategy, err = getCascadingStrategy(cmd) + if err != nil { + return err + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return err + } + + o.PrintObj = printer.PrintObj + + clientset, err := clientGetter.RayClientset() + if err != nil { + return err + } + + o.Client = clientset.RayV1() + + return nil +} + +func (o *RayJobOptions) Run(ctx context.Context) error { + for _, rayJobName := range o.RayJobNames { + rayJob, err := o.Client.RayJobs(o.Namespace).Get(ctx, rayJobName, metav1.GetOptions{}) + if client.IgnoreNotFound(err) != nil { + return err + } + if err != nil { + fmt.Fprintln(o.ErrOut, err) + continue + } + if _, ok := rayJob.Labels[constants.ProfileLabel]; !ok { + fmt.Fprintf(o.ErrOut, "rayjobs.ray.io \"%s\" not created via kjob\n", rayJob.Name) + continue + } + + if o.DryRunStrategy != util.DryRunClient { + deleteOptions := metav1.DeleteOptions{ + PropagationPolicy: ptr.To(o.CascadeStrategy), + } + + if o.DryRunStrategy == util.DryRunServer { + deleteOptions.DryRun = []string{metav1.DryRunAll} + } + + if err := o.Client.RayJobs(o.Namespace).Delete(ctx, rayJobName, deleteOptions); err != nil { + return err + } + } + + if err := o.PrintObj(rayJob, o.Out); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/cmd/delete/delete_ray_job_test.go b/pkg/cmd/delete/delete_ray_job_test.go new file mode 100644 index 0000000..018d5ac --- /dev/null +++ b/pkg/cmd/delete/delete_ray_job_test.go @@ -0,0 +1,190 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "slices" + "testing" + + "github.com/google/go-cmp/cmp" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/fake" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + kubetesting "k8s.io/client-go/testing" + + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestRayJobCmd(t *testing.T) { + testCases := map[string]struct { + ns string + args []string + objs []runtime.Object + wantRayJobs []rayv1.RayJob + wantOut string + wantOutErr string + wantErr string + }{ + "shouldn't delete ray job because it is not found": { + args: []string{"rj"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayJobs: []rayv1.RayJob{ + *wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOutErr: "rayjobs.ray.io \"rj\" not found\n", + }, + "shouldn't delete ray job because it is not created via kjob": { + args: []string{"rj1"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Obj(), + }, + wantRayJobs: []rayv1.RayJob{ + *wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Obj(), + }, + wantOutErr: "rayjobs.ray.io \"rj1\" not created via kjob\n", + }, + "should delete ray job": { + args: []string{"rj1"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayJobs: []rayv1.RayJob{ + *wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "rayjob.ray.io/rj1 deleted\n", + }, + "should delete ray jobs": { + args: []string{"rj1", "rj2"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "rayjob.ray.io/rj1 deleted\nrayjob.ray.io/rj2 deleted\n", + }, + "should delete only one ray job": { + args: []string{"rj1", "rj"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayJobs: []rayv1.RayJob{ + *wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "rayjob.ray.io/rj1 deleted\n", + wantOutErr: "rayjobs.ray.io \"rj\" not found\n", + }, + "shouldn't delete ray job with client dry run": { + args: []string{"rj1", "--dry-run", "client"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayJobs: []rayv1.RayJob{ + *wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "rayjob.ray.io/rj1 deleted (client dry run)\n", + }, + "shouldn't delete ray job with server dry run": { + args: []string{"rj1", "--dry-run", "server"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayJobs: []rayv1.RayJob{ + *wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "rayjob.ray.io/rj1 deleted (server dry run)\n", + }, + "no args": { + args: []string{}, + wantErr: "requires at least 1 arg(s), only received 0", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ns := metav1.NamespaceDefault + if tc.ns != "" { + ns = tc.ns + } + + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := fake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("delete", "rayjobs", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + if slices.Contains(action.(kubetesting.DeleteAction).GetDeleteOptions().DryRun, metav1.DryRunAll) { + handled = true + } + return handled, ret, err + }) + + tcg := cmdtesting.NewTestClientGetter(). + WithRayClientset(clientset). + WithNamespace(ns) + + cmd := NewRayJobCmd(tcg, streams) + cmd.SetOut(out) + cmd.SetErr(outErr) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + if gotErr != nil { + return + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected error output (-want/+got)\n%s", diff) + } + + gotRayJobList, err := clientset.RayV1().RayJobs(tc.ns).List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Error(err) + return + } + + if diff := cmp.Diff(tc.wantRayJobs, gotRayJobList.Items); diff != "" { + t.Errorf("Unexpected ray jobs (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/delete/delete_slurm.go b/pkg/cmd/delete/delete_slurm.go new file mode 100644 index 0000000..4da4896 --- /dev/null +++ b/pkg/cmd/delete/delete_slurm.go @@ -0,0 +1,184 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + slurmExample = templates.Examples(` + # Delete Slurm + kjobctl delete slurm my-application-profile-slurm-k2wzd + `) +) + +type SlurmOptions struct { + PrintFlags *genericclioptions.PrintFlags + + JobNames []string + Namespace string + + CascadeStrategy metav1.DeletionPropagation + DryRunStrategy util.DryRunStrategy + + Clientset k8s.Interface + + PrintObj printers.ResourcePrinterFunc + + genericiooptions.IOStreams +} + +func NewSlurmOptions(streams genericiooptions.IOStreams) *SlurmOptions { + return &SlurmOptions{ + PrintFlags: genericclioptions.NewPrintFlags("deleted").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + } +} + +func NewSlurmCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams) *cobra.Command { + o := NewSlurmOptions(streams) + + cmd := &cobra.Command{ + Use: "slurm NAME [--cascade STRATEGY] [--dry-run STRATEGY]", + DisableFlagsInUseLine: true, + Short: "Delete Slurm", + Example: slurmExample, + Args: cobra.MinimumNArgs(1), + ValidArgsFunction: completion.JobNameFunc(clientGetter, v1alpha1.SlurmMode), + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + + err := o.Complete(clientGetter, cmd, args) + if err != nil { + return err + } + + return o.Run(cmd.Context()) + }, + } + + addCascadingFlag(cmd) + util.AddDryRunFlag(cmd) + + o.PrintFlags.AddFlags(cmd) + + return cmd +} + +func (o *SlurmOptions) Complete(clientGetter util.ClientGetter, cmd *cobra.Command, args []string) error { + o.JobNames = args + + var err error + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + o.DryRunStrategy, err = util.GetDryRunStrategy(cmd) + if err != nil { + return err + } + + err = util.PrintFlagsWithDryRunStrategy(o.PrintFlags, o.DryRunStrategy) + if err != nil { + return err + } + + o.CascadeStrategy, err = getCascadingStrategy(cmd) + if err != nil { + return err + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return err + } + + o.PrintObj = printer.PrintObj + + o.Clientset, err = clientGetter.K8sClientset() + if err != nil { + return err + } + + return nil +} + +func (o *SlurmOptions) Run(ctx context.Context) error { + for _, jobName := range o.JobNames { + if err := o.deleteJob(ctx, jobName); err != nil { + return err + } + } + + return nil +} + +func (o *SlurmOptions) deleteJob(ctx context.Context, jobName string) error { + job, err := o.Clientset.BatchV1().Jobs(o.Namespace).Get(ctx, jobName, metav1.GetOptions{}) + if client.IgnoreNotFound(err) != nil { + return err + } + if err != nil { + fmt.Fprintln(o.ErrOut, err) + return nil + } + if _, ok := job.Labels[constants.ProfileLabel]; !ok { + fmt.Fprintf(o.ErrOut, "jobs.batch \"%s\" not created via kjob\n", job.Name) + return nil + } + if job.Labels[constants.ModeLabel] != string(v1alpha1.SlurmMode) { + fmt.Fprintf(o.ErrOut, "jobs.batch \"%s\" created in \"%s\" mode. Switch to the correct mode to delete it\n", + job.Name, job.Labels[constants.ModeLabel]) + return nil + } + + if o.DryRunStrategy != util.DryRunClient { + deleteOptions := metav1.DeleteOptions{ + PropagationPolicy: ptr.To(o.CascadeStrategy), + } + + if o.DryRunStrategy == util.DryRunServer { + deleteOptions.DryRun = []string{metav1.DryRunAll} + } + + if err := o.Clientset.BatchV1().Jobs(o.Namespace).Delete(ctx, jobName, deleteOptions); err != nil { + return err + } + } + + return o.PrintObj(job, o.Out) +} diff --git a/pkg/cmd/delete/delete_slurm_test.go b/pkg/cmd/delete/delete_slurm_test.go new file mode 100644 index 0000000..5d711cb --- /dev/null +++ b/pkg/cmd/delete/delete_slurm_test.go @@ -0,0 +1,205 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "slices" + "testing" + + "github.com/google/go-cmp/cmp" + batchv1 "k8s.io/api/batch/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + k8sfake "k8s.io/client-go/kubernetes/fake" + kubetesting "k8s.io/client-go/testing" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestSlurmCmd(t *testing.T) { + testCases := map[string]struct { + ns string + args []string + objs []runtime.Object + wantJobs []batchv1.Job + wantOut string + wantOutErr string + wantErr string + }{ + "shouldn't delete slurm job because it is not found": { + args: []string{"j"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantOutErr: "jobs.batch \"j\" not found\n", + }, + "shouldn't delete slurm job because it is not created via kjob": { + args: []string{"j1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Obj(), + }, + wantOutErr: "jobs.batch \"j1\" not created via kjob\n", + }, + "shouldn't delete slurm job because it is not used for Job mode": { + args: []string{"j1", "j2"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + }, + wantOutErr: `jobs.batch "j1" created in "" mode. Switch to the correct mode to delete it +jobs.batch "j2" created in "Job" mode. Switch to the correct mode to delete it +`, + }, + "should delete slurm job": { + args: []string{"j1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantOut: "job.batch/j1 deleted\n", + }, + "should delete slurm jobs": { + args: []string{"j1", "j2"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantOut: "job.batch/j1 deleted\njob.batch/j2 deleted\n", + }, + "should delete only one slurm job": { + args: []string{"j1", "j"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantOut: "job.batch/j1 deleted\n", + wantOutErr: "jobs.batch \"j\" not found\n", + }, + "shouldn't delete slurm job with client dry run": { + args: []string{"j1", "--dry-run", "client"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantOut: "job.batch/j1 deleted (client dry run)\n", + }, + "shouldn't delete slurm job with server dry run": { + args: []string{"j1", "--dry-run", "server"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantOut: "job.batch/j1 deleted (server dry run)\n", + }, + "no args": { + args: []string{}, + wantErr: "requires at least 1 arg(s), only received 0", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ns := metav1.NamespaceDefault + if tc.ns != "" { + ns = tc.ns + } + + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := k8sfake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("delete", "jobs", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + if slices.Contains(action.(kubetesting.DeleteAction).GetDeleteOptions().DryRun, metav1.DryRunAll) { + handled = true + } + return handled, ret, err + }) + + tcg := cmdtesting.NewTestClientGetter(). + WithK8sClientset(clientset). + WithNamespace(ns) + + cmd := NewSlurmCmd(tcg, streams) + cmd.SetOut(out) + cmd.SetErr(outErr) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + if gotErr != nil { + return + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected error output (-want/+got)\n%s", diff) + } + + gotJobList, err := clientset.BatchV1().Jobs(tc.ns).List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Error(err) + return + } + + if diff := cmp.Diff(tc.wantJobs, gotJobList.Items); diff != "" { + t.Errorf("Unexpected jobs (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/delete/helpers.go b/pkg/cmd/delete/helpers.go new file mode 100644 index 0000000..b7b93d6 --- /dev/null +++ b/pkg/cmd/delete/helpers.go @@ -0,0 +1,52 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "fmt" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" +) + +const ( + cascadingFlagName = "cascade" +) + +func addCascadingFlag(cmd *cobra.Command) { + cmd.PersistentFlags().String( + cascadingFlagName, + "background", + `Must be "background", "orphan", or "foreground". Defaults to background.`, + ) +} + +func getCascadingStrategy(cmd *cobra.Command) (metav1.DeletionPropagation, error) { + cascadingFlag := util.FlagString(cmd, cascadingFlagName) + switch cascadingFlag { + case "orphan": + return metav1.DeletePropagationOrphan, nil + case "foreground": + return metav1.DeletePropagationForeground, nil + case "background": + return metav1.DeletePropagationBackground, nil + default: + return metav1.DeletePropagationBackground, fmt.Errorf(`invalid cascade value (%v). Must be "background", "foreground", or "orphan"`, cascadingFlag) + } +} diff --git a/pkg/cmd/describe/describe.go b/pkg/cmd/describe/describe.go new file mode 100644 index 0000000..476cb85 --- /dev/null +++ b/pkg/cmd/describe/describe.go @@ -0,0 +1,352 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package describe + +import ( + "context" + "fmt" + "slices" + "strings" + + "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/resource" + "k8s.io/client-go/kubernetes" + "k8s.io/kubectl/pkg/util/templates" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + jobExample = templates.Examples(` + # Describe a task with job mode + kjobctl describe job sample-job + + # Describe a task with job mode + kjobctl describe job/sample-job + + # Describe all tasks with job mode + kjobctl describe job + + # Describe tasks by label name=myLabel + kjobctl describe job -l name=myLabel + `) +) + +const ( + modeTaskArgsFormat = iota + modeSlashTaskArgsFormat + modeArgsFormat +) + +type DescribeOptions struct { + AllNamespaces bool + Namespace string + ProfileName string + ModeName string + TaskName string + LabelSelector string + + UserSpecifiedTask []string + + argsFormat int + ResourceGVK schema.GroupVersionKind + ResourceBuilder *resource.Builder + + Clientset kubernetes.Interface + + genericiooptions.IOStreams +} + +func NewDescribeOptions(streams genericiooptions.IOStreams) *DescribeOptions { + return &DescribeOptions{ + IOStreams: streams, + } +} + +func NewDescribeCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams) *cobra.Command { + o := NewDescribeOptions(streams) + + cmd := &cobra.Command{ + Use: "describe MODE NAME", + DisableFlagsInUseLine: true, + Short: "Show details of a specific resource or group of resources.", + Example: jobExample, + Args: cobra.MatchAll(cobra.RangeArgs(1, 2), cobra.OnlyValidArgs), + RunE: func(cmd *cobra.Command, args []string) error { + // stop the usage if we've got this far + cmd.SilenceUsage = true + + err := o.Complete(clientGetter, args) + if err != nil { + return err + } + + return o.Run(cmd.Context()) + }, + } + + util.AddAllNamespacesFlagVar(cmd, &o.AllNamespaces) + util.AddLabelSelectorFlagVar(cmd, &o.LabelSelector) + util.AddProfileFlagVar(cmd, &o.ProfileName) + + return cmd +} + +func (o *DescribeOptions) Complete(clientGetter util.ClientGetter, args []string) error { + var err error + + o.findArgsFormat(args) + + err = o.parseArgs(args) + if err != nil { + return err + } + + resource := resourceFor(o.ModeName) + mapper, err := clientGetter.ToRESTMapper() + if err != nil { + return err + } + o.ResourceGVK, err = gvkFor(mapper, resource) + if err != nil { + return err + } + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + o.ResourceBuilder = clientGetter.NewResourceBuilder() + + o.Clientset, err = clientGetter.K8sClientset() + if err != nil { + return err + } + + return nil +} + +func (o *DescribeOptions) findArgsFormat(args []string) { + if len(args) == 2 { + o.argsFormat = modeTaskArgsFormat + } else { + if strings.Contains(args[0], "/") { + o.argsFormat = modeSlashTaskArgsFormat + } else { + o.argsFormat = modeArgsFormat + } + } +} + +func (o *DescribeOptions) parseArgs(args []string) error { + switch o.argsFormat { + case modeTaskArgsFormat: + o.ModeName, o.TaskName = args[0], args[1] + case modeSlashTaskArgsFormat: + parsedArg, err := parseAppProfileModeName(args[0]) + if err != nil { + return err + } + o.ModeName, o.TaskName = parsedArg[0], parsedArg[1] + default: + o.ModeName = args[0] + } + + return nil +} + +func (o *DescribeOptions) Run(ctx context.Context) error { + builder := o.customizeResourceBuilder() + + r := builder.Do() + if err := r.Err(); err != nil { + return err + } + infos, err := r.Infos() + if err != nil { + return err + } + + if strings.EqualFold(o.ModeName, string(v1alpha1.SlurmMode)) { + configMapsAsInfos, err := o.getConfigMaps(ctx) + if err != nil { + return err + } + + infos = append(infos, configMapsAsInfos...) + } + + // In Slurm mode, configMaps are created with the same name as the jobs they are mounted on. + // Since the first half of the slice contains jobs and the second half contains configMaps, + // sorting objects by name will interleave jobs with their corresponding configMap. + slices.SortFunc(infos, func(a, b *resource.Info) int { + return strings.Compare(a.Name, b.Name) + }) + + allErrs := []error{} + errs := sets.NewString() + first := true + for _, info := range infos { + obj, ok := info.Object.(*unstructured.Unstructured) + if !ok { + err := fmt.Errorf("invalid object %+v. Unexpected type %T", obj, info.Object) + if errs.Has(err.Error()) { + continue + } + allErrs = append(allErrs, err) + errs.Insert(err.Error()) + continue + } + + labels := obj.GetLabels() + if _, ok := labels[constants.ProfileLabel]; !ok { + continue + } + + mapping := info.ResourceMapping() + describer, err := NewResourceDescriber(mapping) + if err != nil { + if errs.Has(err.Error()) { + continue + } + allErrs = append(allErrs, err) + errs.Insert(err.Error()) + continue + } + + output, err := describer.Describe(obj) + if err != nil { + if errs.Has(err.Error()) { + continue + } + allErrs = append(allErrs, err) + errs.Insert(err.Error()) + continue + } + + if first { + first = false + fmt.Fprint(o.Out, output) + } else { + fmt.Fprintf(o.Out, "\n\n%s", output) + } + } + + if first && len(allErrs) == 0 { + if o.AllNamespaces { + fmt.Fprintln(o.ErrOut, "No resources found") + } else { + fmt.Fprintf(o.ErrOut, "No resources found in %s namespace.\n", o.Namespace) + } + } + + return errors.NewAggregate(allErrs) +} + +func (o *DescribeOptions) customizeResourceBuilder() *resource.Builder { + builder := o.ResourceBuilder. + Unstructured(). + NamespaceParam(o.Namespace). + DefaultNamespace() + switch o.argsFormat { + case modeTaskArgsFormat, modeSlashTaskArgsFormat: + builder = builder.ResourceTypeOrNameArgs(true, o.ResourceGVK.Kind, o.TaskName). + SingleResourceType() + default: + selector := constants.ProfileLabel + if o.ProfileName != "" { + selector += fmt.Sprintf("=%s", o.ProfileName) + } + if o.LabelSelector != "" { + selector += fmt.Sprintf(",%s", o.LabelSelector) + } + + builder = builder.AllNamespaces(o.AllNamespaces). + LabelSelectorParam(selector). + ResourceTypeOrNameArgs(true, o.ResourceGVK.Kind). + ContinueOnError() + } + + return builder.Flatten() +} + +func (o *DescribeOptions) getConfigMaps(ctx context.Context) ([]*resource.Info, error) { + infos := make([]*resource.Info, 0) + + if o.argsFormat == modeTaskArgsFormat || o.argsFormat == modeSlashTaskArgsFormat { + cm, err := o.Clientset.CoreV1().ConfigMaps(o.Namespace).Get(ctx, o.TaskName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + info, err := configMapToInfo(cm) + if err != nil { + return nil, err + } + + infos = append(infos, info) + } else { + cmList, err := o.Clientset.CoreV1().ConfigMaps(o.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s,%s=%s", constants.ProfileLabel, constants.ModeLabel, v1alpha1.SlurmMode), + }) + if err != nil { + return nil, err + } + + for _, cm := range cmList.Items { + info, err := configMapToInfo(&cm) + if err != nil { + return nil, err + } + + infos = append(infos, info) + } + } + + return infos, nil +} + +func configMapToInfo(cm *corev1.ConfigMap) (*resource.Info, error) { + u, err := runtime.DefaultUnstructuredConverter.ToUnstructured(cm) + if err != nil { + return nil, err + } + + obj := &unstructured.Unstructured{Object: u} + return &resource.Info{ + Mapping: &meta.RESTMapping{ + GroupVersionKind: schema.GroupVersionKind{ + Version: "v1", + Kind: "ConfigMap", + }, + }, + Name: obj.GetName(), + Object: obj, + }, nil +} diff --git a/pkg/cmd/describe/describe_printer.go b/pkg/cmd/describe/describe_printer.go new file mode 100644 index 0000000..c8c3ba0 --- /dev/null +++ b/pkg/cmd/describe/describe_printer.go @@ -0,0 +1,335 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package describe + +import ( + "fmt" + "io" + "strconv" + "time" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/duration" + describehelper "k8s.io/kubectl/pkg/describe" + "k8s.io/utils/ptr" + utilmaps "sigs.k8s.io/kueue/pkg/util/maps" +) + +// ResourceDescriber generates output for the named resource or an error +// if the output could not be generated. Implementers typically +// abstract the retrieval of the named object from a remote server. +type ResourceDescriber interface { + Describe(object *unstructured.Unstructured) (output string, err error) +} + +// NewResourceDescriber returns a Describer for displaying the specified RESTMapping type or an error. +func NewResourceDescriber(mapping *meta.RESTMapping) (ResourceDescriber, error) { + if describer, ok := DescriberFor(mapping.GroupVersionKind.GroupKind()); ok { + return describer, nil + } + + return nil, fmt.Errorf("no description has been implemented for %s", mapping.GroupVersionKind.String()) +} + +// DescriberFor returns the default describe functions for each of the standard +// Kubernetes types. +func DescriberFor(kind schema.GroupKind) (ResourceDescriber, bool) { + describers := map[schema.GroupKind]ResourceDescriber{ + {Group: batchv1.GroupName, Kind: "Job"}: &JobDescriber{}, + {Group: corev1.GroupName, Kind: "Pod"}: &PodDescriber{}, + {Group: rayv1.GroupVersion.Group, Kind: "RayJob"}: &RayJobDescriber{}, + {Group: rayv1.GroupVersion.Group, Kind: "RayCluster"}: &RayClusterDescriber{}, + {Group: corev1.GroupName, Kind: "ConfigMap"}: &ConfigMapDescriber{}, + } + + f, ok := describers[kind] + return f, ok +} + +// JobDescriber generates information about a job and the pods it has created. +type JobDescriber struct{} + +func (d *JobDescriber) Describe(object *unstructured.Unstructured) (string, error) { + job := &batchv1.Job{} + err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), job) + if err != nil { + return "", err + } + + return describeJob(job) +} + +func describeJob(job *batchv1.Job) (string, error) { + return tabbedString(func(out io.Writer) error { + w := describehelper.NewPrefixWriter(out) + + w.Write(IndentLevelZero, "Name:\t%s\n", job.Name) + w.Write(IndentLevelZero, "Namespace:\t%s\n", job.Namespace) + printLabelsMultiline(w, "Labels", job.Labels) + printLabelsMultiline(w, "Annotations", job.Annotations) + if job.Spec.Parallelism != nil { + w.Write(IndentLevelZero, "Parallelism:\t%d\n", *job.Spec.Parallelism) + } + if job.Spec.Completions != nil { + w.Write(IndentLevelZero, "Completions:\t%d\n", *job.Spec.Completions) + } else { + w.Write(IndentLevelZero, "Completions:\t\n") + } + if job.Status.StartTime != nil { + w.Write(IndentLevelZero, "Start Time:\t%s\n", job.Status.StartTime.Time.Format(time.RFC1123Z)) + } + if job.Status.CompletionTime != nil { + w.Write(IndentLevelZero, "Completed At:\t%s\n", job.Status.CompletionTime.Time.Format(time.RFC1123Z)) + } + if job.Status.StartTime != nil && job.Status.CompletionTime != nil { + w.Write(IndentLevelZero, "Duration:\t%s\n", duration.HumanDuration(job.Status.CompletionTime.Sub(job.Status.StartTime.Time))) + } + if job.Status.Ready == nil { + w.Write(IndentLevelZero, "Pods Statuses:\t%d Active / %d Succeeded / %d Failed\n", job.Status.Active, job.Status.Succeeded, job.Status.Failed) + } else { + w.Write(IndentLevelZero, "Pods Statuses:\t%d Active (%d Ready) / %d Succeeded / %d Failed\n", job.Status.Active, *job.Status.Ready, job.Status.Succeeded, job.Status.Failed) + } + + describePodTemplate(&job.Spec.Template, w) + + return nil + }) +} + +// PodDescriber generates information about a pod. +type PodDescriber struct{} + +func (d *PodDescriber) Describe(object *unstructured.Unstructured) (string, error) { + pod := &corev1.Pod{} + err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), pod) + if err != nil { + return "", err + } + + return describePod(pod) +} + +func describePod(pod *corev1.Pod) (string, error) { + return tabbedString(func(out io.Writer) error { + w := describehelper.NewPrefixWriter(out) + + w.Write(IndentLevelZero, "Name:\t%s\n", pod.Name) + w.Write(IndentLevelZero, "Namespace:\t%s\n", pod.Namespace) + if pod.Status.StartTime != nil { + w.Write(IndentLevelZero, "Start Time:\t%s\n", pod.Status.StartTime.Time.Format(time.RFC1123Z)) + } + printLabelsMultiline(w, "Labels", pod.Labels) + + if pod.DeletionTimestamp != nil && pod.Status.Phase != corev1.PodFailed && pod.Status.Phase != corev1.PodSucceeded { + w.Write(IndentLevelZero, "Status:\tTerminating (lasts %s)\n", translateTimestampSince(*pod.DeletionTimestamp)) + w.Write(IndentLevelZero, "Termination Grace Period:\t%ds\n", *pod.DeletionGracePeriodSeconds) + } else { + w.Write(IndentLevelZero, "Status:\t%s\n", string(pod.Status.Phase)) + } + if len(pod.Status.Reason) > 0 { + w.Write(IndentLevelZero, "Reason:\t%s\n", pod.Status.Reason) + } + if len(pod.Status.Message) > 0 { + w.Write(IndentLevelZero, "Message:\t%s\n", pod.Status.Message) + } + + if len(pod.Spec.InitContainers) > 0 { + describeContainers("Init Containers", pod.Spec.InitContainers, w, "") + } + describeContainers("Containers", pod.Spec.Containers, w, "") + if len(pod.Spec.EphemeralContainers) > 0 { + var ec []corev1.Container + for i := range pod.Spec.EphemeralContainers { + ec = append(ec, corev1.Container(pod.Spec.EphemeralContainers[i].EphemeralContainerCommon)) + } + describeContainers("Ephemeral Containers", ec, w, "") + } + + describeVolumes(pod.Spec.Volumes, w, "") + + return nil + }) +} + +// RayJobDescriber generates information about a ray job. +type RayJobDescriber struct{} + +func (d *RayJobDescriber) Describe(object *unstructured.Unstructured) (string, error) { + rayJob := &rayv1.RayJob{} + err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), rayJob) + if err != nil { + return "", err + } + + return describeRayJob(rayJob) +} + +func describeRayJob(rayJob *rayv1.RayJob) (string, error) { + return tabbedString(func(out io.Writer) error { + w := describehelper.NewPrefixWriter(out) + + w.Write(IndentLevelZero, "Name:\t%s\n", rayJob.Name) + w.Write(IndentLevelZero, "Namespace:\t%s\n", rayJob.Namespace) + if rayJob.Status.StartTime != nil { + w.Write(IndentLevelZero, "Start Time:\t%s\n", rayJob.Status.StartTime.Format(time.RFC1123Z)) + } + if rayJob.Status.EndTime != nil { + w.Write(IndentLevelZero, "End Time:\t%s\n", rayJob.Status.EndTime.Format(time.RFC1123Z)) + } + printLabelsMultiline(w, "Labels", rayJob.Labels) + + if rayJob.DeletionTimestamp != nil { + w.Write(IndentLevelZero, "Job Deployment Status:\tTerminating (lasts %s)\n", translateTimestampSince(*rayJob.DeletionTimestamp)) + w.Write(IndentLevelZero, "Job Status:\tTerminating (lasts %s)\n", translateTimestampSince(*rayJob.DeletionTimestamp)) + w.Write(IndentLevelZero, "Termination Grace Period:\t%ds\n", *rayJob.DeletionGracePeriodSeconds) + } else { + if len(rayJob.Status.JobDeploymentStatus) > 0 { + w.Write(IndentLevelZero, "Job Deployment Status:\t%s\n", string(rayJob.Status.JobDeploymentStatus)) + } + if len(rayJob.Status.JobStatus) > 0 { + w.Write(IndentLevelZero, "Job Status:\t%s\n", string(rayJob.Status.JobStatus)) + } + if len(rayJob.Status.Reason) > 0 { + w.Write(IndentLevelZero, "Reason:\t%s\n", rayJob.Status.Reason) + } + if len(rayJob.Status.Message) > 0 { + w.Write(IndentLevelZero, "Message:\t%s\n", rayJob.Status.Message) + } + } + + if len(rayJob.Status.RayClusterName) > 0 { + w.Write(IndentLevelZero, "Ray Cluster Name:\t%s\n", rayJob.Status.RayClusterName) + } + + w.Write(IndentLevelZero, "Ray Cluster Status:\n") + w.Write(IndentLevelOne, "Desired CPU:\t%s\n", rayJob.Status.RayClusterStatus.DesiredCPU.String()) + w.Write(IndentLevelOne, "Desired GPU:\t%s\n", rayJob.Status.RayClusterStatus.DesiredGPU.String()) + w.Write(IndentLevelOne, "Desired Memory:\t%s\n", rayJob.Status.RayClusterStatus.DesiredMemory.String()) + w.Write(IndentLevelOne, "Desired TPU:\t%s\n", rayJob.Status.RayClusterStatus.DesiredTPU.String()) + + return nil + }) +} + +// RayClusterDescriber generates information about a ray cluster. +type RayClusterDescriber struct{} + +func (d *RayClusterDescriber) Describe(object *unstructured.Unstructured) (string, error) { + rayCluster := &rayv1.RayCluster{} + err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), rayCluster) + if err != nil { + return "", err + } + + return describeRayCluster(rayCluster) +} + +func describeRayCluster(rayCluster *rayv1.RayCluster) (string, error) { + return tabbedString(func(out io.Writer) error { + w := describehelper.NewPrefixWriter(out) + + w.Write(IndentLevelZero, "Name:\t%s\n", rayCluster.Name) + w.Write(IndentLevelZero, "Namespace:\t%s\n", rayCluster.Namespace) + + printLabelsMultiline(w, "Labels", rayCluster.Labels) + + w.Write(IndentLevelZero, "Suspend:\t%t\n", ptr.Deref(rayCluster.Spec.Suspend, false)) + w.Write(IndentLevelZero, "State:\t%s\n", string(rayCluster.Status.State)) + if len(rayCluster.Status.Reason) > 0 { + w.Write(IndentLevelZero, "Reason:\t%s\n", rayCluster.Status.Reason) + } + + w.Write(IndentLevelZero, "Desired CPU:\t%s\n", rayCluster.Status.DesiredCPU.String()) + w.Write(IndentLevelZero, "Desired GPU:\t%s\n", rayCluster.Status.DesiredGPU.String()) + w.Write(IndentLevelZero, "Desired Memory:\t%s\n", rayCluster.Status.DesiredMemory.String()) + w.Write(IndentLevelZero, "Desired TPU:\t%s\n", rayCluster.Status.DesiredTPU.String()) + + w.Write(IndentLevelZero, "Ready Worker Replicas:\t%d\n", rayCluster.Status.ReadyWorkerReplicas) + w.Write(IndentLevelZero, "Available Worker Replicas:\t%d\n", rayCluster.Status.AvailableWorkerReplicas) + w.Write(IndentLevelZero, "Desired Worker Replicas:\t%d\n", rayCluster.Status.DesiredWorkerReplicas) + w.Write(IndentLevelZero, "Min Worker Replicas:\t%d\n", rayCluster.Status.MinWorkerReplicas) + w.Write(IndentLevelZero, "Max Worker Replicas:\t%d\n", rayCluster.Status.MaxWorkerReplicas) + + w.Write(IndentLevelZero, "Head Group:\n") + headGroupWriter := describehelper.NewNestedPrefixWriter(w, 1) + printLabelsMultiline(headGroupWriter, "Start Params", rayCluster.Spec.HeadGroupSpec.RayStartParams) + describePodTemplate(&rayCluster.Spec.HeadGroupSpec.Template, headGroupWriter) + + w.Write(IndentLevelZero, "Worker Groups:\n") + for _, wg := range rayCluster.Spec.WorkerGroupSpecs { + w.Write(IndentLevelOne, fmt.Sprintf("%s:\n", wg.GroupName)) + workerGroupWriter := describehelper.NewNestedPrefixWriter(w, 2) + if wg.Replicas != nil { + workerGroupWriter.Write(IndentLevelZero, "Replicas:\t%d\n", *wg.Replicas) + } + if wg.MinReplicas != nil { + workerGroupWriter.Write(IndentLevelZero, "Min Replicas:\t%d\n", *wg.MinReplicas) + } + if wg.MaxReplicas != nil { + workerGroupWriter.Write(IndentLevelZero, "Max Replicas:\t%d\n", *wg.MaxReplicas) + } + printLabelsMultiline(workerGroupWriter, "Start Params", wg.RayStartParams) + describePodTemplate(&wg.Template, workerGroupWriter) + } + + return nil + }) +} + +// ConfigMapDescriber generates information about a configMap. +type ConfigMapDescriber struct{} + +func (d *ConfigMapDescriber) Describe(object *unstructured.Unstructured) (string, error) { + configMap := &corev1.ConfigMap{} + err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), configMap) + if err != nil { + return "", err + } + + return describeConfigMap(configMap) +} + +func describeConfigMap(configMap *corev1.ConfigMap) (string, error) { + return tabbedString(func(out io.Writer) error { + w := describehelper.NewPrefixWriter(out) + + w.Write(IndentLevelZero, "Name:\t%s\n", configMap.Name) + w.Write(IndentLevelZero, "Namespace:\t%s\n", configMap.Namespace) + printLabelsMultiline(w, "Labels", configMap.Labels) + + w.Write(IndentLevelZero, "\nData\n====\n") + for _, k := range utilmaps.SortedKeys(configMap.Data) { + w.Write(IndentLevelZero, "%s:\n----\n", k) + w.Write(IndentLevelZero, "%s\n", configMap.Data[k]) + w.Write(IndentLevelZero, "\n") + } + + w.Write(IndentLevelZero, "\nBinaryData\n====\n") + for _, k := range utilmaps.SortedKeys(configMap.BinaryData) { + w.Write(IndentLevelZero, "%s: %s bytes\n", k, strconv.Itoa(len(configMap.BinaryData[k]))) + } + w.Write(IndentLevelZero, "\n") + + return nil + }) +} diff --git a/pkg/cmd/describe/describe_test.go b/pkg/cmd/describe/describe_test.go new file mode 100644 index 0000000..cced729 --- /dev/null +++ b/pkg/cmd/describe/describe_test.go @@ -0,0 +1,830 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package describe + +import ( + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + apiresource "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/resource" + k8sfake "k8s.io/client-go/kubernetes/fake" + restfake "k8s.io/client-go/rest/fake" + "k8s.io/utils/ptr" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestDescribeCmd(t *testing.T) { + testCases := map[string]struct { + args []string + argsFormat int + objs []runtime.Object + withK8sClientSet bool + mapperKinds []schema.GroupVersionKind + wantOut string + wantOutErr string + wantErr error + }{ + "shouldn't describe none kjobctl owned jobs": { + args: []string{"job", "sample-job-8c7zt"}, + argsFormat: modeTaskArgsFormat, + mapperKinds: []schema.GroupVersionKind{ + batchv1.SchemeGroupVersion.WithKind("Job"), + }, + objs: []runtime.Object{ + wrappers.MakeJob("sample-job-8c7zt", metav1.NamespaceDefault).Obj(), + }, + wantOutErr: "No resources found in default namespace.\n", + }, + "describe job with 'mode task' format": { + args: []string{"job", "sample-job-8c7zt"}, + argsFormat: modeTaskArgsFormat, + mapperKinds: []schema.GroupVersionKind{ + batchv1.SchemeGroupVersion.WithKind("Job"), + }, + objs: []runtime.Object{ + getSampleJob("sample-job-8c7zt"), + }, + wantOut: `Name: sample-job-8c7zt +Namespace: default +Labels: kjobctl.x-k8s.io/profile=sample-profile +Annotations: kjobctl.x-k8s.io/script=test.sh +Parallelism: 3 +Completions: 2 +Start Time: Mon, 01 Jan 2024 00:00:00 +0000 +Completed At: Mon, 01 Jan 2024 00:00:33 +0000 +Duration: 33s +Pods Statuses: 0 Active / 2 Succeeded / 0 Failed +Pod Template: + Containers: + sample-container: + Port: + Host Port: + Command: + sleep + 15s + Args: + 30s + Requests: + cpu: 1 + memory: 200Mi + Environment: + Mounts: + Volumes: +`, + }, + "describe slurm with 'mode task' format": { + args: []string{"slurm", "sample-job-8c7zt"}, + argsFormat: modeTaskArgsFormat, + mapperKinds: []schema.GroupVersionKind{ + batchv1.SchemeGroupVersion.WithKind("Job"), + corev1.SchemeGroupVersion.WithKind("ConfigMap"), + }, + withK8sClientSet: true, + objs: []runtime.Object{ + getSampleJob("sample-job-8c7zt"), + wrappers.MakeConfigMap("sample-job-8c7zt", "default"). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Data(map[string]string{ + "script": "#!/bin/bash\nsleep 300", + "entrypoint.sh": `#!/usr/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +# External variables +# JOB_COMPLETION_INDEX - completion index of the job. +# JOB_CONTAINER_INDEX - container index in the container template. + +# ["COMPLETION_INDEX"]="CONTAINER_INDEX_1,CONTAINER_INDEX_2" +declare -A array_indexes=(["0"]="0") # Requires bash v4+ + +container_indexes=${array_indexes[${JOB_COMPLETION_INDEX}]} +container_indexes=(${container_indexes//,/ }) + +if [[ ! -v container_indexes[${JOB_CONTAINER_INDEX}] ]]; +then +exit 0 +fi + +SBATCH_ARRAY_INX= +SBATCH_GPUS_PER_TASK= +SBATCH_MEM_PER_CPU= +SBATCH_MEM_PER_GPU= +SBATCH_OUTPUT= +SBATCH_ERROR= +SBATCH_INPUT= +SBATCH_JOB_NAME= +SBATCH_PARTITION= + +export SLURM_ARRAY_JOB_ID=1 # Job array’s master job ID number. +export SLURM_ARRAY_TASK_COUNT=1 # Total number of tasks in a job array. +export SLURM_ARRAY_TASK_MAX=0 # Job array’s maximum ID (index) number. +export SLURM_ARRAY_TASK_MIN=0 # Job array’s minimum ID (index) number. +export SLURM_TASKS_PER_NODE=1 # Number of tasks to be initiated on each node. +export SLURM_CPUS_PER_TASK= # Number of CPUs per task. +export SLURM_CPUS_ON_NODE= # Number of CPUs on the allocated node (actually pod). +export SLURM_JOB_CPUS_PER_NODE= # Count of processors available to the job on this node. +export SLURM_CPUS_PER_GPU= # Number of CPUs requested per allocated GPU. +export SLURM_MEM_PER_CPU= # Memory per CPU. Same as --mem-per-cpu . +export SLURM_MEM_PER_GPU= # Memory per GPU. +export SLURM_MEM_PER_NODE= # Memory per node. Same as --mem. +export SLURM_GPUS=0 # Number of GPUs requested (in total). +export SLURM_NTASKS=1 # Same as -n, –ntasks. The number of tasks. +export SLURM_NTASKS_PER_NODE=1 # Number of tasks requested per node. +export SLURM_NPROCS=$SLURM_NTASKS # Same as -n, --ntasks. See $SLURM_NTASKS. +export SLURM_NNODES=1 # Total number of nodes (actually pods) in the job’s resource allocation. +export SLURM_SUBMIT_DIR=/slurm # The path of the job submission directory. +export SLURM_SUBMIT_HOST=$HOSTNAME # The hostname of the node used for job submission. + +export SLURM_JOB_ID=$(( JOB_COMPLETION_INDEX * SLURM_TASKS_PER_NODE + JOB_CONTAINER_INDEX + SLURM_ARRAY_JOB_ID )) # The Job ID. +export SLURM_JOBID=$SLURM_JOB_ID # Deprecated. Same as $SLURM_JOB_ID +export SLURM_ARRAY_TASK_ID=${container_indexes[${JOB_CONTAINER_INDEX}]} # Task ID. + +unmask_filename () { +replaced="$1" + +if [[ "$replaced" == "\\"* ]]; then +replaced="${replaced//\\/}" +echo "${replaced}" +return 0 +fi + +replaced=$(echo "$replaced" | sed -E "s/(%)(%A)/\1\n\2/g;:a s/(^|[^\n])%A/\1$SLURM_ARRAY_JOB_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%a)/\1\n\2/g;:a s/(^|[^\n])%a/\1$SLURM_ARRAY_TASK_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%j)/\1\n\2/g;:a s/(^|[^\n])%j/\1$SLURM_JOB_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%N)/\1\n\2/g;:a s/(^|[^\n])%N/\1$HOSTNAME/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%n)/\1\n\2/g;:a s/(^|[^\n])%n/\1$JOB_COMPLETION_INDEX/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%t)/\1\n\2/g;:a s/(^|[^\n])%t/\1$SLURM_ARRAY_TASK_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%u)/\1\n\2/g;:a s/(^|[^\n])%u/\1$USER_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%x)/\1\n\2/g;:a s/(^|[^\n])%x/\1$SBATCH_JOB_NAME/;ta;s/\n//g") + +replaced="${replaced//%%/%}" + +echo "$replaced" +} + +input_file=$(unmask_filename "$SBATCH_INPUT") +output_file=$(unmask_filename "$SBATCH_OUTPUT") +error_path=$(unmask_filename "$SBATCH_ERROR") + +/slurm/script +`, + }). + Obj(), + }, + wantOut: `Name: sample-job-8c7zt +Namespace: default +Labels: kjobctl.x-k8s.io/profile=sample-profile +Annotations: kjobctl.x-k8s.io/script=test.sh +Parallelism: 3 +Completions: 2 +Start Time: Mon, 01 Jan 2024 00:00:00 +0000 +Completed At: Mon, 01 Jan 2024 00:00:33 +0000 +Duration: 33s +Pods Statuses: 0 Active / 2 Succeeded / 0 Failed +Pod Template: + Containers: + sample-container: + Port: + Host Port: + Command: + sleep + 15s + Args: + 30s + Requests: + cpu: 1 + memory: 200Mi + Environment: + Mounts: + Volumes: + + +Name: sample-job-8c7zt +Namespace: default +Labels: kjobctl.x-k8s.io/mode=Slurm + kjobctl.x-k8s.io/profile=profile + +Data +==== +entrypoint.sh: +---- +#!/usr/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +# External variables +# JOB_COMPLETION_INDEX - completion index of the job. +# JOB_CONTAINER_INDEX - container index in the container template. + +# ["COMPLETION_INDEX"]="CONTAINER_INDEX_1,CONTAINER_INDEX_2" +declare -A array_indexes=(["0"]="0") # Requires bash v4+ + +container_indexes=${array_indexes[${JOB_COMPLETION_INDEX}]} +container_indexes=(${container_indexes//,/ }) + +if [[ ! -v container_indexes[${JOB_CONTAINER_INDEX}] ]]; +then +exit 0 +fi + +SBATCH_ARRAY_INX= +SBATCH_GPUS_PER_TASK= +SBATCH_MEM_PER_CPU= +SBATCH_MEM_PER_GPU= +SBATCH_OUTPUT= +SBATCH_ERROR= +SBATCH_INPUT= +SBATCH_JOB_NAME= +SBATCH_PARTITION= + +export SLURM_ARRAY_JOB_ID=1 # Job array’s master job ID number. +export SLURM_ARRAY_TASK_COUNT=1 # Total number of tasks in a job array. +export SLURM_ARRAY_TASK_MAX=0 # Job array’s maximum ID (index) number. +export SLURM_ARRAY_TASK_MIN=0 # Job array’s minimum ID (index) number. +export SLURM_TASKS_PER_NODE=1 # Number of tasks to be initiated on each node. +export SLURM_CPUS_PER_TASK= # Number of CPUs per task. +export SLURM_CPUS_ON_NODE= # Number of CPUs on the allocated node (actually pod). +export SLURM_JOB_CPUS_PER_NODE= # Count of processors available to the job on this node. +export SLURM_CPUS_PER_GPU= # Number of CPUs requested per allocated GPU. +export SLURM_MEM_PER_CPU= # Memory per CPU. Same as --mem-per-cpu . +export SLURM_MEM_PER_GPU= # Memory per GPU. +export SLURM_MEM_PER_NODE= # Memory per node. Same as --mem. +export SLURM_GPUS=0 # Number of GPUs requested (in total). +export SLURM_NTASKS=1 # Same as -n, –ntasks. The number of tasks. +export SLURM_NTASKS_PER_NODE=1 # Number of tasks requested per node. +export SLURM_NPROCS=$SLURM_NTASKS # Same as -n, --ntasks. See $SLURM_NTASKS. +export SLURM_NNODES=1 # Total number of nodes (actually pods) in the job’s resource allocation. +export SLURM_SUBMIT_DIR=/slurm # The path of the job submission directory. +export SLURM_SUBMIT_HOST=$HOSTNAME # The hostname of the node used for job submission. + +export SLURM_JOB_ID=$(( JOB_COMPLETION_INDEX * SLURM_TASKS_PER_NODE + JOB_CONTAINER_INDEX + SLURM_ARRAY_JOB_ID )) # The Job ID. +export SLURM_JOBID=$SLURM_JOB_ID # Deprecated. Same as $SLURM_JOB_ID +export SLURM_ARRAY_TASK_ID=${container_indexes[${JOB_CONTAINER_INDEX}]} # Task ID. + +unmask_filename () { +replaced="$1" + +if [[ "$replaced" == "\\"* ]]; then +replaced="${replaced//\\/}" +echo "${replaced}" +return 0 +fi + +replaced=$(echo "$replaced" | sed -E "s/(%)(%A)/\1\n\2/g;:a s/(^|[^\n])%A/\1$SLURM_ARRAY_JOB_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%a)/\1\n\2/g;:a s/(^|[^\n])%a/\1$SLURM_ARRAY_TASK_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%j)/\1\n\2/g;:a s/(^|[^\n])%j/\1$SLURM_JOB_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%N)/\1\n\2/g;:a s/(^|[^\n])%N/\1$HOSTNAME/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%n)/\1\n\2/g;:a s/(^|[^\n])%n/\1$JOB_COMPLETION_INDEX/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%t)/\1\n\2/g;:a s/(^|[^\n])%t/\1$SLURM_ARRAY_TASK_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%u)/\1\n\2/g;:a s/(^|[^\n])%u/\1$USER_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%x)/\1\n\2/g;:a s/(^|[^\n])%x/\1$SBATCH_JOB_NAME/;ta;s/\n//g") + +replaced="${replaced//%%/%}" + +echo "$replaced" +} + +input_file=$(unmask_filename "$SBATCH_INPUT") +output_file=$(unmask_filename "$SBATCH_OUTPUT") +error_path=$(unmask_filename "$SBATCH_ERROR") + +/slurm/script + + +script: +---- +#!/bin/bash +sleep 300 + + +BinaryData +==== + +`, + }, + "describe specific task with 'mode slash task' format": { + args: []string{"job/sample-job-8c7zt"}, + argsFormat: modeTaskArgsFormat, + mapperKinds: []schema.GroupVersionKind{ + batchv1.SchemeGroupVersion.WithKind("Job"), + }, + objs: []runtime.Object{ + getSampleJob("sample-job-8c7zt"), + }, + wantOut: `Name: sample-job-8c7zt +Namespace: default +Labels: kjobctl.x-k8s.io/profile=sample-profile +Annotations: kjobctl.x-k8s.io/script=test.sh +Parallelism: 3 +Completions: 2 +Start Time: Mon, 01 Jan 2024 00:00:00 +0000 +Completed At: Mon, 01 Jan 2024 00:00:33 +0000 +Duration: 33s +Pods Statuses: 0 Active / 2 Succeeded / 0 Failed +Pod Template: + Containers: + sample-container: + Port: + Host Port: + Command: + sleep + 15s + Args: + 30s + Requests: + cpu: 1 + memory: 200Mi + Environment: + Mounts: + Volumes: +`, + }, + "describe all jobs": { + args: []string{"job"}, + argsFormat: modeTaskArgsFormat, + mapperKinds: []schema.GroupVersionKind{ + batchv1.SchemeGroupVersion.WithKind("Job"), + }, + objs: []runtime.Object{ + &batchv1.JobList{ + Items: []batchv1.Job{ + *getSampleJob("sample-job-5zd6r"), + *getSampleJob("sample-job-8c7zt"), + }, + }, + }, + wantOut: `Name: sample-job-5zd6r +Namespace: default +Labels: kjobctl.x-k8s.io/profile=sample-profile +Annotations: kjobctl.x-k8s.io/script=test.sh +Parallelism: 3 +Completions: 2 +Start Time: Mon, 01 Jan 2024 00:00:00 +0000 +Completed At: Mon, 01 Jan 2024 00:00:33 +0000 +Duration: 33s +Pods Statuses: 0 Active / 2 Succeeded / 0 Failed +Pod Template: + Containers: + sample-container: + Port: + Host Port: + Command: + sleep + 15s + Args: + 30s + Requests: + cpu: 1 + memory: 200Mi + Environment: + Mounts: + Volumes: + + +Name: sample-job-8c7zt +Namespace: default +Labels: kjobctl.x-k8s.io/profile=sample-profile +Annotations: kjobctl.x-k8s.io/script=test.sh +Parallelism: 3 +Completions: 2 +Start Time: Mon, 01 Jan 2024 00:00:00 +0000 +Completed At: Mon, 01 Jan 2024 00:00:33 +0000 +Duration: 33s +Pods Statuses: 0 Active / 2 Succeeded / 0 Failed +Pod Template: + Containers: + sample-container: + Port: + Host Port: + Command: + sleep + 15s + Args: + 30s + Requests: + cpu: 1 + memory: 200Mi + Environment: + Mounts: + Volumes: +`, + }, + "describe interactive with 'mode task' format": { + args: []string{"interactive", "sample-interactive-fgnh9"}, + argsFormat: modeTaskArgsFormat, + mapperKinds: []schema.GroupVersionKind{ + corev1.SchemeGroupVersion.WithKind("Pod"), + }, + objs: []runtime.Object{ + getSampleInteractive("sample-interactive-fgnh9"), + }, + wantOut: `Name: sample-interactive-fgnh9 +Namespace: default +Start Time: Mon, 01 Jan 2024 00:00:00 +0000 +Labels: kjobctl.x-k8s.io/profile=sample-profile +Status: Running +Containers: + sample-container: + Port: + Host Port: + Command: + /bin/sh + Environment: + TASK_NAME: sample-interactive + Mounts: + /sample from sample-volume (rw) +Volumes: + sample-volume: + Type: EmptyDir (a temporary directory that shares a pod's lifetime) + Medium: + SizeLimit: +`, + }, + "describe all interactive tasks": { + args: []string{"interactive"}, + argsFormat: modeTaskArgsFormat, + mapperKinds: []schema.GroupVersionKind{ + corev1.SchemeGroupVersion.WithKind("Pod"), + }, + objs: []runtime.Object{ + &corev1.PodList{ + Items: []corev1.Pod{ + *getSampleInteractive("sample-interactive-fgnh9"), + *getSampleInteractive("sample-interactive-hs2b2"), + }, + }, + }, + wantOut: `Name: sample-interactive-fgnh9 +Namespace: default +Start Time: Mon, 01 Jan 2024 00:00:00 +0000 +Labels: kjobctl.x-k8s.io/profile=sample-profile +Status: Running +Containers: + sample-container: + Port: + Host Port: + Command: + /bin/sh + Environment: + TASK_NAME: sample-interactive + Mounts: + /sample from sample-volume (rw) +Volumes: + sample-volume: + Type: EmptyDir (a temporary directory that shares a pod's lifetime) + Medium: + SizeLimit: + + +Name: sample-interactive-hs2b2 +Namespace: default +Start Time: Mon, 01 Jan 2024 00:00:00 +0000 +Labels: kjobctl.x-k8s.io/profile=sample-profile +Status: Running +Containers: + sample-container: + Port: + Host Port: + Command: + /bin/sh + Environment: + TASK_NAME: sample-interactive + Mounts: + /sample from sample-volume (rw) +Volumes: + sample-volume: + Type: EmptyDir (a temporary directory that shares a pod's lifetime) + Medium: + SizeLimit: +`, + }, + "describe ray job with 'mode task' format": { + args: []string{"rayjob", "sample-ray-job"}, + argsFormat: modeTaskArgsFormat, + mapperKinds: []schema.GroupVersionKind{ + rayv1.SchemeGroupVersion.WithKind("RayJob"), + }, + objs: []runtime.Object{ + wrappers.MakeRayJob("sample-ray-job", metav1.NamespaceDefault). + Profile("my-profile"). + LocalQueue("lq"). + StartTime(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)). + EndTime(time.Date(2024, 1, 1, 1, 0, 0, 0, time.UTC)). + JobDeploymentStatus(rayv1.JobDeploymentStatusRunning). + JobStatus(rayv1.JobStatusFailed). + Reason(rayv1.DeadlineExceeded). + Message("Error message"). + RayClusterName("my-cluster"). + Obj(), + }, + wantOut: `Name: sample-ray-job +Namespace: default +Start Time: Mon, 01 Jan 2024 00:00:00 +0000 +End Time: Mon, 01 Jan 2024 01:00:00 +0000 +Labels: kjobctl.x-k8s.io/profile=my-profile + kueue.x-k8s.io/queue-name=lq +Job Deployment Status: Running +Job Status: FAILED +Reason: DeadlineExceeded +Message: Error message +Ray Cluster Name: my-cluster +Ray Cluster Status: + Desired CPU: 0 + Desired GPU: 0 + Desired Memory: 0 + Desired TPU: 0 +`, + }, + "describe ray cluster with 'mode task' format": { + args: []string{"raycluster", "sample-ray-cluster"}, + argsFormat: modeTaskArgsFormat, + mapperKinds: []schema.GroupVersionKind{ + rayv1.SchemeGroupVersion.WithKind("RayCluster"), + }, + objs: []runtime.Object{ + wrappers.MakeRayCluster("sample-ray-job", metav1.NamespaceDefault). + Profile("my-profile"). + LocalQueue("lq"). + State(rayv1.Failed). + Reason("Reason message"). + DesiredCPU(apiresource.MustParse("1")). + DesiredGPU(apiresource.MustParse("5")). + DesiredMemory(apiresource.MustParse("2Gi")). + DesiredTPU(apiresource.MustParse("10")). + ReadyWorkerReplicas(1). + AvailableWorkerReplicas(1). + DesiredWorkerReplicas(1). + MinWorkerReplicas(1). + MaxWorkerReplicas(5). + Spec( + *wrappers.MakeRayClusterSpec(). + HeadGroupSpec(rayv1.HeadGroupSpec{ + RayStartParams: map[string]string{"p1": "v1", "p2": "v2"}, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + *wrappers.MakeContainer("ray-head", "rayproject/ray:2.9.0"). + WithEnvVar(corev1.EnvVar{Name: "TASK_NAME", Value: "sample-interactive"}). + WithVolumeMount(corev1.VolumeMount{Name: "sample-volume", MountPath: "/sample"}). + Obj(), + }, + }, + }, + }). + WithWorkerGroupSpec( + *wrappers.MakeWorkerGroupSpec("group1"). + Replicas(1). + MinReplicas(1). + MaxReplicas(5). + RayStartParams(map[string]string{"p1": "v1", "p2": "v2"}). + WithContainer( + *wrappers.MakeContainer("ray-worker", "rayproject/ray:2.9.0"). + WithEnvVar(corev1.EnvVar{Name: "TASK_NAME", Value: "sample-interactive"}). + WithVolumeMount(corev1.VolumeMount{Name: "sample-volume", MountPath: "/sample"}). + Obj(), + ). + Obj(), + ). + Suspend(true). + Obj(), + ). + Obj(), + }, + wantOut: `Name: sample-ray-job +Namespace: default +Labels: kjobctl.x-k8s.io/profile=my-profile + kueue.x-k8s.io/queue-name=lq +Suspend: true +State: failed +Reason: Reason message +Desired CPU: 1 +Desired GPU: 5 +Desired Memory: 2Gi +Desired TPU: 10 +Ready Worker Replicas: 1 +Available Worker Replicas: 1 +Desired Worker Replicas: 1 +Min Worker Replicas: 1 +Max Worker Replicas: 5 +Head Group: + Start Params: p1=v1 + p2=v2 + Pod Template: + Containers: + ray-head: + Port: + Host Port: + Environment: + TASK_NAME: sample-interactive + Mounts: + /sample from sample-volume (rw) + Volumes: +Worker Groups: + group1: + Replicas: 1 + Min Replicas: 1 + Max Replicas: 5 + Start Params: p1=v1 + p2=v2 + Pod Template: + Containers: + ray-worker: + Port: + Host Port: + Environment: + TASK_NAME: sample-interactive + Mounts: + /sample from sample-volume (rw) + Volumes: +`, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + t.Setenv("TZ", "") + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + tcg := cmdtesting.NewTestClientGetter() + if tc.withK8sClientSet { + tcg.WithK8sClientset(k8sfake.NewSimpleClientset(tc.objs...)) + } + + if len(tc.mapperKinds) != 0 { + mapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{}) + for _, k := range tc.mapperKinds { + mapper.Add(k, meta.RESTScopeNamespace) + } + tcg.WithRESTMapper(mapper) + } + + if len(tc.objs) != 0 { + scheme := runtime.NewScheme() + + if err := batchv1.AddToScheme(scheme); err != nil { + t.Errorf("Unexpected error\n%s", err) + } + + if err := corev1.AddToScheme(scheme); err != nil { + t.Errorf("Unexpected error\n%s", err) + } + + if err := rayv1.AddToScheme(scheme); err != nil { + t.Errorf("Unexpected error\n%s", err) + } + + codec := serializer.NewCodecFactory(scheme).LegacyCodec(scheme.PrioritizedVersionsAllGroups()...) + tcg.WithRESTClient(&restfake.RESTClient{ + NegotiatedSerializer: resource.UnstructuredPlusDefaultContentConfig().NegotiatedSerializer, + Resp: &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(runtime.EncodeOrDie(codec, tc.objs[0]))), + }, + }) + } + + cmd := NewDescribeCmd(tcg, streams) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + if diff := cmp.Diff(tc.wantErr, gotErr, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + }) + } +} + +func getSampleJob(name string) *batchv1.Job { + return &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + Labels: map[string]string{ + "kjobctl.x-k8s.io/profile": "sample-profile", + }, + Annotations: map[string]string{ + constants.ScriptAnnotation: "test.sh", + }, + }, + Spec: batchv1.JobSpec{ + Parallelism: ptr.To[int32](3), + Completions: ptr.To[int32](2), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "sample-container", + Command: []string{"sleep", "15s"}, + Args: []string{"30s"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: apiresource.MustParse("1"), + corev1.ResourceMemory: apiresource.MustParse("200Mi"), + }, + }, + }, + }, + }, + }, + }, + Status: batchv1.JobStatus{ + Active: 0, + Succeeded: 2, + Failed: 0, + StartTime: ptr.To(metav1.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)), + CompletionTime: ptr.To(metav1.Date(2024, 1, 1, 0, 0, 33, 0, time.UTC)), + }, + } +} + +func getSampleInteractive(name string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + Labels: map[string]string{ + "kjobctl.x-k8s.io/profile": "sample-profile", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "sample-container", + Command: []string{"/bin/sh"}, + Env: []corev1.EnvVar{ + { + Name: "TASK_NAME", + Value: "sample-interactive", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "sample-volume", + MountPath: "/sample", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "sample-volume", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + StartTime: ptr.To(metav1.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)), + }, + } +} diff --git a/pkg/cmd/describe/helpers.go b/pkg/cmd/describe/helpers.go new file mode 100644 index 0000000..0005df8 --- /dev/null +++ b/pkg/cmd/describe/helpers.go @@ -0,0 +1,868 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package describe + +import ( + "bytes" + "errors" + "fmt" + "io" + "sort" + "strconv" + "strings" + "text/tabwriter" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/duration" + "k8s.io/apimachinery/pkg/util/sets" + describehelper "k8s.io/kubectl/pkg/describe" + resourcehelper "k8s.io/kubectl/pkg/util/resource" + storageutil "k8s.io/kubectl/pkg/util/storage" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// Each level has 2 spaces for describehelper.PrefixWriter +const ( + IndentLevelZero = iota + IndentLevelOne + IndentLevelTwo + IndentLevelThree +) + +var maxAnnotationLen = 140 + +func parseAppProfileModeName(s string) ([]string, error) { + seg := strings.Split(s, "/") + if len(seg) != 2 { + if len(seg) > 2 { + return nil, errors.New("arguments in mode/name form may not have more than one slash") + } + return nil, errors.New("argument must be in mode/name form") + } + + return seg, nil +} + +func resourceFor(mode string) string { + if strings.EqualFold(mode, string(v1alpha1.InteractiveMode)) { + return "pod" + } + + if strings.EqualFold(mode, string(v1alpha1.SlurmMode)) { + return "job" + } + + return strings.ToLower(mode) +} + +func gvkFor(mapper meta.RESTMapper, resource string) (schema.GroupVersionKind, error) { + var gvk schema.GroupVersionKind + var err error + + fullySpecifiedGVR, groupResource := schema.ParseResourceArg(strings.ToLower(resource)) + gvr := schema.GroupVersionResource{} + if fullySpecifiedGVR != nil { + gvr, _ = mapper.ResourceFor(*fullySpecifiedGVR) + } + if gvr.Empty() { + gvr, err = mapper.ResourceFor(groupResource.WithVersion("")) + if err != nil { + return gvk, err + } + } + + gvk, err = mapper.KindFor(gvr) + if err != nil { + return gvk, err + } + + return gvk, err +} + +// below functions copied from https://github.com/kubernetes/kubectl/blob/master/pkg/describe/describe.go + +func tabbedString(f func(io.Writer) error) (string, error) { + out := new(tabwriter.Writer) + buf := &bytes.Buffer{} + out.Init(buf, 0, 8, 2, ' ', 0) + + err := f(out) + if err != nil { + return "", err + } + + out.Flush() + return buf.String(), nil +} + +// printLabelsMultiline prints multiple labels with a proper alignment. +func printLabelsMultiline(w describehelper.PrefixWriter, title string, labels map[string]string) { + printLabelsMultilineWithIndent(w, "", title, "\t", labels, sets.New[string]()) +} + +// printLabelsMultiline prints multiple labels with a user-defined alignment. +func printLabelsMultilineWithIndent(w describehelper.PrefixWriter, initialIndent, title, innerIndent string, labels map[string]string, skip sets.Set[string]) { + w.Write(IndentLevelZero, "%s%s:%s", initialIndent, title, innerIndent) + + if len(labels) == 0 { + w.WriteLine("") + return + } + + // to print labels in the sorted order + keys := make([]string, 0, len(labels)) + for key := range labels { + if skip.Has(key) { + continue + } + keys = append(keys, key) + } + if len(keys) == 0 { + w.WriteLine("") + return + } + sort.Strings(keys) + + for i, key := range keys { + if i != 0 { + w.Write(IndentLevelZero, "%s", initialIndent) + w.Write(IndentLevelZero, "%s", innerIndent) + } + w.Write(IndentLevelZero, "%s=%s\n", key, labels[key]) + } +} + +func describePodTemplate(template *corev1.PodTemplateSpec, w describehelper.PrefixWriter) { + w.Write(IndentLevelZero, "Pod Template:\n") + if template == nil { + w.Write(IndentLevelOne, "") + return + } + + if len(template.Spec.InitContainers) > 0 { + describeContainers("Init Containers", template.Spec.InitContainers, w, " ") + } + describeContainers("Containers", template.Spec.Containers, w, " ") + describeVolumes(template.Spec.Volumes, w, " ") +} + +func describeContainers(label string, containers []corev1.Container, w describehelper.PrefixWriter, space string) { + describeContainersLabel(containers, label, space, w) + + for _, container := range containers { + describeContainerBasicInfo(container, space, w) + describeContainerCommand(container, w) + describeContainerResource(container, w) + describeContainerProbe(container, w) + if len(container.EnvFrom) > 0 { + describeContainerEnvFrom(container, w) + } + describeContainerEnvVars(container, w) + describeContainerVolumes(container, w) + } +} + +func describeContainersLabel(containers []corev1.Container, label, space string, w describehelper.PrefixWriter) { + none := "" + if len(containers) == 0 { + none = " " + } + w.Write(IndentLevelZero, "%s%s:%s\n", space, label, none) +} + +func describeContainerBasicInfo(container corev1.Container, space string, w describehelper.PrefixWriter) { + nameIndent := "" + if len(space) > 0 { + nameIndent = " " + } + w.Write(IndentLevelOne, "%s%v:\n", nameIndent, container.Name) + portString := describeContainerPorts(container.Ports) + if strings.Contains(portString, ",") { + w.Write(IndentLevelTwo, "Ports:\t%s\n", portString) + } else { + w.Write(IndentLevelTwo, "Port:\t%s\n", stringOrNone(portString)) + } + hostPortString := describeContainerHostPorts(container.Ports) + if strings.Contains(hostPortString, ",") { + w.Write(IndentLevelTwo, "Host Ports:\t%s\n", hostPortString) + } else { + w.Write(IndentLevelTwo, "Host Port:\t%s\n", stringOrNone(hostPortString)) + } +} + +func stringOrNone(s string) string { + return stringOrDefaultValue(s, "") +} + +func stringOrDefaultValue(s, defaultValue string) string { + if len(s) > 0 { + return s + } + return defaultValue +} + +func describeContainerPorts(cPorts []corev1.ContainerPort) string { + ports := make([]string, 0, len(cPorts)) + for _, cPort := range cPorts { + ports = append(ports, fmt.Sprintf("%d/%s", cPort.ContainerPort, cPort.Protocol)) + } + return strings.Join(ports, ", ") +} + +func describeContainerHostPorts(cPorts []corev1.ContainerPort) string { + ports := make([]string, 0, len(cPorts)) + for _, cPort := range cPorts { + ports = append(ports, fmt.Sprintf("%d/%s", cPort.HostPort, cPort.Protocol)) + } + return strings.Join(ports, ", ") +} + +func describeContainerCommand(container corev1.Container, w describehelper.PrefixWriter) { + if len(container.Command) > 0 { + w.Write(IndentLevelTwo, "Command:\n") + for _, c := range container.Command { + for _, s := range strings.Split(c, "\n") { + w.Write(IndentLevelThree, "%s\n", s) + } + } + } + if len(container.Args) > 0 { + w.Write(IndentLevelTwo, "Args:\n") + for _, arg := range container.Args { + for _, s := range strings.Split(arg, "\n") { + w.Write(IndentLevelThree, "%s\n", s) + } + } + } +} + +func describeContainerResource(container corev1.Container, w describehelper.PrefixWriter) { + resources := container.Resources + if len(resources.Limits) > 0 { + w.Write(IndentLevelTwo, "Limits:\n") + } + for _, name := range describehelper.SortedResourceNames(resources.Limits) { + quantity := resources.Limits[name] + w.Write(IndentLevelThree, "%s:\t%s\n", name, quantity.String()) + } + + if len(resources.Requests) > 0 { + w.Write(IndentLevelTwo, "Requests:\n") + } + for _, name := range describehelper.SortedResourceNames(resources.Requests) { + quantity := resources.Requests[name] + w.Write(IndentLevelThree, "%s:\t%s\n", name, quantity.String()) + } +} + +func describeContainerProbe(container corev1.Container, w describehelper.PrefixWriter) { + if container.LivenessProbe != nil { + probe := describehelper.DescribeProbe(container.LivenessProbe) + w.Write(IndentLevelTwo, "Liveness:\t%s\n", probe) + } + if container.ReadinessProbe != nil { + probe := describehelper.DescribeProbe(container.ReadinessProbe) + w.Write(IndentLevelTwo, "Readiness:\t%s\n", probe) + } + if container.StartupProbe != nil { + probe := describehelper.DescribeProbe(container.StartupProbe) + w.Write(IndentLevelTwo, "Startup:\t%s\n", probe) + } +} + +func describeContainerVolumes(container corev1.Container, w describehelper.PrefixWriter) { + // Show volumeMounts + none := "" + if len(container.VolumeMounts) == 0 { + none = "\t" + } + w.Write(IndentLevelTwo, "Mounts:%s\n", none) + sort.Sort(describehelper.SortableVolumeMounts(container.VolumeMounts)) + for _, mount := range container.VolumeMounts { + flags := []string{} + if mount.ReadOnly { + flags = append(flags, "ro") + } else { + flags = append(flags, "rw") + } + if len(mount.SubPath) > 0 { + flags = append(flags, fmt.Sprintf("path=%q", mount.SubPath)) + } + w.Write(IndentLevelThree, "%s from %s (%s)\n", mount.MountPath, mount.Name, strings.Join(flags, ",")) + } + // Show volumeDevices if exists + if len(container.VolumeDevices) > 0 { + w.Write(IndentLevelTwo, "Devices:%s\n", none) + sort.Sort(describehelper.SortableVolumeDevices(container.VolumeDevices)) + for _, device := range container.VolumeDevices { + w.Write(IndentLevelThree, "%s from %s\n", device.DevicePath, device.Name) + } + } +} + +func describeContainerEnvVars(container corev1.Container, w describehelper.PrefixWriter) { + none := "" + if len(container.Env) == 0 { + none = "\t" + } + w.Write(IndentLevelTwo, "Environment:%s\n", none) + + for _, e := range container.Env { + if e.ValueFrom == nil { + for i, s := range strings.Split(e.Value, "\n") { + if i == 0 { + w.Write(IndentLevelThree, "%s:\t%s\n", e.Name, s) + } else { + w.Write(IndentLevelThree, "\t%s\n", s) + } + } + continue + } + + switch { + case e.ValueFrom.FieldRef != nil: + w.Write(IndentLevelThree, "%s:\t%s (%s:%s)\n", e.Name, "", e.ValueFrom.FieldRef.APIVersion, e.ValueFrom.FieldRef.FieldPath) + case e.ValueFrom.ResourceFieldRef != nil: + valueFrom, err := resourcehelper.ExtractContainerResourceValue(e.ValueFrom.ResourceFieldRef, &container) + if err != nil { + valueFrom = "" + } + resource := e.ValueFrom.ResourceFieldRef.Resource + if valueFrom == "0" && (resource == "limits.cpu" || resource == "limits.memory") { + valueFrom = "node allocatable" + } + w.Write(IndentLevelThree, "%s:\t%s (%s)\n", e.Name, valueFrom, resource) + case e.ValueFrom.SecretKeyRef != nil: + optional := e.ValueFrom.SecretKeyRef.Optional != nil && *e.ValueFrom.SecretKeyRef.Optional + w.Write(IndentLevelThree, "%s:\t\tOptional: %t\n", e.Name, e.ValueFrom.SecretKeyRef.Key, e.ValueFrom.SecretKeyRef.Name, optional) + case e.ValueFrom.ConfigMapKeyRef != nil: + optional := e.ValueFrom.ConfigMapKeyRef.Optional != nil && *e.ValueFrom.ConfigMapKeyRef.Optional + w.Write(IndentLevelThree, "%s:\t\tOptional: %t\n", e.Name, e.ValueFrom.ConfigMapKeyRef.Key, e.ValueFrom.ConfigMapKeyRef.Name, optional) + } + } +} + +func describeContainerEnvFrom(container corev1.Container, w describehelper.PrefixWriter) { + none := "" + if len(container.EnvFrom) == 0 { + none = "\t" + } + w.Write(IndentLevelTwo, "Environment Variables from:%s\n", none) + + for _, e := range container.EnvFrom { + from := "" + name := "" + optional := false + if e.ConfigMapRef != nil { + from = "ConfigMap" + name = e.ConfigMapRef.Name + optional = e.ConfigMapRef.Optional != nil && *e.ConfigMapRef.Optional + } else if e.SecretRef != nil { + from = "Secret" + name = e.SecretRef.Name + optional = e.SecretRef.Optional != nil && *e.SecretRef.Optional + } + if len(e.Prefix) == 0 { + w.Write(IndentLevelThree, "%s\t%s\tOptional: %t\n", name, from, optional) + } else { + w.Write(IndentLevelThree, "%s\t%s with prefix '%s'\tOptional: %t\n", name, from, e.Prefix, optional) + } + } +} + +func describeVolumes(volumes []corev1.Volume, w describehelper.PrefixWriter, space string) { + if len(volumes) == 0 { + w.Write(IndentLevelZero, "%sVolumes:\t\n", space) + return + } + + w.Write(IndentLevelZero, "%sVolumes:\n", space) + for _, volume := range volumes { + nameIndent := "" + if len(space) > 0 { + nameIndent = " " + } + w.Write(IndentLevelOne, "%s%v:\n", nameIndent, volume.Name) + switch { + case volume.VolumeSource.HostPath != nil: + printHostPathVolumeSource(volume.VolumeSource.HostPath, w) + case volume.VolumeSource.EmptyDir != nil: + printEmptyDirVolumeSource(volume.VolumeSource.EmptyDir, w) + case volume.VolumeSource.GCEPersistentDisk != nil: + printGCEPersistentDiskVolumeSource(volume.VolumeSource.GCEPersistentDisk, w) + case volume.VolumeSource.AWSElasticBlockStore != nil: + printAWSElasticBlockStoreVolumeSource(volume.VolumeSource.AWSElasticBlockStore, w) + case volume.VolumeSource.GitRepo != nil: + printGitRepoVolumeSource(volume.VolumeSource.GitRepo, w) + case volume.VolumeSource.Secret != nil: + printSecretVolumeSource(volume.VolumeSource.Secret, w) + case volume.VolumeSource.ConfigMap != nil: + printConfigMapVolumeSource(volume.VolumeSource.ConfigMap, w) + case volume.VolumeSource.NFS != nil: + printNFSVolumeSource(volume.VolumeSource.NFS, w) + case volume.VolumeSource.ISCSI != nil: + printISCSIVolumeSource(volume.VolumeSource.ISCSI, w) + case volume.VolumeSource.Glusterfs != nil: + printGlusterfsVolumeSource(volume.VolumeSource.Glusterfs, w) + case volume.VolumeSource.PersistentVolumeClaim != nil: + printPersistentVolumeClaimVolumeSource(volume.VolumeSource.PersistentVolumeClaim, w) + case volume.VolumeSource.Ephemeral != nil: + printEphemeralVolumeSource(volume.VolumeSource.Ephemeral, w) + case volume.VolumeSource.RBD != nil: + printRBDVolumeSource(volume.VolumeSource.RBD, w) + case volume.VolumeSource.Quobyte != nil: + printQuobyteVolumeSource(volume.VolumeSource.Quobyte, w) + case volume.VolumeSource.DownwardAPI != nil: + printDownwardAPIVolumeSource(volume.VolumeSource.DownwardAPI, w) + case volume.VolumeSource.AzureDisk != nil: + printAzureDiskVolumeSource(volume.VolumeSource.AzureDisk, w) + case volume.VolumeSource.VsphereVolume != nil: + printVsphereVolumeSource(volume.VolumeSource.VsphereVolume, w) + case volume.VolumeSource.Cinder != nil: + printCinderVolumeSource(volume.VolumeSource.Cinder, w) + case volume.VolumeSource.PhotonPersistentDisk != nil: + printPhotonPersistentDiskVolumeSource(volume.VolumeSource.PhotonPersistentDisk, w) + case volume.VolumeSource.PortworxVolume != nil: + printPortworxVolumeSource(volume.VolumeSource.PortworxVolume, w) + case volume.VolumeSource.ScaleIO != nil: + printScaleIOVolumeSource(volume.VolumeSource.ScaleIO, w) + case volume.VolumeSource.CephFS != nil: + printCephFSVolumeSource(volume.VolumeSource.CephFS, w) + case volume.VolumeSource.StorageOS != nil: + printStorageOSVolumeSource(volume.VolumeSource.StorageOS, w) + case volume.VolumeSource.FC != nil: + printFCVolumeSource(volume.VolumeSource.FC, w) + case volume.VolumeSource.AzureFile != nil: + printAzureFileVolumeSource(volume.VolumeSource.AzureFile, w) + case volume.VolumeSource.FlexVolume != nil: + printFlexVolumeSource(volume.VolumeSource.FlexVolume, w) + case volume.VolumeSource.Flocker != nil: + printFlockerVolumeSource(volume.VolumeSource.Flocker, w) + case volume.VolumeSource.Projected != nil: + printProjectedVolumeSource(volume.VolumeSource.Projected, w) + case volume.VolumeSource.CSI != nil: + printCSIVolumeSource(volume.VolumeSource.CSI, w) + default: + w.Write(IndentLevelOne, "\n") + } + } +} + +func printHostPathVolumeSource(hostPath *corev1.HostPathVolumeSource, w describehelper.PrefixWriter) { + hostPathType := "" + if hostPath.Type != nil { + hostPathType = string(*hostPath.Type) + } + w.Write(IndentLevelTwo, "Type:\tHostPath (bare host directory volume)\n"+ + " Path:\t%v\n"+ + " HostPathType:\t%v\n", + hostPath.Path, hostPathType) +} + +func printEmptyDirVolumeSource(emptyDir *corev1.EmptyDirVolumeSource, w describehelper.PrefixWriter) { + var sizeLimit string + if emptyDir.SizeLimit != nil && emptyDir.SizeLimit.Cmp(resource.Quantity{}) > 0 { + sizeLimit = fmt.Sprintf("%v", emptyDir.SizeLimit) + } else { + sizeLimit = "" + } + w.Write(IndentLevelTwo, "Type:\tEmptyDir (a temporary directory that shares a pod's lifetime)\n"+ + " Medium:\t%v\n"+ + " SizeLimit:\t%v\n", + emptyDir.Medium, sizeLimit) +} + +func printGCEPersistentDiskVolumeSource(gce *corev1.GCEPersistentDiskVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tGCEPersistentDisk (a Persistent Disk resource in Google Compute Engine)\n"+ + " PDName:\t%v\n"+ + " FSType:\t%v\n"+ + " Partition:\t%v\n"+ + " ReadOnly:\t%v\n", + gce.PDName, gce.FSType, gce.Partition, gce.ReadOnly) +} + +func printAWSElasticBlockStoreVolumeSource(aws *corev1.AWSElasticBlockStoreVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tAWSElasticBlockStore (a Persistent Disk resource in AWS)\n"+ + " VolumeID:\t%v\n"+ + " FSType:\t%v\n"+ + " Partition:\t%v\n"+ + " ReadOnly:\t%v\n", + aws.VolumeID, aws.FSType, aws.Partition, aws.ReadOnly) +} + +func printGitRepoVolumeSource(git *corev1.GitRepoVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tGitRepo (a volume that is pulled from git when the pod is created)\n"+ + " Repository:\t%v\n"+ + " Revision:\t%v\n", + git.Repository, git.Revision) +} + +func printSecretVolumeSource(secret *corev1.SecretVolumeSource, w describehelper.PrefixWriter) { + optional := secret.Optional != nil && *secret.Optional + w.Write(IndentLevelTwo, "Type:\tSecret (a volume populated by a Secret)\n"+ + " SecretName:\t%v\n"+ + " Optional:\t%v\n", + secret.SecretName, optional) +} + +func printConfigMapVolumeSource(configMap *corev1.ConfigMapVolumeSource, w describehelper.PrefixWriter) { + optional := configMap.Optional != nil && *configMap.Optional + w.Write(IndentLevelTwo, "Type:\tConfigMap (a volume populated by a ConfigMap)\n"+ + " Name:\t%v\n"+ + " Optional:\t%v\n", + configMap.Name, optional) +} + +func printProjectedVolumeSource(projected *corev1.ProjectedVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tProjected (a volume that contains injected data from multiple sources)\n") + for _, source := range projected.Sources { + if source.Secret != nil { + w.Write(IndentLevelTwo, "SecretName:\t%v\n"+ + " SecretOptionalName:\t%v\n", + source.Secret.Name, source.Secret.Optional) + continue + } + if source.DownwardAPI != nil { + w.Write(IndentLevelTwo, "DownwardAPI:\ttrue\n") + continue + } + if source.ConfigMap != nil { + w.Write(IndentLevelTwo, "ConfigMapName:\t%v\n"+ + " ConfigMapOptional:\t%v\n", + source.ConfigMap.Name, source.ConfigMap.Optional) + continue + } + if source.ServiceAccountToken != nil { + w.Write(IndentLevelTwo, "TokenExpirationSeconds:\t%d\n", + *source.ServiceAccountToken.ExpirationSeconds) + } + } +} + +func printNFSVolumeSource(nfs *corev1.NFSVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tNFS (an NFS mount that lasts the lifetime of a pod)\n"+ + " Server:\t%v\n"+ + " Path:\t%v\n"+ + " ReadOnly:\t%v\n", + nfs.Server, nfs.Path, nfs.ReadOnly) +} + +func printQuobyteVolumeSource(quobyte *corev1.QuobyteVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tQuobyte (a Quobyte mount on the host that shares a pod's lifetime)\n"+ + " Registry:\t%v\n"+ + " Volume:\t%v\n"+ + " ReadOnly:\t%v\n", + quobyte.Registry, quobyte.Volume, quobyte.ReadOnly) +} + +func printPortworxVolumeSource(pwxVolume *corev1.PortworxVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tPortworxVolume (a Portworx Volume resource)\n"+ + " VolumeID:\t%v\n", + pwxVolume.VolumeID) +} + +func printISCSIVolumeSource(iscsi *corev1.ISCSIVolumeSource, w describehelper.PrefixWriter) { + initiator := "" + if iscsi.InitiatorName != nil { + initiator = *iscsi.InitiatorName + } + w.Write(IndentLevelTwo, "Type:\tISCSI (an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod)\n"+ + " TargetPortal:\t%v\n"+ + " IQN:\t%v\n"+ + " Lun:\t%v\n"+ + " ISCSIInterface\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n"+ + " Portals:\t%v\n"+ + " DiscoveryCHAPAuth:\t%v\n"+ + " SessionCHAPAuth:\t%v\n"+ + " SecretRef:\t%v\n"+ + " InitiatorName:\t%v\n", + iscsi.TargetPortal, iscsi.IQN, iscsi.Lun, iscsi.ISCSIInterface, iscsi.FSType, iscsi.ReadOnly, iscsi.Portals, iscsi.DiscoveryCHAPAuth, iscsi.SessionCHAPAuth, iscsi.SecretRef, initiator) +} + +func printGlusterfsVolumeSource(glusterfs *corev1.GlusterfsVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tGlusterfs (a Glusterfs mount on the host that shares a pod's lifetime)\n"+ + " EndpointsName:\t%v\n"+ + " Path:\t%v\n"+ + " ReadOnly:\t%v\n", + glusterfs.EndpointsName, glusterfs.Path, glusterfs.ReadOnly) +} + +func printPersistentVolumeClaimVolumeSource(claim *corev1.PersistentVolumeClaimVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tPersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)\n"+ + " ClaimName:\t%v\n"+ + " ReadOnly:\t%v\n", + claim.ClaimName, claim.ReadOnly) +} + +func printEphemeralVolumeSource(ephemeral *corev1.EphemeralVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tEphemeralVolume (an inline specification for a volume that gets created and deleted with the pod)\n") + if ephemeral.VolumeClaimTemplate != nil { + printPersistentVolumeClaim(describehelper.NewNestedPrefixWriter(w, IndentLevelTwo), + &corev1.PersistentVolumeClaim{ + ObjectMeta: ephemeral.VolumeClaimTemplate.ObjectMeta, + Spec: ephemeral.VolumeClaimTemplate.Spec, + }, false /* not a full PVC */) + } +} + +// printPersistentVolumeClaim is used for both PVCs and PersistentVolumeClaimTemplate. For the latter, +// we need to skip some fields which have no meaning. +func printPersistentVolumeClaim(w describehelper.PrefixWriter, pvc *corev1.PersistentVolumeClaim, isFullPVC bool) { + if isFullPVC { + w.Write(IndentLevelZero, "Name:\t%s\n", pvc.Name) + w.Write(IndentLevelZero, "Namespace:\t%s\n", pvc.Namespace) + } + w.Write(IndentLevelZero, "StorageClass:\t%s\n", storageutil.GetPersistentVolumeClaimClass(pvc)) + if isFullPVC { + if pvc.ObjectMeta.DeletionTimestamp != nil { + w.Write(IndentLevelZero, "Status:\tTerminating (lasts %s)\n", translateTimestampSince(*pvc.ObjectMeta.DeletionTimestamp)) + } else { + w.Write(IndentLevelZero, "Status:\t%v\n", pvc.Status.Phase) + } + } + w.Write(IndentLevelZero, "Volume:\t%s\n", pvc.Spec.VolumeName) + printLabelsMultiline(w, "Labels", pvc.Labels) + if isFullPVC { + w.Write(IndentLevelZero, "Finalizers:\t%v\n", pvc.ObjectMeta.Finalizers) + } + storage := pvc.Spec.Resources.Requests[corev1.ResourceStorage] + capacity := "" + accessModes := "" + if pvc.Spec.VolumeName != "" { + accessModes = storageutil.GetAccessModesAsString(pvc.Status.AccessModes) + storage = pvc.Status.Capacity[corev1.ResourceStorage] + capacity = storage.String() + } + w.Write(IndentLevelZero, "Capacity:\t%s\n", capacity) + w.Write(IndentLevelZero, "Access Modes:\t%s\n", accessModes) + if pvc.Spec.VolumeMode != nil { + w.Write(IndentLevelZero, "VolumeMode:\t%v\n", *pvc.Spec.VolumeMode) + } + if pvc.Spec.DataSource != nil { + w.Write(IndentLevelZero, "DataSource:\n") + if pvc.Spec.DataSource.APIGroup != nil { + w.Write(IndentLevelOne, "APIGroup:\t%v\n", *pvc.Spec.DataSource.APIGroup) + } + w.Write(IndentLevelOne, "Kind:\t%v\n", pvc.Spec.DataSource.Kind) + w.Write(IndentLevelOne, "Name:\t%v\n", pvc.Spec.DataSource.Name) + } +} + +// translateTimestampSince returns the elapsed time since timestamp in +// human-readable approximation. +func translateTimestampSince(timestamp metav1.Time) string { + if timestamp.IsZero() { + return "" + } + + return duration.HumanDuration(time.Since(timestamp.Time)) +} + +func printRBDVolumeSource(rbd *corev1.RBDVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tRBD (a Rados Block Device mount on the host that shares a pod's lifetime)\n"+ + " CephMonitors:\t%v\n"+ + " RBDImage:\t%v\n"+ + " FSType:\t%v\n"+ + " RBDPool:\t%v\n"+ + " RadosUser:\t%v\n"+ + " Keyring:\t%v\n"+ + " SecretRef:\t%v\n"+ + " ReadOnly:\t%v\n", + rbd.CephMonitors, rbd.RBDImage, rbd.FSType, rbd.RBDPool, rbd.RadosUser, rbd.Keyring, rbd.SecretRef, rbd.ReadOnly) +} + +func printDownwardAPIVolumeSource(d *corev1.DownwardAPIVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tDownwardAPI (a volume populated by information about the pod)\n Items:\n") + for _, mapping := range d.Items { + if mapping.FieldRef != nil { + w.Write(IndentLevelThree, "%v -> %v\n", mapping.FieldRef.FieldPath, mapping.Path) + } + if mapping.ResourceFieldRef != nil { + w.Write(IndentLevelThree, "%v -> %v\n", mapping.ResourceFieldRef.Resource, mapping.Path) + } + } +} + +func printAzureDiskVolumeSource(d *corev1.AzureDiskVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tAzureDisk (an Azure Data Disk mount on the host and bind mount to the pod)\n"+ + " DiskName:\t%v\n"+ + " DiskURI:\t%v\n"+ + " Kind: \t%v\n"+ + " FSType:\t%v\n"+ + " CachingMode:\t%v\n"+ + " ReadOnly:\t%v\n", + d.DiskName, d.DataDiskURI, *d.Kind, *d.FSType, *d.CachingMode, *d.ReadOnly) +} + +func printVsphereVolumeSource(vsphere *corev1.VsphereVirtualDiskVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tvSphereVolume (a Persistent Disk resource in vSphere)\n"+ + " VolumePath:\t%v\n"+ + " FSType:\t%v\n"+ + " StoragePolicyName:\t%v\n", + vsphere.VolumePath, vsphere.FSType, vsphere.StoragePolicyName) +} + +func printPhotonPersistentDiskVolumeSource(photon *corev1.PhotonPersistentDiskVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tPhotonPersistentDisk (a Persistent Disk resource in photon platform)\n"+ + " PdID:\t%v\n"+ + " FSType:\t%v\n", + photon.PdID, photon.FSType) +} + +func printCinderVolumeSource(cinder *corev1.CinderVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tCinder (a Persistent Disk resource in OpenStack)\n"+ + " VolumeID:\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n"+ + " SecretRef:\t%v\n", + cinder.VolumeID, cinder.FSType, cinder.ReadOnly, cinder.SecretRef) +} + +func printScaleIOVolumeSource(sio *corev1.ScaleIOVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tScaleIO (a persistent volume backed by a block device in ScaleIO)\n"+ + " Gateway:\t%v\n"+ + " System:\t%v\n"+ + " Protection Domain:\t%v\n"+ + " Storage Pool:\t%v\n"+ + " Storage Mode:\t%v\n"+ + " VolumeName:\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n", + sio.Gateway, sio.System, sio.ProtectionDomain, sio.StoragePool, sio.StorageMode, sio.VolumeName, sio.FSType, sio.ReadOnly) +} + +func printCephFSVolumeSource(cephfs *corev1.CephFSVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tCephFS (a CephFS mount on the host that shares a pod's lifetime)\n"+ + " Monitors:\t%v\n"+ + " Path:\t%v\n"+ + " User:\t%v\n"+ + " SecretFile:\t%v\n"+ + " SecretRef:\t%v\n"+ + " ReadOnly:\t%v\n", + cephfs.Monitors, cephfs.Path, cephfs.User, cephfs.SecretFile, cephfs.SecretRef, cephfs.ReadOnly) +} + +func printStorageOSVolumeSource(storageos *corev1.StorageOSVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tStorageOS (a StorageOS Persistent Disk resource)\n"+ + " VolumeName:\t%v\n"+ + " VolumeNamespace:\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n", + storageos.VolumeName, storageos.VolumeNamespace, storageos.FSType, storageos.ReadOnly) +} + +func printFCVolumeSource(fc *corev1.FCVolumeSource, w describehelper.PrefixWriter) { + lun := "" + if fc.Lun != nil { + lun = strconv.Itoa(int(*fc.Lun)) + } + w.Write(IndentLevelTwo, "Type:\tFC (a Fibre Channel disk)\n"+ + " TargetWWNs:\t%v\n"+ + " LUN:\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n", + strings.Join(fc.TargetWWNs, ", "), lun, fc.FSType, fc.ReadOnly) +} + +func printAzureFileVolumeSource(azureFile *corev1.AzureFileVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tAzureFile (an Azure File Service mount on the host and bind mount to the pod)\n"+ + " SecretName:\t%v\n"+ + " ShareName:\t%v\n"+ + " ReadOnly:\t%v\n", + azureFile.SecretName, azureFile.ShareName, azureFile.ReadOnly) +} + +func printFlexVolumeSource(flex *corev1.FlexVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tFlexVolume (a generic volume resource that is provisioned/attached using an exec based plugin)\n"+ + " Driver:\t%v\n"+ + " FSType:\t%v\n"+ + " SecretRef:\t%v\n"+ + " ReadOnly:\t%v\n"+ + " Options:\t%v\n", + flex.Driver, flex.FSType, flex.SecretRef, flex.ReadOnly, flex.Options) +} + +func printFlockerVolumeSource(flocker *corev1.FlockerVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tFlocker (a Flocker volume mounted by the Flocker agent)\n"+ + " DatasetName:\t%v\n"+ + " DatasetUUID:\t%v\n", + flocker.DatasetName, flocker.DatasetUUID) +} + +func printCSIVolumeSource(csi *corev1.CSIVolumeSource, w describehelper.PrefixWriter) { + var readOnly bool + var fsType string + if csi.ReadOnly != nil && *csi.ReadOnly { + readOnly = true + } + if csi.FSType != nil { + fsType = *csi.FSType + } + w.Write(IndentLevelTwo, "Type:\tCSI (a Container Storage Interface (CSI) volume source)\n"+ + " Driver:\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n", + csi.Driver, fsType, readOnly) + printCSIPersistentVolumeAttributesMultiline(w, "VolumeAttributes", csi.VolumeAttributes) +} + +func printCSIPersistentVolumeAttributesMultiline(w describehelper.PrefixWriter, title string, annotations map[string]string) { + printCSIPersistentVolumeAttributesMultilineIndent(w, "", title, "\t", annotations, sets.New[string]()) +} + +func printCSIPersistentVolumeAttributesMultilineIndent(w describehelper.PrefixWriter, initialIndent, title, innerIndent string, attributes map[string]string, skip sets.Set[string]) { + w.Write(IndentLevelTwo, "%s%s:%s", initialIndent, title, innerIndent) + + if len(attributes) == 0 { + w.WriteLine("") + return + } + + // to print labels in the sorted order + keys := make([]string, 0, len(attributes)) + for key := range attributes { + if skip.Has(key) { + continue + } + keys = append(keys, key) + } + if len(attributes) == 0 { + w.WriteLine("") + return + } + sort.Strings(keys) + + for i, key := range keys { + if i != 0 { + w.Write(IndentLevelTwo, initialIndent) + w.Write(IndentLevelTwo, innerIndent) + } + line := fmt.Sprintf("%s=%s", key, attributes[key]) + if len(line) > maxAnnotationLen { + w.Write(IndentLevelTwo, "%s...\n", line[:maxAnnotationLen]) + } else { + w.Write(IndentLevelTwo, "%s\n", line) + } + } +} diff --git a/pkg/cmd/list/helpers.go b/pkg/cmd/list/helpers.go new file mode 100644 index 0000000..d27f72f --- /dev/null +++ b/pkg/cmd/list/helpers.go @@ -0,0 +1,47 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "errors" + "os" + "strconv" +) + +const ( + defaultListRequestLimit = 100 + KjobctlListRequestLimitEnvName = "KJOBCTL_LIST_REQUEST_LIMIT" +) + +var ( + invalidListRequestLimitError = errors.New("invalid list request limit") +) + +func listRequestLimit() (int64, error) { + listRequestLimitEnv := os.Getenv(KjobctlListRequestLimitEnvName) + + if len(listRequestLimitEnv) == 0 { + return defaultListRequestLimit, nil + } + + limit, err := strconv.ParseInt(listRequestLimitEnv, 10, 64) + if err != nil { + return 0, invalidListRequestLimitError + } + + return limit, nil +} diff --git a/pkg/cmd/list/list.go b/pkg/cmd/list/list.go new file mode 100644 index 0000000..79394c9 --- /dev/null +++ b/pkg/cmd/list/list.go @@ -0,0 +1,50 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/clock" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" +) + +var ( + listExample = templates.Examples(` + # List Job + kjobctl list job + `) +) + +func NewListCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams, clock clock.Clock) *cobra.Command { + cmd := &cobra.Command{ + Use: "list", + Short: "Display resources", + Example: listExample, + SuggestFor: []string{"ps"}, + } + + cmd.AddCommand(NewJobCmd(clientGetter, streams, clock)) + cmd.AddCommand(NewInteractiveCmd(clientGetter, streams, clock)) + cmd.AddCommand(NewRayJobCmd(clientGetter, streams, clock)) + cmd.AddCommand(NewRayClusterCmd(clientGetter, streams, clock)) + cmd.AddCommand(NewSlurmCmd(clientGetter, streams, clock)) + + return cmd +} diff --git a/pkg/cmd/list/list_interactive.go b/pkg/cmd/list/list_interactive.go new file mode 100644 index 0000000..fade0ac --- /dev/null +++ b/pkg/cmd/list/list_interactive.go @@ -0,0 +1,220 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/client-go/kubernetes/scheme" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/clock" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + interactiveExample = templates.Examples(` + # List Interactive + kjobctl list interactive + + # List Interactive with profile filter + kjobctl list interactive --profile my-profile + `) +) + +type InteractiveOptions struct { + Clock clock.Clock + PrintFlags *genericclioptions.PrintFlags + + Limit int64 + AllNamespaces bool + Namespace string + ProfileFilter string + LocalQueueFilter string + FieldSelector string + LabelSelector string + ClusterQueueFilter string + + Client corev1.CoreV1Interface + + genericiooptions.IOStreams +} + +func NewInteractiveOptions(streams genericiooptions.IOStreams, clock clock.Clock) *InteractiveOptions { + return &InteractiveOptions{ + PrintFlags: genericclioptions.NewPrintFlags("").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + Clock: clock, + } +} + +func NewInteractiveCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams, clock clock.Clock) *cobra.Command { + o := NewInteractiveOptions(streams, clock) + + cmd := &cobra.Command{ + Use: "interactive" + + " [--profile PROFILE_NAME]" + + " [--localqueue LOCALQUEUE_NAME]" + + " [--selector key1=value1]" + + " [--field-selector key1=value1]" + + " [--all-namespaces]", + DisableFlagsInUseLine: true, + Short: "List Interactive", + Example: interactiveExample, + RunE: func(cmd *cobra.Command, _ []string) error { + cmd.SilenceUsage = true + err := o.Complete(clientGetter) + if err != nil { + return err + } + return o.Run(cmd.Context()) + }, + } + + o.PrintFlags.AddFlags(cmd) + + util.AddAllNamespacesFlagVar(cmd, &o.AllNamespaces) + util.AddFieldSelectorFlagVar(cmd, &o.FieldSelector) + util.AddLabelSelectorFlagVar(cmd, &o.LabelSelector) + util.AddProfileFlagVar(cmd, &o.ProfileFilter) + util.AddLocalQueueFlagVar(cmd, &o.LocalQueueFilter) + + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("profile", completion.ApplicationProfileNameFunc(clientGetter))) + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("localqueue", completion.LocalQueueNameFunc(clientGetter))) + + return cmd +} + +// Complete completes all the required options +func (o *InteractiveOptions) Complete(clientGetter util.ClientGetter) error { + var err error + + o.Limit, err = listRequestLimit() + if err != nil { + return err + } + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + clientset, err := clientGetter.K8sClientset() + if err != nil { + return err + } + + o.Client = clientset.CoreV1() + + return nil +} + +func (o *InteractiveOptions) ToPrinter(headers bool) (printers.ResourcePrinterFunc, error) { + if !o.PrintFlags.OutputFlagSpecified() { + printer := newInteractiveTablePrinter(). + WithNamespace(o.AllNamespaces). + WithHeaders(headers). + WithClock(o.Clock) + return printer.PrintObj, nil + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return nil, err + } + + return printer.PrintObj, nil +} + +// Run performs the list operation. +func (o *InteractiveOptions) Run(ctx context.Context) error { + var totalCount int + + namespace := o.Namespace + if o.AllNamespaces { + namespace = "" + } + + opts := metav1.ListOptions{ + FieldSelector: o.FieldSelector, + Limit: o.Limit, + } + + if len(o.ProfileFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s=%s", constants.ProfileLabel, o.ProfileFilter) + } else { + opts.LabelSelector = constants.ProfileLabel + } + if len(o.LocalQueueFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s=%s", opts.LabelSelector, kueueconstants.QueueLabel, o.LocalQueueFilter) + } + if len(o.LabelSelector) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s", opts.LabelSelector, o.LabelSelector) + } + + tabWriter := printers.GetNewTabWriter(o.Out) + + for { + headers := totalCount == 0 + + list, err := o.Client.Pods(namespace).List(ctx, opts) + if err != nil { + return err + } + + totalCount += len(list.Items) + + printer, err := o.ToPrinter(headers) + if err != nil { + return err + } + + if err := printer.PrintObj(list, tabWriter); err != nil { + return err + } + + if list.Continue != "" { + opts.Continue = list.Continue + continue + } + + if totalCount == 0 { + if !o.AllNamespaces { + fmt.Fprintf(o.ErrOut, "No resources found in %s namespace.\n", o.Namespace) + } else { + fmt.Fprintln(o.ErrOut, "No resources found") + } + return nil + } + + if err := tabWriter.Flush(); err != nil { + return err + } + + return nil + } +} diff --git a/pkg/cmd/list/list_interactive_printer.go b/pkg/cmd/list/list_interactive_printer.go new file mode 100644 index 0000000..475d118 --- /dev/null +++ b/pkg/cmd/list/list_interactive_printer.go @@ -0,0 +1,105 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "errors" + "io" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/duration" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/utils/clock" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +type listInteractivePrinter struct { + clock clock.Clock + printOptions printers.PrintOptions +} + +var _ printers.ResourcePrinter = (*listInteractivePrinter)(nil) + +func (p *listInteractivePrinter) PrintObj(obj runtime.Object, out io.Writer) error { + printer := printers.NewTablePrinter(p.printOptions) + + list, ok := obj.(*corev1.PodList) + if !ok { + return errors.New("invalid object type") + } + + table := &metav1.Table{ + ColumnDefinitions: []metav1.TableColumnDefinition{ + {Name: "Name", Type: "string", Format: "name"}, + {Name: "Profile", Type: "string"}, + {Name: "Local Queue", Type: "string"}, + {Name: "Status", Type: "string"}, + {Name: "Age", Type: "string"}, + }, + Rows: p.printPodList(list), + } + + return printer.PrintObj(table, out) +} + +func (p *listInteractivePrinter) WithNamespace(f bool) *listInteractivePrinter { + p.printOptions.WithNamespace = f + return p +} + +func (p *listInteractivePrinter) WithHeaders(f bool) *listInteractivePrinter { + p.printOptions.NoHeaders = !f + return p +} + +func (p *listInteractivePrinter) WithClock(c clock.Clock) *listInteractivePrinter { + p.clock = c + return p +} + +func newInteractiveTablePrinter() *listInteractivePrinter { + return &listInteractivePrinter{ + clock: clock.RealClock{}, + } +} + +func (p *listInteractivePrinter) printPodList(list *corev1.PodList) []metav1.TableRow { + rows := make([]metav1.TableRow, len(list.Items)) + for index := range list.Items { + rows[index] = p.printPod(&list.Items[index]) + } + return rows +} + +func (p *listInteractivePrinter) printPod(pod *corev1.Pod) metav1.TableRow { + row := metav1.TableRow{ + Object: runtime.RawExtension{Object: pod}, + } + + row.Cells = []any{ + pod.Name, + pod.ObjectMeta.Labels[constants.ProfileLabel], + pod.ObjectMeta.Labels[kueueconstants.QueueLabel], + pod.Status.Phase, + duration.HumanDuration(p.clock.Since(pod.CreationTimestamp.Time)), + } + return row +} diff --git a/pkg/cmd/list/list_interactive_test.go b/pkg/cmd/list/list_interactive_test.go new file mode 100644 index 0000000..dbce757 --- /dev/null +++ b/pkg/cmd/list/list_interactive_test.go @@ -0,0 +1,306 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/client-go/kubernetes/fake" + kubetesting "k8s.io/client-go/testing" + testingclock "k8s.io/utils/clock/testing" + + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestInteractiveCmd(t *testing.T) { + testStartTime := time.Now() + + testCases := map[string]struct { + ns string + objs []runtime.Object + args []string + wantOut string + wantOutErr string + wantErr error + }{ + "should print only kjobctl pods": { + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakePod("i1", "ns1"). + Profile("profile1"). + LocalQueue("lq1"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + wrappers.MakePod("j2", "ns2"). + LocalQueue("lq2"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE STATUS AGE +i1 profile1 lq1 Succeeded 60m +`, + }, + "should print interactive list with namespace filter": { + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakePod("i1", "ns1"). + Profile("profile1"). + LocalQueue("lq1"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + wrappers.MakePod("j2", "ns2"). + Profile("profile2"). + LocalQueue("lq2"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE STATUS AGE +i1 profile1 lq1 Succeeded 60m +`, + }, + "should print interactive list with profile filter": { + args: []string{"--profile", "profile1"}, + objs: []runtime.Object{ + wrappers.MakePod("i1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq1"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + wrappers.MakePod("i2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("lq2"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE STATUS AGE +i1 profile1 lq1 Succeeded 60m +`, + }, + "should print interactive list with profile filter (short flag)": { + args: []string{"-p", "profile1"}, + objs: []runtime.Object{ + wrappers.MakePod("i1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq1"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + wrappers.MakePod("i2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("lq2"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE STATUS AGE +i1 profile1 lq1 Succeeded 60m +`, + }, + "should print interactive list with localqueue filter": { + args: []string{"--localqueue", "lq1"}, + objs: []runtime.Object{ + wrappers.MakePod("i1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq1"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + wrappers.MakePod("i2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq2"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE STATUS AGE +i1 profile1 lq1 Succeeded 60m +`, + }, + "should print interactive list with localqueue filter (short flag)": { + args: []string{"-q", "lq1"}, + objs: []runtime.Object{ + wrappers.MakePod("i1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq1"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + wrappers.MakePod("i2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq2"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE STATUS AGE +i1 profile1 lq1 Succeeded 60m +`, + }, + "should print interactive list with label selector filter": { + args: []string{"--selector", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakePod("i1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq1"). + Label("foo", "bar"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + wrappers.MakePod("i2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("lq2"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE STATUS AGE +i1 profile1 lq1 Succeeded 60m +`, + }, + "should print interactive list with label selector filter (short flag)": { + args: []string{"-l", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakePod("i1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq1"). + Label("foo", "bar"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + wrappers.MakePod("i2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("lq2"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE STATUS AGE +i1 profile1 lq1 Succeeded 60m +`, + }, + "should print interactive list with field selector filter": { + args: []string{"--field-selector", "metadata.name=i1"}, + objs: []runtime.Object{ + wrappers.MakePod("i1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq1"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + wrappers.MakePod("i2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("lq2"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE STATUS AGE +i1 profile1 lq1 Succeeded 60m +`, + }, + "should print not found error": { + wantOutErr: fmt.Sprintf("No resources found in %s namespace.\n", metav1.NamespaceDefault), + }, + "should print not found error with all-namespaces filter": { + args: []string{"-A"}, + wantOutErr: "No resources found\n", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := fake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("list", "pods", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + listAction := action.(kubetesting.ListActionImpl) + fieldsSelector := listAction.GetListRestrictions().Fields + + obj, err := clientset.Tracker().List(listAction.GetResource(), listAction.GetKind(), listAction.Namespace) + podList := obj.(*corev1.PodList) + + filtered := make([]corev1.Pod, 0, len(podList.Items)) + for _, item := range podList.Items { + fieldsSet := fields.Set{ + "metadata.name": item.Name, + } + if fieldsSelector.Matches(fieldsSet) { + filtered = append(filtered, item) + } + } + podList.Items = filtered + return true, podList, err + }) + + tcg := cmdtesting.NewTestClientGetter().WithK8sClientset(clientset) + if len(tc.ns) > 0 { + tcg.WithNamespace(tc.ns) + } + + cmd := NewInteractiveCmd(tcg, streams, testingclock.NewFakeClock(testStartTime)) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + if diff := cmp.Diff(tc.wantErr, gotErr, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/list/list_job.go b/pkg/cmd/list/list_job.go new file mode 100644 index 0000000..7d63438 --- /dev/null +++ b/pkg/cmd/list/list_job.go @@ -0,0 +1,223 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/client-go/kubernetes/scheme" + batchv1 "k8s.io/client-go/kubernetes/typed/batch/v1" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/clock" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + jobExample = templates.Examples(` + # List Job + kjobctl list job + + # List Job with profile filter + kjobctl list job --profile my-profile + `) +) + +type JobOptions struct { + Clock clock.Clock + PrintFlags *genericclioptions.PrintFlags + + Limit int64 + AllNamespaces bool + Namespace string + ProfileFilter string + LocalQueueFilter string + FieldSelector string + LabelSelector string + + Client batchv1.BatchV1Interface + + genericiooptions.IOStreams +} + +func NewJobOptions(streams genericiooptions.IOStreams, clock clock.Clock) *JobOptions { + return &JobOptions{ + PrintFlags: genericclioptions.NewPrintFlags("").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + Clock: clock, + } +} + +func NewJobCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams, clock clock.Clock) *cobra.Command { + o := NewJobOptions(streams, clock) + + cmd := &cobra.Command{ + Use: "job" + + " [--profile PROFILE_NAME]" + + " [--localqueue LOCALQUEUE_NAME]" + + " [--selector key1=value1]" + + " [--field-selector key1=value1]" + + " [--all-namespaces]", + DisableFlagsInUseLine: true, + Short: "List Job", + Example: jobExample, + RunE: func(cmd *cobra.Command, _ []string) error { + cmd.SilenceUsage = true + err := o.Complete(clientGetter) + if err != nil { + return err + } + return o.Run(cmd.Context()) + }, + } + + o.PrintFlags.AddFlags(cmd) + + util.AddAllNamespacesFlagVar(cmd, &o.AllNamespaces) + util.AddFieldSelectorFlagVar(cmd, &o.FieldSelector) + util.AddLabelSelectorFlagVar(cmd, &o.LabelSelector) + util.AddProfileFlagVar(cmd, &o.ProfileFilter) + util.AddLocalQueueFlagVar(cmd, &o.LocalQueueFilter) + + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("profile", completion.ApplicationProfileNameFunc(clientGetter))) + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("localqueue", completion.LocalQueueNameFunc(clientGetter))) + + return cmd +} + +// Complete completes all the required options +func (o *JobOptions) Complete(clientGetter util.ClientGetter) error { + var err error + + o.Limit, err = listRequestLimit() + if err != nil { + return err + } + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + clientset, err := clientGetter.K8sClientset() + if err != nil { + return err + } + + o.Client = clientset.BatchV1() + + return nil +} + +func (o *JobOptions) ToPrinter(headers bool) (printers.ResourcePrinterFunc, error) { + if !o.PrintFlags.OutputFlagSpecified() { + printer := newJobTablePrinter(). + WithNamespace(o.AllNamespaces). + WithHeaders(headers). + WithClock(o.Clock) + return printer.PrintObj, nil + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return nil, err + } + + return printer.PrintObj, nil +} + +// Run performs the list operation. +func (o *JobOptions) Run(ctx context.Context) error { + var totalCount int + + namespace := o.Namespace + if o.AllNamespaces { + namespace = "" + } + + opts := metav1.ListOptions{ + FieldSelector: o.FieldSelector, + Limit: o.Limit, + } + + if len(o.ProfileFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s=%s", constants.ProfileLabel, o.ProfileFilter) + } else { + opts.LabelSelector = constants.ProfileLabel + } + + opts.LabelSelector = fmt.Sprintf("%s,%s=%s", opts.LabelSelector, constants.ModeLabel, v1alpha1.JobMode) + + if len(o.LocalQueueFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s=%s", opts.LabelSelector, kueueconstants.QueueLabel, o.LocalQueueFilter) + } + if len(o.LabelSelector) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s", opts.LabelSelector, o.LabelSelector) + } + + tabWriter := printers.GetNewTabWriter(o.Out) + + for { + headers := totalCount == 0 + + list, err := o.Client.Jobs(namespace).List(ctx, opts) + if err != nil { + return err + } + + totalCount += len(list.Items) + + printer, err := o.ToPrinter(headers) + if err != nil { + return err + } + + if err := printer.PrintObj(list, tabWriter); err != nil { + return err + } + + if list.Continue != "" { + opts.Continue = list.Continue + continue + } + + if totalCount == 0 { + if !o.AllNamespaces { + fmt.Fprintf(o.ErrOut, "No resources found in %s namespace.\n", o.Namespace) + } else { + fmt.Fprintln(o.ErrOut, "No resources found") + } + return nil + } + + if err := tabWriter.Flush(); err != nil { + return err + } + + return nil + } +} diff --git a/pkg/cmd/list/list_job_printer.go b/pkg/cmd/list/list_job_printer.go new file mode 100644 index 0000000..41ae59a --- /dev/null +++ b/pkg/cmd/list/list_job_printer.go @@ -0,0 +1,117 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "errors" + "fmt" + "io" + "time" + + batchv1 "k8s.io/api/batch/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/duration" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/utils/clock" + "k8s.io/utils/ptr" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +type listJobPrinter struct { + clock clock.Clock + printOptions printers.PrintOptions +} + +var _ printers.ResourcePrinter = (*listJobPrinter)(nil) + +func (p *listJobPrinter) PrintObj(obj runtime.Object, out io.Writer) error { + printer := printers.NewTablePrinter(p.printOptions) + + list, ok := obj.(*batchv1.JobList) + if !ok { + return errors.New("invalid object type") + } + + table := &metav1.Table{ + ColumnDefinitions: []metav1.TableColumnDefinition{ + {Name: "Name", Type: "string", Format: "name"}, + {Name: "Profile", Type: "string"}, + {Name: "Local Queue", Type: "string"}, + {Name: "Completions", Type: "string"}, + {Name: "Duration", Type: "string"}, + {Name: "Age", Type: "string"}, + }, + Rows: p.printJobList(list), + } + + return printer.PrintObj(table, out) +} + +func (p *listJobPrinter) WithNamespace(f bool) *listJobPrinter { + p.printOptions.WithNamespace = f + return p +} + +func (p *listJobPrinter) WithHeaders(f bool) *listJobPrinter { + p.printOptions.NoHeaders = !f + return p +} + +func (p *listJobPrinter) WithClock(c clock.Clock) *listJobPrinter { + p.clock = c + return p +} + +func newJobTablePrinter() *listJobPrinter { + return &listJobPrinter{ + clock: clock.RealClock{}, + } +} + +func (p *listJobPrinter) printJobList(list *batchv1.JobList) []metav1.TableRow { + rows := make([]metav1.TableRow, len(list.Items)) + for index := range list.Items { + rows[index] = p.printJob(&list.Items[index]) + } + return rows +} + +func (p *listJobPrinter) printJob(job *batchv1.Job) metav1.TableRow { + row := metav1.TableRow{ + Object: runtime.RawExtension{Object: job}, + } + var durationStr string + if job.Status.StartTime != nil { + completionTime := time.Now() + if job.Status.CompletionTime != nil { + completionTime = job.Status.CompletionTime.Time + } + durationStr = duration.HumanDuration(completionTime.Sub(job.Status.StartTime.Time)) + } + row.Cells = []any{ + job.Name, + job.ObjectMeta.Labels[constants.ProfileLabel], + job.ObjectMeta.Labels[kueueconstants.QueueLabel], + fmt.Sprintf("%d/%d", job.Status.Succeeded, ptr.Deref(job.Spec.Completions, 1)), + durationStr, + duration.HumanDuration(p.clock.Since(job.CreationTimestamp.Time)), + } + return row +} diff --git a/pkg/cmd/list/list_job_test.go b/pkg/cmd/list/list_job_test.go new file mode 100644 index 0000000..e4bc324 --- /dev/null +++ b/pkg/cmd/list/list_job_test.go @@ -0,0 +1,390 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + batchv1 "k8s.io/api/batch/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/client-go/kubernetes/fake" + kubetesting "k8s.io/client-go/testing" + testingclock "k8s.io/utils/clock/testing" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestJobCmd(t *testing.T) { + testStartTime := time.Now() + + testCases := map[string]struct { + ns string + objs []runtime.Object + args []string + wantOut string + wantOutErr string + wantErr error + }{ + "should print only kjobctl jobs": { + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakeJob("j1", "ns1"). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", "ns2"). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print only Job mode jobs": { + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakeJob("j1", "ns1"). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", "ns2"). + LocalQueue("lq2"). + Mode(v1alpha1.SlurmMode). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print job list with namespace filter": { + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakeJob("j1", "ns1"). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", "ns2"). + Profile("profile2"). + Mode(v1alpha1.JobMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print job list with profile filter": { + args: []string{"--profile", "profile1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.JobMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print job list with profile filter (short flag)": { + args: []string{"-p", "profile1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.JobMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print job list with localqueue filter": { + args: []string{"--localqueue", "lq1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.JobMode). + LocalQueue("lq2"). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print job list with localqueue filter (short flag)": { + args: []string{"-q", "lq1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.JobMode). + LocalQueue("lq2"). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print job list with label selector filter": { + args: []string{"--selector", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Label("foo", "bar"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.JobMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print job list with label selector filter (short flag)": { + args: []string{"-l", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Label("foo", "bar"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.JobMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print job list with field selector filter": { + args: []string{"--field-selector", "metadata.name=j1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.JobMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print not found error": { + wantOutErr: fmt.Sprintf("No resources found in %s namespace.\n", metav1.NamespaceDefault), + }, + "should print not found error with all-namespaces filter": { + args: []string{"-A"}, + wantOutErr: "No resources found\n", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := fake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("list", "jobs", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + listAction := action.(kubetesting.ListActionImpl) + fieldsSelector := listAction.GetListRestrictions().Fields + + obj, err := clientset.Tracker().List(listAction.GetResource(), listAction.GetKind(), listAction.Namespace) + jobList := obj.(*batchv1.JobList) + + filtered := make([]batchv1.Job, 0, len(jobList.Items)) + for _, item := range jobList.Items { + fieldsSet := fields.Set{ + "metadata.name": item.Name, + } + if fieldsSelector.Matches(fieldsSet) { + filtered = append(filtered, item) + } + } + jobList.Items = filtered + return true, jobList, err + }) + + tcg := cmdtesting.NewTestClientGetter().WithK8sClientset(clientset) + if len(tc.ns) > 0 { + tcg.WithNamespace(tc.ns) + } + + cmd := NewJobCmd(tcg, streams, testingclock.NewFakeClock(testStartTime)) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + if diff := cmp.Diff(tc.wantErr, gotErr, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/list/list_ray_cluster.go b/pkg/cmd/list/list_ray_cluster.go new file mode 100644 index 0000000..9bf7b2e --- /dev/null +++ b/pkg/cmd/list/list_ray_cluster.go @@ -0,0 +1,220 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "context" + "fmt" + + "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/scheme" + rayv1 "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/typed/ray/v1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/clock" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + rayClusterExample = templates.Examples(` + # List RayCluster + kjobctl list raycluster + + # List RayCluster with profile filter + kjobctl list raycluster --profile my-profile + `) +) + +type RayClusterOptions struct { + Clock clock.Clock + PrintFlags *genericclioptions.PrintFlags + + Limit int64 + AllNamespaces bool + Namespace string + ProfileFilter string + LocalQueueFilter string + FieldSelector string + LabelSelector string + + Client rayv1.RayV1Interface + + genericiooptions.IOStreams +} + +func NewRayClusterOptions(streams genericiooptions.IOStreams, clock clock.Clock) *RayClusterOptions { + return &RayClusterOptions{ + PrintFlags: genericclioptions.NewPrintFlags("").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + Clock: clock, + } +} + +func NewRayClusterCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams, clock clock.Clock) *cobra.Command { + o := NewRayClusterOptions(streams, clock) + + cmd := &cobra.Command{ + Use: "raycluster" + + " [--profile PROFILE_NAME]" + + " [--localqueue LOCALQUEUE_NAME]" + + " [--selector key1=value1]" + + " [--field-selector key1=value1]" + + " [--all-namespaces]", + DisableFlagsInUseLine: true, + Short: "List RayCluster", + Example: rayClusterExample, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + cmd.SilenceUsage = true + err := o.Complete(clientGetter) + if err != nil { + return err + } + return o.Run(cmd.Context()) + }, + } + + o.PrintFlags.AddFlags(cmd) + + util.AddAllNamespacesFlagVar(cmd, &o.AllNamespaces) + util.AddFieldSelectorFlagVar(cmd, &o.FieldSelector) + util.AddLabelSelectorFlagVar(cmd, &o.LabelSelector) + util.AddProfileFlagVar(cmd, &o.ProfileFilter) + util.AddLocalQueueFlagVar(cmd, &o.LocalQueueFilter) + + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("profile", completion.ApplicationProfileNameFunc(clientGetter))) + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("localqueue", completion.LocalQueueNameFunc(clientGetter))) + + return cmd +} + +// Complete completes all the required options +func (o *RayClusterOptions) Complete(clientGetter util.ClientGetter) error { + var err error + + o.Limit, err = listRequestLimit() + if err != nil { + return err + } + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + clientset, err := clientGetter.RayClientset() + if err != nil { + return err + } + + o.Client = clientset.RayV1() + + return nil +} + +func (o *RayClusterOptions) ToPrinter(headers bool) (printers.ResourcePrinterFunc, error) { + if !o.PrintFlags.OutputFlagSpecified() { + printer := newRayClusterTablePrinter(). + WithNamespace(o.AllNamespaces). + WithHeaders(headers). + WithClock(o.Clock) + return printer.PrintObj, nil + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return nil, err + } + + return printer.PrintObj, nil +} + +// Run performs the list operation. +func (o *RayClusterOptions) Run(ctx context.Context) error { + var totalCount int + + namespace := o.Namespace + if o.AllNamespaces { + namespace = "" + } + + opts := metav1.ListOptions{ + FieldSelector: o.FieldSelector, + Limit: o.Limit, + } + + if len(o.ProfileFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s=%s", constants.ProfileLabel, o.ProfileFilter) + } else { + opts.LabelSelector = constants.ProfileLabel + } + if len(o.LocalQueueFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s=%s", opts.LabelSelector, kueueconstants.QueueLabel, o.LocalQueueFilter) + } + if len(o.LabelSelector) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s", opts.LabelSelector, o.LabelSelector) + } + + tabWriter := printers.GetNewTabWriter(o.Out) + + for { + headers := totalCount == 0 + + list, err := o.Client.RayClusters(namespace).List(ctx, opts) + if err != nil { + return err + } + + totalCount += len(list.Items) + + printer, err := o.ToPrinter(headers) + if err != nil { + return err + } + + if err := printer.PrintObj(list, tabWriter); err != nil { + return err + } + + if list.Continue != "" { + opts.Continue = list.Continue + continue + } + + if totalCount == 0 { + if !o.AllNamespaces { + fmt.Fprintf(o.ErrOut, "No resources found in %s namespace.\n", o.Namespace) + } else { + fmt.Fprintln(o.ErrOut, "No resources found") + } + return nil + } + + if err := tabWriter.Flush(); err != nil { + return err + } + + return nil + } +} diff --git a/pkg/cmd/list/list_ray_cluster_printer.go b/pkg/cmd/list/list_ray_cluster_printer.go new file mode 100644 index 0000000..6b4cc24 --- /dev/null +++ b/pkg/cmd/list/list_ray_cluster_printer.go @@ -0,0 +1,114 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "errors" + "io" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/duration" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/utils/clock" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +type listRayClusterPrinter struct { + clock clock.Clock + printOptions printers.PrintOptions +} + +var _ printers.ResourcePrinter = (*listRayClusterPrinter)(nil) + +func (p *listRayClusterPrinter) PrintObj(obj runtime.Object, out io.Writer) error { + printer := printers.NewTablePrinter(p.printOptions) + + list, ok := obj.(*rayv1.RayClusterList) + if !ok { + return errors.New("invalid object type") + } + + table := &metav1.Table{ + ColumnDefinitions: []metav1.TableColumnDefinition{ + {Name: "Name", Type: "string", Format: "name"}, + {Name: "Profile", Type: "string"}, + {Name: "Local Queue", Type: "string"}, + {Name: "Desired Workers", Type: "string"}, + {Name: "Available Workers", Type: "string"}, + {Name: "CPUs", Type: "string"}, + {Name: "Memory", Type: "string"}, + {Name: "GPUs", Type: "string"}, + {Name: "Status", Type: "string"}, + {Name: "Age", Type: "string"}, + }, + Rows: p.printRayClusterList(list), + } + + return printer.PrintObj(table, out) +} + +func (p *listRayClusterPrinter) WithNamespace(f bool) *listRayClusterPrinter { + p.printOptions.WithNamespace = f + return p +} + +func (p *listRayClusterPrinter) WithHeaders(f bool) *listRayClusterPrinter { + p.printOptions.NoHeaders = !f + return p +} + +func (p *listRayClusterPrinter) WithClock(c clock.Clock) *listRayClusterPrinter { + p.clock = c + return p +} + +func newRayClusterTablePrinter() *listRayClusterPrinter { + return &listRayClusterPrinter{ + clock: clock.RealClock{}, + } +} + +func (p *listRayClusterPrinter) printRayClusterList(list *rayv1.RayClusterList) []metav1.TableRow { + rows := make([]metav1.TableRow, len(list.Items)) + for index := range list.Items { + rows[index] = p.printRayCluster(&list.Items[index]) + } + return rows +} + +func (p *listRayClusterPrinter) printRayCluster(rayCluster *rayv1.RayCluster) metav1.TableRow { + row := metav1.TableRow{ + Object: runtime.RawExtension{Object: rayCluster}, + } + row.Cells = []any{ + rayCluster.Name, + rayCluster.ObjectMeta.Labels[constants.ProfileLabel], + rayCluster.ObjectMeta.Labels[kueueconstants.QueueLabel], + rayCluster.Status.DesiredWorkerReplicas, + rayCluster.Status.AvailableWorkerReplicas, + rayCluster.Status.DesiredCPU.String(), + rayCluster.Status.DesiredMemory.String(), + rayCluster.Status.DesiredGPU.String(), + rayCluster.Status.State, + duration.HumanDuration(p.clock.Since(rayCluster.CreationTimestamp.Time)), + } + return row +} diff --git a/pkg/cmd/list/list_ray_cluster_test.go b/pkg/cmd/list/list_ray_cluster_test.go new file mode 100644 index 0000000..de3ab09 --- /dev/null +++ b/pkg/cmd/list/list_ray_cluster_test.go @@ -0,0 +1,381 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/fake" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + kubetesting "k8s.io/client-go/testing" + testingclock "k8s.io/utils/clock/testing" + + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestRayClusterCmd(t *testing.T) { + testStartTime := time.Now().Truncate(time.Second) + + testCases := map[string]struct { + ns string + objs []runtime.Object + args []string + wantOut string + wantOutErr string + wantErr error + }{ + "should print only kjobctl ray clusters": { + args: []string{}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", "ns1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 localqueue1 2 3 5 10Gi 10 ready 120m +`, + }, + "should print ray cluster list with namespace filter": { + args: []string{}, + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", "ns1"). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", "ns2"). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 localqueue1 2 3 5 10Gi 10 ready 120m +`, + }, + "should print ray cluster list with profile filter": { + args: []string{"--profile", "profile1"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 localqueue1 2 3 5 10Gi 10 ready 120m +`, + }, + "should print ray cluster list with profile filter (short flag)": { + args: []string{"-p", "profile1"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 localqueue1 2 3 5 10Gi 10 ready 120m +`, + }, + "should print ray cluster list with localqueue filter": { + args: []string{"--localqueue", "localqueue1"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue2"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 localqueue1 2 3 5 10Gi 10 ready 120m +`, + }, + "should print ray cluster list with localqueue filter (short filter)": { + args: []string{"-q", "localqueue1"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue2"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 localqueue1 2 3 5 10Gi 10 ready 120m +`, + }, + "should print ray cluster list with label selector filter": { + args: []string{"--selector", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault). + Label("foo", "bar"). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 localqueue1 2 3 5 10Gi 10 ready 120m +`, + }, + "should print ray cluster list with label selector filter (short flag)": { + args: []string{"-l", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault). + Label("foo", "bar"). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 localqueue1 2 3 5 10Gi 10 ready 120m +`, + }, + "should print ray cluster list with field selector filter": { + args: []string{"--field-selector", "metadata.name=rc1"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 localqueue1 2 3 5 10Gi 10 ready 120m +`, + }, + "should print not found error": { + args: []string{}, + wantOutErr: fmt.Sprintf("No resources found in %s namespace.\n", metav1.NamespaceDefault), + }, + "should print not found error with all-namespaces filter": { + args: []string{"-A"}, + wantOutErr: "No resources found\n", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := fake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("list", "rayclusters", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + listAction := action.(kubetesting.ListActionImpl) + fieldsSelector := listAction.GetListRestrictions().Fields + + obj, err := clientset.Tracker().List(listAction.GetResource(), listAction.GetKind(), listAction.Namespace) + rayClusterList := obj.(*rayv1.RayClusterList) + + filtered := make([]rayv1.RayCluster, 0, len(rayClusterList.Items)) + for _, item := range rayClusterList.Items { + fieldsSet := fields.Set{ + "metadata.name": item.Name, + } + if fieldsSelector.Matches(fieldsSet) { + filtered = append(filtered, item) + } + } + rayClusterList.Items = filtered + return true, rayClusterList, err + }) + + tcg := cmdtesting.NewTestClientGetter().WithRayClientset(clientset) + if len(tc.ns) > 0 { + tcg.WithNamespace(tc.ns) + } + + cmd := NewRayClusterCmd(tcg, streams, testingclock.NewFakeClock(testStartTime)) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + if diff := cmp.Diff(tc.wantErr, gotErr, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/list/list_ray_job.go b/pkg/cmd/list/list_ray_job.go new file mode 100644 index 0000000..7c08304 --- /dev/null +++ b/pkg/cmd/list/list_ray_job.go @@ -0,0 +1,220 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "context" + "fmt" + + "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/scheme" + rayv1 "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/typed/ray/v1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/clock" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + rayJobExample = templates.Examples(` + # List RayJob + kjobctl list rayjob + + # List RayJob with profile filter + kjobctl list rayjob --profile my-profile + `) +) + +type RayJobOptions struct { + Clock clock.Clock + PrintFlags *genericclioptions.PrintFlags + + Limit int64 + AllNamespaces bool + Namespace string + ProfileFilter string + LocalQueueFilter string + FieldSelector string + LabelSelector string + + Client rayv1.RayV1Interface + + genericiooptions.IOStreams +} + +func NewRayJobOptions(streams genericiooptions.IOStreams, clock clock.Clock) *RayJobOptions { + return &RayJobOptions{ + PrintFlags: genericclioptions.NewPrintFlags("").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + Clock: clock, + } +} + +func NewRayJobCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams, clock clock.Clock) *cobra.Command { + o := NewRayJobOptions(streams, clock) + + cmd := &cobra.Command{ + Use: "rayjob" + + " [--profile PROFILE_NAME]" + + " [--localqueue LOCALQUEUE_NAME]" + + " [--selector key1=value1]" + + " [--field-selector key1=value1]" + + " [--all-namespaces]", + DisableFlagsInUseLine: true, + Short: "List RayJob", + Example: rayJobExample, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + cmd.SilenceUsage = true + err := o.Complete(clientGetter) + if err != nil { + return err + } + return o.Run(cmd.Context()) + }, + } + + o.PrintFlags.AddFlags(cmd) + + util.AddAllNamespacesFlagVar(cmd, &o.AllNamespaces) + util.AddFieldSelectorFlagVar(cmd, &o.FieldSelector) + util.AddLabelSelectorFlagVar(cmd, &o.LabelSelector) + util.AddProfileFlagVar(cmd, &o.ProfileFilter) + util.AddLocalQueueFlagVar(cmd, &o.LocalQueueFilter) + + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("profile", completion.ApplicationProfileNameFunc(clientGetter))) + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("localqueue", completion.LocalQueueNameFunc(clientGetter))) + + return cmd +} + +// Complete completes all the required options +func (o *RayJobOptions) Complete(clientGetter util.ClientGetter) error { + var err error + + o.Limit, err = listRequestLimit() + if err != nil { + return err + } + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + clientset, err := clientGetter.RayClientset() + if err != nil { + return err + } + + o.Client = clientset.RayV1() + + return nil +} + +func (o *RayJobOptions) ToPrinter(headers bool) (printers.ResourcePrinterFunc, error) { + if !o.PrintFlags.OutputFlagSpecified() { + printer := newRayJobTablePrinter(). + WithNamespace(o.AllNamespaces). + WithHeaders(headers). + WithClock(o.Clock) + return printer.PrintObj, nil + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return nil, err + } + + return printer.PrintObj, nil +} + +// Run performs the list operation. +func (o *RayJobOptions) Run(ctx context.Context) error { + var totalCount int + + namespace := o.Namespace + if o.AllNamespaces { + namespace = "" + } + + opts := metav1.ListOptions{ + FieldSelector: o.FieldSelector, + Limit: o.Limit, + } + + if len(o.ProfileFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s=%s", constants.ProfileLabel, o.ProfileFilter) + } else { + opts.LabelSelector = constants.ProfileLabel + } + if len(o.LocalQueueFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s=%s", opts.LabelSelector, kueueconstants.QueueLabel, o.LocalQueueFilter) + } + if len(o.LabelSelector) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s", opts.LabelSelector, o.LabelSelector) + } + + tabWriter := printers.GetNewTabWriter(o.Out) + + for { + headers := totalCount == 0 + + list, err := o.Client.RayJobs(namespace).List(ctx, opts) + if err != nil { + return err + } + + totalCount += len(list.Items) + + printer, err := o.ToPrinter(headers) + if err != nil { + return err + } + + if err := printer.PrintObj(list, tabWriter); err != nil { + return err + } + + if list.Continue != "" { + opts.Continue = list.Continue + continue + } + + if totalCount == 0 { + if !o.AllNamespaces { + fmt.Fprintf(o.ErrOut, "No resources found in %s namespace.\n", o.Namespace) + } else { + fmt.Fprintln(o.ErrOut, "No resources found") + } + return nil + } + + if err := tabWriter.Flush(); err != nil { + return err + } + + return nil + } +} diff --git a/pkg/cmd/list/list_ray_job_printer.go b/pkg/cmd/list/list_ray_job_printer.go new file mode 100644 index 0000000..006ad24 --- /dev/null +++ b/pkg/cmd/list/list_ray_job_printer.go @@ -0,0 +1,120 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "errors" + "io" + "time" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/duration" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/utils/clock" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +type listRayJobPrinter struct { + clock clock.Clock + printOptions printers.PrintOptions +} + +var _ printers.ResourcePrinter = (*listRayJobPrinter)(nil) + +func (p *listRayJobPrinter) PrintObj(obj runtime.Object, out io.Writer) error { + printer := printers.NewTablePrinter(p.printOptions) + + list, ok := obj.(*rayv1.RayJobList) + if !ok { + return errors.New("invalid object type") + } + + table := &metav1.Table{ + ColumnDefinitions: []metav1.TableColumnDefinition{ + {Name: "Name", Type: "string", Format: "name"}, + {Name: "Profile", Type: "string"}, + {Name: "Local Queue", Type: "string"}, + {Name: "Ray Cluster Name", Type: "string"}, + {Name: "Job Status", Type: "string"}, + {Name: "Deployment Status", Type: "string"}, + {Name: "Start Time", Type: "dateTime"}, + {Name: "End Time", Type: "string"}, + {Name: "Age", Type: "string"}, + }, + Rows: p.printRayJobList(list), + } + + return printer.PrintObj(table, out) +} + +func (p *listRayJobPrinter) WithNamespace(f bool) *listRayJobPrinter { + p.printOptions.WithNamespace = f + return p +} + +func (p *listRayJobPrinter) WithHeaders(f bool) *listRayJobPrinter { + p.printOptions.NoHeaders = !f + return p +} + +func (p *listRayJobPrinter) WithClock(c clock.Clock) *listRayJobPrinter { + p.clock = c + return p +} + +func newRayJobTablePrinter() *listRayJobPrinter { + return &listRayJobPrinter{ + clock: clock.RealClock{}, + } +} + +func (p *listRayJobPrinter) printRayJobList(list *rayv1.RayJobList) []metav1.TableRow { + rows := make([]metav1.TableRow, len(list.Items)) + for index := range list.Items { + rows[index] = p.printRayJob(&list.Items[index]) + } + return rows +} + +func (p *listRayJobPrinter) printRayJob(rayJob *rayv1.RayJob) metav1.TableRow { + row := metav1.TableRow{ + Object: runtime.RawExtension{Object: rayJob}, + } + var startTime, endTime string + if rayJob.Status.StartTime != nil { + startTime = rayJob.Status.StartTime.Format(time.DateTime) + } + if rayJob.Status.EndTime != nil { + endTime = rayJob.Status.EndTime.Format(time.DateTime) + } + row.Cells = []any{ + rayJob.Name, + rayJob.ObjectMeta.Labels[constants.ProfileLabel], + rayJob.ObjectMeta.Labels[kueueconstants.QueueLabel], + rayJob.Status.RayClusterName, + rayJob.Status.JobStatus, + rayJob.Status.JobDeploymentStatus, + startTime, + endTime, + duration.HumanDuration(p.clock.Since(rayJob.CreationTimestamp.Time)), + } + return row +} diff --git a/pkg/cmd/list/list_ray_job_test.go b/pkg/cmd/list/list_ray_job_test.go new file mode 100644 index 0000000..674b795 --- /dev/null +++ b/pkg/cmd/list/list_ray_job_test.go @@ -0,0 +1,387 @@ +/* +Copyright %s Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/fake" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + kubetesting "k8s.io/client-go/testing" + testingclock "k8s.io/utils/clock/testing" + + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestRayJobCmd(t *testing.T) { + testStartTime := time.Now().Truncate(time.Second) + + testCases := map[string]struct { + ns string + objs []runtime.Object + args []string + wantOut string + wantOutErr string + wantErr error + }{ + "should print only kjobctl ray jobs": { + args: []string{}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", "ns1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 localqueue1 rc1 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print ray job list with namespace filter": { + args: []string{}, + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", "ns1"). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", "ns2"). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 localqueue1 rc1 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print ray job list with profile filter": { + args: []string{"--profile", "profile1"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 localqueue1 rc1 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print ray job list with profile filter (short flag)": { + args: []string{"-p", "profile1"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 localqueue1 rc1 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print ray job list with localqueue filter": { + args: []string{"--localqueue", "localqueue1"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue2"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 localqueue1 rc1 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print ray job list with localqueue filter (short filter)": { + args: []string{"-q", "localqueue1"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue2"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 localqueue1 rc1 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print ray job list with label selector filter": { + args: []string{"--selector", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault). + Label("foo", "bar"). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 localqueue1 rc1 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print ray job list with label selector filter (short flag)": { + args: []string{"-l", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault). + Label("foo", "bar"). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 localqueue1 rc1 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print ray job list with field selector filter": { + args: []string{"--field-selector", "metadata.name=rj1"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 localqueue1 rc1 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print not found error": { + args: []string{}, + wantOutErr: fmt.Sprintf("No resources found in %s namespace.\n", metav1.NamespaceDefault), + }, + "should print not found error with all-namespaces filter": { + args: []string{"-A"}, + wantOutErr: "No resources found\n", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := fake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("list", "rayjobs", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + listAction := action.(kubetesting.ListActionImpl) + fieldsSelector := listAction.GetListRestrictions().Fields + + obj, err := clientset.Tracker().List(listAction.GetResource(), listAction.GetKind(), listAction.Namespace) + rayJobList := obj.(*rayv1.RayJobList) + + filtered := make([]rayv1.RayJob, 0, len(rayJobList.Items)) + for _, item := range rayJobList.Items { + fieldsSet := fields.Set{ + "metadata.name": item.Name, + } + if fieldsSelector.Matches(fieldsSet) { + filtered = append(filtered, item) + } + } + rayJobList.Items = filtered + return true, rayJobList, err + }) + + tcg := cmdtesting.NewTestClientGetter().WithRayClientset(clientset) + if len(tc.ns) > 0 { + tcg.WithNamespace(tc.ns) + } + + cmd := NewRayJobCmd(tcg, streams, testingclock.NewFakeClock(testStartTime)) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + if diff := cmp.Diff(tc.wantErr, gotErr, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/list/list_slurm.go b/pkg/cmd/list/list_slurm.go new file mode 100644 index 0000000..5ac6756 --- /dev/null +++ b/pkg/cmd/list/list_slurm.go @@ -0,0 +1,223 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/client-go/kubernetes/scheme" + batchv1 "k8s.io/client-go/kubernetes/typed/batch/v1" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/clock" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + slurmExample = templates.Examples(` + # List Slurm + kjobctl list slurm + + # List Slurm with profile filter + kjobctl list slurm --profile my-profile + `) +) + +type SlurmOptions struct { + Clock clock.Clock + PrintFlags *genericclioptions.PrintFlags + + Limit int64 + AllNamespaces bool + Namespace string + ProfileFilter string + LocalQueueFilter string + FieldSelector string + LabelSelector string + + Client batchv1.BatchV1Interface + + genericiooptions.IOStreams +} + +func NewSlurmOptions(streams genericiooptions.IOStreams, clock clock.Clock) *SlurmOptions { + return &SlurmOptions{ + PrintFlags: genericclioptions.NewPrintFlags("").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + Clock: clock, + } +} + +func NewSlurmCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams, clock clock.Clock) *cobra.Command { + o := NewSlurmOptions(streams, clock) + + cmd := &cobra.Command{ + Use: "slurm" + + " [--profile PROFILE_NAME]" + + " [--localqueue LOCALQUEUE_NAME]" + + " [--selector key1=value1]" + + " [--field-selector key1=value1]" + + " [--all-namespaces]", + DisableFlagsInUseLine: true, + Short: "List Slurm", + Example: slurmExample, + RunE: func(cmd *cobra.Command, _ []string) error { + cmd.SilenceUsage = true + err := o.Complete(clientGetter) + if err != nil { + return err + } + return o.Run(cmd.Context()) + }, + } + + o.PrintFlags.AddFlags(cmd) + + util.AddAllNamespacesFlagVar(cmd, &o.AllNamespaces) + util.AddFieldSelectorFlagVar(cmd, &o.FieldSelector) + util.AddLabelSelectorFlagVar(cmd, &o.LabelSelector) + util.AddProfileFlagVar(cmd, &o.ProfileFilter) + util.AddLocalQueueFlagVar(cmd, &o.LocalQueueFilter) + + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("profile", completion.ApplicationProfileNameFunc(clientGetter))) + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("localqueue", completion.LocalQueueNameFunc(clientGetter))) + + return cmd +} + +// Complete completes all the required options +func (o *SlurmOptions) Complete(clientGetter util.ClientGetter) error { + var err error + + o.Limit, err = listRequestLimit() + if err != nil { + return err + } + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + clientset, err := clientGetter.K8sClientset() + if err != nil { + return err + } + + o.Client = clientset.BatchV1() + + return nil +} + +func (o *SlurmOptions) ToPrinter(headers bool) (printers.ResourcePrinterFunc, error) { + if !o.PrintFlags.OutputFlagSpecified() { + printer := newJobTablePrinter(). + WithNamespace(o.AllNamespaces). + WithHeaders(headers). + WithClock(o.Clock) + return printer.PrintObj, nil + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return nil, err + } + + return printer.PrintObj, nil +} + +// Run performs the list operation. +func (o *SlurmOptions) Run(ctx context.Context) error { + var totalCount int + + namespace := o.Namespace + if o.AllNamespaces { + namespace = "" + } + + opts := metav1.ListOptions{ + FieldSelector: o.FieldSelector, + Limit: o.Limit, + } + + if len(o.ProfileFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s=%s", constants.ProfileLabel, o.ProfileFilter) + } else { + opts.LabelSelector = constants.ProfileLabel + } + + opts.LabelSelector = fmt.Sprintf("%s,%s=%s", opts.LabelSelector, constants.ModeLabel, v1alpha1.SlurmMode) + + if len(o.LocalQueueFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s=%s", opts.LabelSelector, kueueconstants.QueueLabel, o.LocalQueueFilter) + } + if len(o.LabelSelector) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s", opts.LabelSelector, o.LabelSelector) + } + + tabWriter := printers.GetNewTabWriter(o.Out) + + for { + headers := totalCount == 0 + + list, err := o.Client.Jobs(namespace).List(ctx, opts) + if err != nil { + return err + } + + totalCount += len(list.Items) + + printer, err := o.ToPrinter(headers) + if err != nil { + return err + } + + if err := printer.PrintObj(list, tabWriter); err != nil { + return err + } + + if list.Continue != "" { + opts.Continue = list.Continue + continue + } + + if totalCount == 0 { + if !o.AllNamespaces { + fmt.Fprintf(o.ErrOut, "No resources found in %s namespace.\n", o.Namespace) + } else { + fmt.Fprintln(o.ErrOut, "No resources found") + } + return nil + } + + if err := tabWriter.Flush(); err != nil { + return err + } + + return nil + } +} diff --git a/pkg/cmd/list/list_slurm_test.go b/pkg/cmd/list/list_slurm_test.go new file mode 100644 index 0000000..0bfc814 --- /dev/null +++ b/pkg/cmd/list/list_slurm_test.go @@ -0,0 +1,389 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + batchv1 "k8s.io/api/batch/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/client-go/kubernetes/fake" + kubetesting "k8s.io/client-go/testing" + testingclock "k8s.io/utils/clock/testing" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestSlurmCmd(t *testing.T) { + testStartTime := time.Now() + + testCases := map[string]struct { + ns string + objs []runtime.Object + args []string + wantOut string + wantOutErr string + wantErr error + }{ + "should print only kjobctl jobs": { + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakeJob("j1", "ns1"). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", "ns2"). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print only Slurm mode jobs": { + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakeJob("j1", "ns1"). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", "ns2"). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print slurm job list with namespace filter": { + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakeJob("j1", "ns1"). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", "ns2"). + Profile("profile2"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print slurm job list with profile filter": { + args: []string{"--profile", "profile1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print slurm job list with profile filter (short flag)": { + args: []string{"-p", "profile1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print slurm job list with localqueue filter": { + args: []string{"--localqueue", "lq1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq2"). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print slurm job list with localqueue filter (short flag)": { + args: []string{"-q", "lq1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq2"). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print slurm job list with label selector filter": { + args: []string{"--selector", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Label("foo", "bar"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print slurm job list with label selector filter (short flag)": { + args: []string{"-l", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Label("foo", "bar"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print slurm job list with field selector filter": { + args: []string{"--field-selector", "metadata.name=j1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print not found error": { + wantOutErr: fmt.Sprintf("No resources found in %s namespace.\n", metav1.NamespaceDefault), + }, + "should print not found error with all-namespaces filter": { + args: []string{"-A"}, + wantOutErr: "No resources found\n", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := fake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("list", "jobs", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + listAction := action.(kubetesting.ListActionImpl) + fieldsSelector := listAction.GetListRestrictions().Fields + + obj, err := clientset.Tracker().List(listAction.GetResource(), listAction.GetKind(), listAction.Namespace) + jobList := obj.(*batchv1.JobList) + + filtered := make([]batchv1.Job, 0, len(jobList.Items)) + for _, item := range jobList.Items { + fieldsSet := fields.Set{ + "metadata.name": item.Name, + } + if fieldsSelector.Matches(fieldsSet) { + filtered = append(filtered, item) + } + } + jobList.Items = filtered + return true, jobList, err + }) + + tcg := cmdtesting.NewTestClientGetter().WithK8sClientset(clientset) + if len(tc.ns) > 0 { + tcg.WithNamespace(tc.ns) + } + + cmd := NewSlurmCmd(tcg, streams, testingclock.NewFakeClock(testStartTime)) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + if diff := cmp.Diff(tc.wantErr, gotErr, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/list/list_test.go b/pkg/cmd/list/list_test.go new file mode 100644 index 0000000..f700218 --- /dev/null +++ b/pkg/cmd/list/list_test.go @@ -0,0 +1,206 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + rayfake "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/fake" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + k8sfake "k8s.io/client-go/kubernetes/fake" + testingclock "k8s.io/utils/clock/testing" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestListCmd(t *testing.T) { + testStartTime := time.Now() + + testCases := map[string]struct { + ns string + k8sObjs []runtime.Object + rayObjs []runtime.Object + args []string + wantOut string + wantOutErr string + wantErr error + }{ + "should print job list with all namespaces": { + args: []string{"job", "--all-namespaces"}, + k8sObjs: []runtime.Object{ + wrappers.MakeJob("j1", "ns1"). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", "ns2"). + Profile("profile2"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAMESPACE NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +ns1 j1 profile1 lq1 3/3 60m 60m +ns2 j2 profile2 lq1 3/3 60m 60m +`, + }, + "should print slurm job list with all namespaces": { + args: []string{"slurm", "--all-namespaces"}, + k8sObjs: []runtime.Object{ + wrappers.MakeJob("j1", "ns1"). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", "ns2"). + Profile("profile2"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAMESPACE NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +ns1 j1 profile1 lq1 3/3 60m 60m +ns2 j2 profile2 lq1 3/3 60m 60m +`, + }, + "should print rayjob list with all namespaces": { + args: []string{"rayjob", "--all-namespaces"}, + rayObjs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("lq2"). + JobStatus(rayv1.JobStatusSucceeded). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAMESPACE NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +default rj1 profile1 lq1 rc1 SUCCEEDED Complete %s %s 120m +default rj2 profile2 lq2 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print raycluster list with all namespaces": { + args: []string{"raycluster", "--all-namespaces"}, + rayObjs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("lq2"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAMESPACE NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +default rc1 profile1 lq1 2 3 5 10Gi 10 ready 120m +default rc2 profile2 lq2 2 3 5 10Gi 10 ready 120m +`, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + tcg := cmdtesting.NewTestClientGetter() + if len(tc.ns) > 0 { + tcg.WithNamespace(tc.ns) + } + + tcg.WithK8sClientset(k8sfake.NewSimpleClientset(tc.k8sObjs...)) + tcg.WithRayClientset(rayfake.NewSimpleClientset(tc.rayObjs...)) + + cmd := NewListCmd(tcg, streams, testingclock.NewFakeClock(testStartTime)) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + if diff := cmp.Diff(tc.wantErr, gotErr, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/printcrds/embed/manifest.gz b/pkg/cmd/printcrds/embed/manifest.gz new file mode 100644 index 0000000..8061f42 Binary files /dev/null and b/pkg/cmd/printcrds/embed/manifest.gz differ diff --git a/pkg/cmd/printcrds/printcrds.go b/pkg/cmd/printcrds/printcrds.go new file mode 100644 index 0000000..06f9a5e --- /dev/null +++ b/pkg/cmd/printcrds/printcrds.go @@ -0,0 +1,59 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package printcrds + +import ( + "bytes" + "compress/gzip" + "io" + + "github.com/spf13/cobra" + "k8s.io/kubectl/pkg/util/templates" + + _ "embed" +) + +var ( + crdsExample = templates.Examples(` + # Install or update the kjobctl CRDs + kjobctl printcrds | kubectl apply --server-side -f - + + # Remove the kjobctl CRDs + kjobctl printcrds | kubectl delete --ignore-not-found=true -f - + `) +) + +//go:embed embed/manifest.gz +var crds []byte + +func NewCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "printcrds", + Short: "Print the kjobctl CRDs", + Example: crdsExample, + SilenceUsage: false, + RunE: func(cmd *cobra.Command, args []string) error { + reader, err := gzip.NewReader(bytes.NewReader(crds)) + if err != nil { + return err + } + _, err = io.Copy(cmd.OutOrStdout(), reader) + return err + }, + } + return cmd +} diff --git a/pkg/cmd/printcrds/printcrds_test.go b/pkg/cmd/printcrds/printcrds_test.go new file mode 100644 index 0000000..5824391 --- /dev/null +++ b/pkg/cmd/printcrds/printcrds_test.go @@ -0,0 +1,51 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package printcrds + +import ( + "bytes" + "io" + "os/exec" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestCRDs(t *testing.T) { + kustcmd := exec.Command("../../../bin/kustomize", "build", "../../../config/crd/") + wantOut, err := kustcmd.CombinedOutput() + if err != nil { + t.Fatal(err) + } + + wr := bytes.NewBuffer(nil) + cmd := NewCmd() + cmd.SetOut(wr) + err = cmd.Execute() + if err != nil { + t.Fatal(err) + } + + gotOut, err := io.ReadAll(wr) + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(string(wantOut), string(gotOut)); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } +} diff --git a/pkg/cmd/testing/fake.go b/pkg/cmd/testing/fake.go new file mode 100644 index 0000000..2be6e8b --- /dev/null +++ b/pkg/cmd/testing/fake.go @@ -0,0 +1,148 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + rayversioned "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/resource" + "k8s.io/client-go/dynamic" + k8s "k8s.io/client-go/kubernetes" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/tools/clientcmd" + kueueversioned "sigs.k8s.io/kueue/client-go/clientset/versioned" + + kjobctlversioned "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned" + kjobctlfake "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/fake" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" +) + +type TestClientGetter struct { + util.ClientGetter + + k8sClientset k8s.Interface + kueueClientset kueueversioned.Interface + kjobctlClientset kjobctlversioned.Interface + rayClientset rayversioned.Interface + dynamicClient dynamic.Interface + restClient resource.RESTClient + restConfig *rest.Config + + configFlags *genericclioptions.TestConfigFlags +} + +func NewTestClientGetter() *TestClientGetter { + clientConfig := &clientcmd.DeferredLoadingClientConfig{} + restConfig := &rest.Config{} + + configFlags := genericclioptions.NewTestConfigFlags(). + WithClientConfig(clientConfig). + WithNamespace(metav1.NamespaceDefault) + return &TestClientGetter{ + ClientGetter: util.NewClientGetter(configFlags), + kjobctlClientset: kjobctlfake.NewSimpleClientset(), + k8sClientset: k8sfake.NewSimpleClientset(), + restConfig: restConfig, + configFlags: configFlags, + } +} + +func (cg *TestClientGetter) WithNamespace(ns string) *TestClientGetter { + cg.configFlags.WithNamespace(ns) + return cg +} + +func (cg *TestClientGetter) WithRESTMapper(mapper meta.RESTMapper) *TestClientGetter { + cg.configFlags.WithRESTMapper(mapper) + return cg +} + +func (cg *TestClientGetter) WithRESTConfig(config *rest.Config) *TestClientGetter { + cg.restConfig = config + return cg +} + +func (cg *TestClientGetter) ToRESTConfig() (*rest.Config, error) { + return cg.restConfig, nil +} + +func (cg *TestClientGetter) WithK8sClientset(clientset k8s.Interface) *TestClientGetter { + cg.k8sClientset = clientset + return cg +} + +func (cg *TestClientGetter) WithKueueClientset(clientset kueueversioned.Interface) *TestClientGetter { + cg.kueueClientset = clientset + return cg +} + +func (cg *TestClientGetter) WithRayClientset(clientset rayversioned.Interface) *TestClientGetter { + cg.rayClientset = clientset + return cg +} + +func (cg *TestClientGetter) WithKjobctlClientset(clientset kjobctlversioned.Interface) *TestClientGetter { + cg.kjobctlClientset = clientset + return cg +} + +func (cg *TestClientGetter) WithDynamicClient(dynamicClient dynamic.Interface) *TestClientGetter { + cg.dynamicClient = dynamicClient + return cg +} + +func (cg *TestClientGetter) WithRESTClient(restClient resource.RESTClient) *TestClientGetter { + cg.restClient = restClient + return cg +} + +func (cg *TestClientGetter) K8sClientset() (k8s.Interface, error) { + return cg.k8sClientset, nil +} + +func (cg *TestClientGetter) KueueClientset() (kueueversioned.Interface, error) { + return cg.kueueClientset, nil +} + +func (cg *TestClientGetter) KjobctlClientset() (kjobctlversioned.Interface, error) { + return cg.kjobctlClientset, nil +} + +func (cg *TestClientGetter) RayClientset() (rayversioned.Interface, error) { + return cg.rayClientset, nil +} + +func (cg *TestClientGetter) DynamicClient() (dynamic.Interface, error) { + return cg.dynamicClient, nil +} + +func (cg *TestClientGetter) NewResourceBuilder() *resource.Builder { + return resource.NewFakeBuilder( + func(version schema.GroupVersion) (resource.RESTClient, error) { + return cg.restClient, nil + }, + cg.ToRESTMapper, + func() (restmapper.CategoryExpander, error) { + return resource.FakeCategoryExpander, nil + }, + ) +} diff --git a/pkg/cmd/util/client_getter.go b/pkg/cmd/util/client_getter.go new file mode 100644 index 0000000..e97d653 --- /dev/null +++ b/pkg/cmd/util/client_getter.go @@ -0,0 +1,129 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + rayversioned "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/resource" + "k8s.io/client-go/dynamic" + k8s "k8s.io/client-go/kubernetes" + kueueversioned "sigs.k8s.io/kueue/client-go/clientset/versioned" + + kjobctlversioned "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned" + + _ "k8s.io/client-go/plugin/pkg/client/auth" +) + +type ClientGetter interface { + genericclioptions.RESTClientGetter + + K8sClientset() (k8s.Interface, error) + KueueClientset() (kueueversioned.Interface, error) + KjobctlClientset() (kjobctlversioned.Interface, error) + RayClientset() (rayversioned.Interface, error) + DynamicClient() (dynamic.Interface, error) + NewResourceBuilder() *resource.Builder +} + +type clientGetterImpl struct { + genericclioptions.RESTClientGetter +} + +var _ ClientGetter = (*clientGetterImpl)(nil) + +func NewClientGetter(clientGetter genericclioptions.RESTClientGetter) ClientGetter { + return &clientGetterImpl{ + RESTClientGetter: clientGetter, + } +} + +func (cg *clientGetterImpl) K8sClientset() (k8s.Interface, error) { + config, err := cg.ToRESTConfig() + if err != nil { + return nil, err + } + + config.ContentType = runtime.ContentTypeProtobuf + clientset, err := k8s.NewForConfig(config) + if err != nil { + return nil, err + } + + return clientset, nil +} + +func (cg *clientGetterImpl) KueueClientset() (kueueversioned.Interface, error) { + config, err := cg.ToRESTConfig() + if err != nil { + return nil, err + } + + clientset, err := kueueversioned.NewForConfig(config) + if err != nil { + return nil, err + } + + return clientset, nil +} + +func (cg *clientGetterImpl) KjobctlClientset() (kjobctlversioned.Interface, error) { + config, err := cg.ToRESTConfig() + if err != nil { + return nil, err + } + + clientset, err := kjobctlversioned.NewForConfig(config) + if err != nil { + return nil, err + } + + return clientset, nil +} + +func (cg *clientGetterImpl) RayClientset() (rayversioned.Interface, error) { + config, err := cg.ToRESTConfig() + if err != nil { + return nil, err + } + + clientset, err := rayversioned.NewForConfig(config) + if err != nil { + return nil, err + } + + return clientset, nil +} + +func (cg *clientGetterImpl) DynamicClient() (dynamic.Interface, error) { + config, err := cg.ToRESTConfig() + if err != nil { + return nil, err + } + + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return nil, err + } + + return dynamicClient, nil +} + +func (cg *clientGetterImpl) NewResourceBuilder() *resource.Builder { + return resource.NewBuilder(cg.RESTClientGetter) +} diff --git a/pkg/cmd/util/dry_run.go b/pkg/cmd/util/dry_run.go new file mode 100644 index 0000000..5546705 --- /dev/null +++ b/pkg/cmd/util/dry_run.go @@ -0,0 +1,85 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" +) + +type DryRunStrategy int + +const ( + // DryRunNone indicates the client will make all mutating calls + DryRunNone DryRunStrategy = iota + + // DryRunClient or client-side dry-run, indicates the client will prevent + // making mutating calls such as CREATE, PATCH, and DELETE + DryRunClient + + // DryRunServer or server-side dry-run, indicates the client will send + // mutating calls to the APIServer with the dry-run parameter to prevent + // persisting changes. + // + // Note that clients sending server-side dry-run calls should verify that + // the APIServer and the resource supports server-side dry-run, and otherwise + // clients should fail early. + // + // If a client sends a server-side dry-run call to an APIServer that doesn't + // support server-side dry-run, then the APIServer will persist changes inadvertently. + DryRunServer +) + +// AddDryRunFlag adds dry-run flag to a command. +func AddDryRunFlag(cmd *cobra.Command) { + cmd.PersistentFlags().String( + "dry-run", + "none", + `Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`, + ) +} + +func GetDryRunStrategy(cmd *cobra.Command) (DryRunStrategy, error) { + var dryRunFlag = FlagString(cmd, "dry-run") + switch dryRunFlag { + case "client": + return DryRunClient, nil + case "server": + return DryRunServer, nil + case "none": + return DryRunNone, nil + default: + return DryRunNone, fmt.Errorf(`Invalid dry-run value (%v). Must be "none", "server", or "client".`, dryRunFlag) + } +} + +// PrintFlagsWithDryRunStrategy sets a success message at print time for the dry run strategy +func PrintFlagsWithDryRunStrategy(printFlags *genericclioptions.PrintFlags, dryRunStrategy DryRunStrategy) error { + switch dryRunStrategy { + case DryRunClient: + if err := printFlags.Complete("%s (client dry run)"); err != nil { + return err + } + case DryRunServer: + if err := printFlags.Complete("%s (server dry run)"); err != nil { + return err + } + } + return nil +} diff --git a/pkg/cmd/util/helpers.go b/pkg/cmd/util/helpers.go new file mode 100644 index 0000000..4dc6d3d --- /dev/null +++ b/pkg/cmd/util/helpers.go @@ -0,0 +1,55 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "github.com/spf13/cobra" + "k8s.io/klog/v2" +) + +func AddFieldSelectorFlagVar(cmd *cobra.Command, p *string) { + cmd.Flags().StringVar(p, "field-selector", "", + "Selector (field query) to filter on, supports '=', '==', and '!='.(e.g. --field-selector key1=value1,key2=value2). The server only supports a limited number of field queries per type.") +} + +func AddLabelSelectorFlagVar(cmd *cobra.Command, p *string) { + cmd.Flags().StringVarP(p, "selector", "l", "", + "Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints.") +} + +func AddAllNamespacesFlagVar(cmd *cobra.Command, p *bool) { + cmd.Flags().BoolVarP(p, "all-namespaces", "A", false, + "If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.") +} + +func AddProfileFlagVar(cmd *cobra.Command, p *string) { + cmd.Flags().StringVarP(p, "profile", "p", "", + "Filter by profile name which is associated with the resource.") +} + +func AddLocalQueueFlagVar(cmd *cobra.Command, p *string) { + cmd.Flags().StringVarP(p, "localqueue", "q", "", + "Filter by localqueue which is associated with the resource.") +} + +func FlagString(cmd *cobra.Command, flag string) string { + s, err := cmd.Flags().GetString(flag) + if err != nil { + klog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) + } + return s +} diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go new file mode 100644 index 0000000..778cb95 --- /dev/null +++ b/pkg/constants/constants.go @@ -0,0 +1,35 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package constants + +const ( + ProfileLabel = "kjobctl.x-k8s.io/profile" + ModeLabel = "kjobctl.x-k8s.io/mode" + ScriptAnnotation = "kjobctl.x-k8s.io/script" +) + +const ( + SystemEnvVarNameUser = "USER" +) + +const ( + EnvVarNameUserID = "USER_ID" + EnvVarTaskName = "TASK_NAME" + EnvVarTaskID = "TASK_ID" + EnvVarNameProfile = "PROFILE" + EnvVarNameTimestamp = "TIMESTAMP" +) diff --git a/pkg/parser/array_indexes.go b/pkg/parser/array_indexes.go new file mode 100644 index 0000000..24e8435 --- /dev/null +++ b/pkg/parser/array_indexes.go @@ -0,0 +1,170 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "errors" + "regexp" + "slices" + "strconv" + "strings" + + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/utils/ptr" +) + +var ( + invalidArrayFlagFormatErr = errors.New("invalid array flag format") +) + +type ArrayIndexes struct { + Indexes []int32 + Step *int32 + Parallelism *int32 +} + +func (ai ArrayIndexes) Count() int { + return len(ai.Indexes) +} + +func (ai ArrayIndexes) Min() int32 { + return slices.Min(ai.Indexes) +} + +func (ai ArrayIndexes) Max() int32 { + return slices.Max(ai.Indexes) +} + +func GenerateArrayIndexes(count int32) ArrayIndexes { + ai := ArrayIndexes{} + for i := int32(0); i < count; i++ { + ai.Indexes = append(ai.Indexes, i) + } + return ai +} + +// ParseArrayIndexes parse array flag to ArrayIndexes. +// Include syntax like: +// - 1-5 - which results in indexes: 1, 2, 3, 4, 5 +// - 1,4,5 - which results exactly in the mentioned indexes 1, 4, 5 +// - 3-9:3 - with step indicator, which results in 3,6,9 +// - 1-5%2 - which results in indexes: 1, 2, 3, 4, 5 but only 2 of them are processed at the same time. +func ParseArrayIndexes(str string) (ArrayIndexes, error) { + arrayIndexes := ArrayIndexes{} + + var ( + indexes []int32 + step *int32 + parallelism *int32 + err error + ) + + if regexp.MustCompile(`^[0-9]\d*(,[1-9]\d*)*$`).MatchString(str) { + indexes, err = parseCommaSeparatedIndexes(str) + } else if matches := regexp.MustCompile(`(^[0-9]\d*-[1-9]\d*)(([:%])([1-9]\d*))?$`).FindStringSubmatch(str); matches != nil { + var num int64 + + if matches[4] != "" { + num, err = strconv.ParseInt(matches[4], 10, 32) + if err != nil { + return arrayIndexes, invalidArrayFlagFormatErr + } + } + + switch matches[3] { + case ":": + step = ptr.To(int32(num)) + case "%": + parallelism = ptr.To(int32(num)) + } + + indexes, err = parseRangeIndexes(matches[1], step) + } else { + return arrayIndexes, invalidArrayFlagFormatErr + } + + if err != nil { + return arrayIndexes, err + } + + slices.Sort(indexes) + + arrayIndexes.Indexes = indexes + arrayIndexes.Step = step + arrayIndexes.Parallelism = parallelism + + return arrayIndexes, nil +} + +func parseRangeIndexes(str string, step *int32) ([]int32, error) { + if step == nil { + step = ptr.To[int32](1) + } + + if *step <= 0 { + return nil, invalidArrayFlagFormatErr + } + + parts := strings.Split(str, "-") + if len(parts) != 2 { + return nil, invalidArrayFlagFormatErr + } + + from, err := strconv.ParseInt(parts[0], 10, 32) + if err != nil { + return nil, invalidArrayFlagFormatErr + } + + to, err := strconv.ParseInt(parts[1], 10, 32) + if err != nil { + return nil, invalidArrayFlagFormatErr + } + + if from > to { + return nil, invalidArrayFlagFormatErr + } + + var indexes []int32 + + for i := int32(from); i <= int32(to); i += *step { + indexes = append(indexes, i) + } + + return indexes, nil +} + +func parseCommaSeparatedIndexes(str string) ([]int32, error) { + strIndexes := strings.Split(str, ",") + maxValue := int32(-1) + set := sets.New[int32]() + for _, strIndex := range strIndexes { + num, err := strconv.ParseInt(strIndex, 10, 32) + if err != nil { + return nil, invalidArrayFlagFormatErr + } + index := int32(num) + if index < maxValue { + return nil, invalidArrayFlagFormatErr + } + if set.Has(index) { + return nil, invalidArrayFlagFormatErr + } + set.Insert(index) + maxValue = index + } + return set.UnsortedList(), nil +} diff --git a/pkg/parser/array_indexes_test.go b/pkg/parser/array_indexes_test.go new file mode 100644 index 0000000..94dfadc --- /dev/null +++ b/pkg/parser/array_indexes_test.go @@ -0,0 +1,115 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "k8s.io/utils/ptr" +) + +func TestParseSlurmArrayIndexes(t *testing.T) { + testCases := map[string]struct { + array string + wantArrayIndexes ArrayIndexes + wantErr string + }{ + "should parse 0": { + array: "0", + wantArrayIndexes: ArrayIndexes{ + Indexes: []int32{0}, + }, + }, + "should parse 1,4,5": { + array: "1,4,5", + wantArrayIndexes: ArrayIndexes{ + Indexes: []int32{1, 4, 5}, + }, + }, + "should parse 1,4,5, because duplicate indexes": { + array: "1,1,4,5", + wantErr: invalidArrayFlagFormatErr.Error(), + }, + "shouldn't parse 4,1,5, because invalid sequence": { + array: "4,1,5", + wantErr: invalidArrayFlagFormatErr.Error(), + }, + "should parse 1-5": { + array: "1-5", + wantArrayIndexes: ArrayIndexes{ + Indexes: []int32{1, 2, 3, 4, 5}, + }, + }, + "shouldn't parse 5-1, because from > to": { + array: "5-1", + wantErr: invalidArrayFlagFormatErr.Error(), + }, + "should parse 3-9:3": { + array: "3-9:3", + wantArrayIndexes: ArrayIndexes{ + Indexes: []int32{3, 6, 9}, + Step: ptr.To[int32](3), + }, + }, + "should parse 1-5%2": { + array: "1-5%2", + wantArrayIndexes: ArrayIndexes{ + Indexes: []int32{1, 2, 3, 4, 5}, + Parallelism: ptr.To[int32](2), + }, + }, + "shouldn't parse 1-5?2": { + array: "1-5?2", + wantErr: invalidArrayFlagFormatErr.Error(), + }, + "shouldn't parse 1-5:2147483648, because value out of range": { + array: "1-5:2147483648", + wantErr: invalidArrayFlagFormatErr.Error(), + }, + "should parse 0-5": { + array: "0-5", + wantArrayIndexes: ArrayIndexes{ + Indexes: []int32{0, 1, 2, 3, 4, 5}, + }, + }, + "should parse 0,2,4": { + array: "0,2,4", + wantArrayIndexes: ArrayIndexes{ + Indexes: []int32{0, 2, 4}, + }, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + gotArrayIndexes, gotErr := ParseArrayIndexes(tc.array) + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + return + } + + if diff := cmp.Diff(tc.wantArrayIndexes, gotArrayIndexes); diff != "" { + t.Errorf("Unexpected array indexes (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/parser/slurm.go b/pkg/parser/slurm.go new file mode 100644 index 0000000..ebdb2f9 --- /dev/null +++ b/pkg/parser/slurm.go @@ -0,0 +1,185 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +const slurmDirective = "#SBATCH" + +var longFlagFormat = regexp.MustCompile(`(\w+[\w+-]*)=(\S+)`) +var shortFlagFormat = regexp.MustCompile(`-(\w)\s+(\S+)`) + +type ParsedSlurmFlags struct { + Array string + CpusPerTask *resource.Quantity + GpusPerTask map[string]*resource.Quantity + MemPerNode *resource.Quantity + MemPerTask *resource.Quantity + MemPerCPU *resource.Quantity + MemPerGPU *resource.Quantity + Nodes *int32 + NTasks *int32 + Output string + Error string + Input string + JobName string + Partition string + TimeLimit string +} + +func SlurmFlags(script string, ignoreUnknown bool) (ParsedSlurmFlags, error) { + var flags ParsedSlurmFlags + + scanner := bufio.NewScanner(strings.NewReader(script)) + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if len(line) == 0 { + continue + } + + if !strings.HasPrefix(line, slurmDirective) { + if strings.HasPrefix(line, "#") { + continue + } + break + } + + key, val := extractKeyValue(line) + if len(key) == 0 || len(val) == 0 { + continue + } + + switch key { + case "a", string(v1alpha1.ArrayFlag): + flags.Array = val + case string(v1alpha1.CpusPerTaskFlag): + cpusPerTask, err := resource.ParseQuantity(val) + if err != nil { + return flags, fmt.Errorf("cannot parse '%s': %w", val, err) + } + flags.CpusPerTask = ptr.To(cpusPerTask) + case "e", string(v1alpha1.ErrorFlag): + flags.Error = val + case string(v1alpha1.GpusPerTaskFlag): + gpusPerTask, err := GpusFlag(val) + if err != nil { + return flags, fmt.Errorf("cannot parse '%s': %w", val, err) + } + flags.GpusPerTask = gpusPerTask + case "i", string(v1alpha1.InputFlag): + flags.Input = val + case "J", string(v1alpha1.JobNameFlag): + flags.JobName = val + case string(v1alpha1.MemPerNodeFlag): + memPerNode, err := resource.ParseQuantity(val) + if err != nil { + return flags, fmt.Errorf("cannot parse '%s': %w", val, err) + } + flags.MemPerNode = ptr.To(memPerNode) + case string(v1alpha1.MemPerCPUFlag): + memPerCPU, err := resource.ParseQuantity(val) + if err != nil { + return flags, fmt.Errorf("cannot parse '%s': %w", val, err) + } + flags.MemPerCPU = ptr.To(memPerCPU) + case string(v1alpha1.MemPerGPUFlag): + memPerGPU, err := resource.ParseQuantity(val) + if err != nil { + return flags, fmt.Errorf("cannot parse '%s': %w", val, err) + } + flags.MemPerGPU = ptr.To(memPerGPU) + case string(v1alpha1.MemPerTaskFlag): + memPerTask, err := resource.ParseQuantity(val) + if err != nil { + return flags, fmt.Errorf("cannot parse '%s': %w", val, err) + } + flags.MemPerTask = ptr.To(memPerTask) + case "N", string(v1alpha1.NodesFlag): + intVal, err := strconv.ParseInt(val, 10, 32) + if err != nil { + return flags, err + } + flags.Nodes = ptr.To(int32(intVal)) + case "n", string(v1alpha1.NTasksFlag): + intVal, err := strconv.ParseInt(val, 10, 32) + if err != nil { + return flags, err + } + flags.NTasks = ptr.To(int32(intVal)) + case "o", string(v1alpha1.OutputFlag): + flags.Output = val + case "p", string(v1alpha1.PartitionFlag): + flags.Partition = val + case "t", string(v1alpha1.TimeFlag): + flags.TimeLimit = val + default: + if !ignoreUnknown { + return ParsedSlurmFlags{}, fmt.Errorf("unknown flag: %s", key) + } + } + } + + return flags, nil +} + +func extractKeyValue(s string) (string, string) { + matches := longFlagFormat.FindStringSubmatch(s) + + if len(matches) != 3 { + matches = shortFlagFormat.FindStringSubmatch(s) + if len(matches) != 3 { + return "", "" + } + } + + return matches[1], matches[2] +} + +func GpusFlag(val string) (map[string]*resource.Quantity, error) { + gpus := make(map[string]*resource.Quantity) + + items := strings.Split(val, ",") + for _, v := range items { + gpu := strings.Split(v, ":") + if len(gpu) != 2 { + return nil, errors.New("invalid GPU format. It must be :") + } + + name := gpu[0] + quantity, err := resource.ParseQuantity(gpu[1]) + if err != nil { + return nil, err + } + + gpus[name] = &quantity + } + + return gpus, nil +} diff --git a/pkg/parser/slurm_test.go b/pkg/parser/slurm_test.go new file mode 100644 index 0000000..abb1871 --- /dev/null +++ b/pkg/parser/slurm_test.go @@ -0,0 +1,141 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" +) + +func TestParseSlurmOptions(t *testing.T) { + testCases := map[string]struct { + script string + ignoreUnknown bool + want ParsedSlurmFlags + wantErr string + }{ + "should parse simple script": { + script: `#!/bin/bash +#SBATCH --job-name=single_Cpu +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=1 + +sleep 30 +echo "hello"`, + want: ParsedSlurmFlags{ + JobName: "single_Cpu", + NTasks: ptr.To[int32](1), + CpusPerTask: ptr.To(resource.MustParse("1")), + }, + }, + "should parse script with short flags": { + script: `#SBATCH -J serial_Test_Job +#SBATCH -n 1 +#SBATCH -o output.%j +#SBATCH -e error.%j + +./myexec +exit 0`, + want: ParsedSlurmFlags{ + JobName: "serial_Test_Job", + NTasks: ptr.To[int32](1), + Output: "output.%j", + Error: "error.%j", + }, + }, + "should parse script with comments": { + script: `#!/bin/bash +# Job name +#SBATCH --job-name=job-array +# Defines a job array from task ID 1 to 20 +#SBATCH --array=1-20 +# Number of tasks (in this case, one task per array element) +#SBATCH -n 1 +# Partition or queue name +#SBATCH --partition=shared +#SBATCH # This is an empty line to separate Slurm directives from the job commands + +echo "Start Job $SLURM_ARRAY_TASK_ID on $HOSTNAME" # Display job start information + +sleep 10`, + want: ParsedSlurmFlags{ + JobName: "job-array", + Array: "1-20", + NTasks: ptr.To[int32](1), + Partition: "shared", + }, + }, + "should parse script and ignore unknown flags": { + script: `#!/bin/bash +#SBATCH --job-name=my_job_name +#SBATCH --output=output.txt +#SBATCH --error=error.txt +#SBATCH --partition=partition_name +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=1 +#SBATCH --time=1:00:00 +#SBATCH --mail-type=END +#SBATCH --mail-user=your@email.com +#SBATCH --unknown=unknown + +python my_script.py`, + ignoreUnknown: true, + want: ParsedSlurmFlags{ + JobName: "my_job_name", + Output: "output.txt", + Error: "error.txt", + Partition: "partition_name", + Nodes: ptr.To[int32](1), + NTasks: ptr.To[int32](1), + CpusPerTask: ptr.To(resource.MustParse("1")), + TimeLimit: "1:00:00", + }, + }, + "should fail due to unknown flags": { + script: `#!/bin/bash +#SBATCH --job-name=my_job_name +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=1 +#SBATCH --unknown=unknown + +python my_script.py`, + wantErr: "unknown flag: unknown", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + got, gotErr := SlurmFlags(tc.script, tc.ignoreUnknown) + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + return + } + + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("Unexpected options (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/parser/time_limit.go b/pkg/parser/time_limit.go new file mode 100644 index 0000000..4125a52 --- /dev/null +++ b/pkg/parser/time_limit.go @@ -0,0 +1,148 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "errors" + "fmt" + "strings" +) + +var ( + invalidTimeFormatErr = errors.New("invalid time format") +) + +// isValidTimeSpec validates that time format follows is supported. +// Inspired by https://github.com/SchedMD/slurm/blob/9322d01b1aadde28181254499dd0d80638a3df89/src/common/parse_time.c#L87-L145. +func isValidTimeSpec(val string) bool { + var digit, dash, colon int + + var alreadyDigit bool + for _, ch := range val { + switch { + case ch >= '0' && ch <= '9': + if !alreadyDigit { + digit++ + alreadyDigit = true + } + case ch == '-': + alreadyDigit = false + dash++ + if colon > 0 { + return false + } + case ch == ':': + alreadyDigit = false + colon++ + default: + return false + } + } + + if digit == 0 { + return false + } + + if dash > 1 || colon > 2 { + return false + } + + if dash > 0 { + if colon == 0 && digit != 2 { + return false + } + + if colon == 1 && digit < 3 { + return false + } + + if colon == 2 && digit < 4 { + return false + } + } else { + if colon == 1 && digit < 2 { + return false + } + + if colon == 2 && digit < 3 { + return false + } + } + + return true +} + +// TimeLimitToSeconds converts a string to an equivalent time value. +// If val is empty, it returns nil. If format is invalid, it returns +// invalidTimeFormatErr. +// +// Possible formats: +// - "minutes" +// - "minutes:seconds" +// - "hours:minutes:seconds" +// - "days-hours" +// - "days-hours:minutes" +// - "days-hours:minutes:seconds" +// - "INFINITE" +// - "UNLIMITED" +// - "-1" +// +// For more details, see https://slurm.schedmd.com/sbatch.html#OPT_time. +// +// Inspired by https://github.com/SchedMD/slurm/blob/9322d01b1aadde28181254499dd0d80638a3df89/src/common/parse_time.c#L731-L784 +func TimeLimitToSeconds(val string) (*int32, error) { + val = strings.TrimSpace(val) + + if val == "" || strings.EqualFold(val, "-1") || strings.EqualFold(val, "INFINITE") || strings.EqualFold(val, "UNLIMITED") { + return nil, nil + } + + if !isValidTimeSpec(val) { + return nil, invalidTimeFormatErr + } + + var d, h, m, s int32 + + if strings.Contains(val, "-") { + // days-[hours[:minutes[:seconds]]] + // nolint:errcheck + fmt.Sscanf(val, "%d-%d:%d:%d", &d, &h, &m, &s) + d *= 86400 + h *= 3600 + m *= 60 + } else { + // nolint:errcheck + if n, _ := fmt.Sscanf(val, "%d:%d:%d", &h, &m, &s); n == 3 { + // hours:minutes:seconds + h *= 3600 + m *= 60 + } else { + // minutes[:seconds] + // h is minutes here and m is seconds due to sscanf parsing left to right + s = m + m = h * 60 + h = 0 + } + } + + res := d + h + m + s + if res == int32(0) { + return nil, nil + } + + return &res, nil +} diff --git a/pkg/parser/time_limit_test.go b/pkg/parser/time_limit_test.go new file mode 100644 index 0000000..1e666be --- /dev/null +++ b/pkg/parser/time_limit_test.go @@ -0,0 +1,121 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "k8s.io/utils/ptr" +) + +func TestParseTimeLimit(t *testing.T) { + testCases := map[string]struct { + val string + want *int32 + wantErr error + }{ + "empty value": {}, + "-1": { + val: "-1", + }, + "INFINITE": { + val: "INFINITE", + }, + "UNLIMITED": { + val: "UNLIMITED", + }, + "infinite": { + val: "infinite", + }, + "unlimited": { + val: "unlimited", + }, + "incomplete formats 12-": { + val: "12-", + wantErr: invalidTimeFormatErr, + }, + "incomplete formats 12:": { + val: "12:", + wantErr: invalidTimeFormatErr, + }, + "incomplete formats 12-05:": { + val: "12-05:", + wantErr: invalidTimeFormatErr, + }, + "single digit minimal values 0": { + val: "0", + }, + "single digit minimal values 0:0:0": { + val: "0:0:0", + }, + "single digit minimal values 0-0:0:0": { + val: "0-0:0:0", + }, + "single digit minimal values 1-0:0:0": { + val: "1-0:0:0", + want: ptr.To[int32](24 * 60 * 60), + }, + "not supported chars": { + val: "12-0m-23", + wantErr: invalidTimeFormatErr, + }, + "more than one dashes": { + val: "12-05-23", + wantErr: invalidTimeFormatErr, + }, + "more than two colons": { + val: "2:12:05:23", + wantErr: invalidTimeFormatErr, + }, + "minutes": { + val: "05", + want: ptr.To[int32](5 * 60), + }, + "minutes:seconds": { + val: "05:23", + want: ptr.To[int32](5*60 + 23), + }, + "hours:minutes:seconds": { + val: "12:05:23", + want: ptr.To[int32](12*60*60 + 5*60 + 23), + }, + "days-hours:minutes:seconds": { + val: "2-12:05:23", + want: ptr.To[int32](2*24*60*60 + 12*60*60 + 5*60 + 23), + }, + "days-hours": { + val: "02-12", + want: ptr.To[int32](2*24*60*60 + 12*60*60), + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + got, gotErr := TimeLimitToSeconds(tc.val) + + if diff := cmp.Diff(tc.wantErr, gotErr, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + return + } + + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("Unexpected resutl (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/testing/wrappers/application_profile_wrappers.go b/pkg/testing/wrappers/application_profile_wrappers.go new file mode 100644 index 0000000..de6cb88 --- /dev/null +++ b/pkg/testing/wrappers/application_profile_wrappers.go @@ -0,0 +1,59 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// ApplicationProfileWrapper wraps an ApplicationProfile. +type ApplicationProfileWrapper struct{ v1alpha1.ApplicationProfile } + +// MakeApplicationProfile creates a wrapper for an ApplicationProfile +func MakeApplicationProfile(name, ns string) *ApplicationProfileWrapper { + return &ApplicationProfileWrapper{ + ApplicationProfile: v1alpha1.ApplicationProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + Spec: v1alpha1.ApplicationProfileSpec{ + SupportedModes: nil, + VolumeBundles: nil, + }, + }, + } +} + +// Obj returns the inner ApplicationProfile. +func (ap *ApplicationProfileWrapper) Obj() *v1alpha1.ApplicationProfile { + return &ap.ApplicationProfile +} + +// WithSupportedMode adds the SupportedMode +func (ap *ApplicationProfileWrapper) WithSupportedMode(supportedMode v1alpha1.SupportedMode) *ApplicationProfileWrapper { + ap.Spec.SupportedModes = append(ap.Spec.SupportedModes, supportedMode) + return ap +} + +// WithVolumeBundleReferences adds WithVolumeBundleReferences to VolumeBundles. +func (ap *ApplicationProfileWrapper) WithVolumeBundleReferences(volumeBundleReferences ...v1alpha1.VolumeBundleReference) *ApplicationProfileWrapper { + ap.Spec.VolumeBundles = append(ap.Spec.VolumeBundles, volumeBundleReferences...) + return ap +} diff --git a/pkg/testing/wrappers/configmap_wrappers.go b/pkg/testing/wrappers/configmap_wrappers.go new file mode 100644 index 0000000..e73e4e2 --- /dev/null +++ b/pkg/testing/wrappers/configmap_wrappers.go @@ -0,0 +1,83 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +// ConfigMapWrapper wraps a ConfigMap. +type ConfigMapWrapper struct{ corev1.ConfigMap } + +// MakeConfigMap creates a wrapper for a ConfigMap +func MakeConfigMap(name, ns string) *ConfigMapWrapper { + return &ConfigMapWrapper{ + corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + }, + } +} + +// Obj returns the inner ConfigMap. +func (w *ConfigMapWrapper) Obj() *corev1.ConfigMap { + return &w.ConfigMap +} + +// WithOwnerReference adds the owner reference. +func (w *ConfigMapWrapper) WithOwnerReference(ref metav1.OwnerReference) *ConfigMapWrapper { + w.OwnerReferences = append(w.OwnerReferences, ref) + return w +} + +// Profile sets the profile label. +func (w *ConfigMapWrapper) Profile(v string) *ConfigMapWrapper { + return w.Label(constants.ProfileLabel, v) +} + +// Mode sets the profile label. +func (w *ConfigMapWrapper) Mode(v v1alpha1.ApplicationProfileMode) *ConfigMapWrapper { + return w.Label(constants.ModeLabel, string(v)) +} + +// LocalQueue sets the localqueue label. +func (w *ConfigMapWrapper) LocalQueue(v string) *ConfigMapWrapper { + return w.Label(kueueconstants.QueueLabel, v) +} + +// Label sets the label key and value. +func (w *ConfigMapWrapper) Label(key, value string) *ConfigMapWrapper { + if w.Labels == nil { + w.Labels = make(map[string]string) + } + w.ObjectMeta.Labels[key] = value + return w +} + +// Data sets data. +func (w *ConfigMapWrapper) Data(data map[string]string) *ConfigMapWrapper { + w.ConfigMap.Data = data + return w +} diff --git a/pkg/testing/wrappers/container_wrappers.go b/pkg/testing/wrappers/container_wrappers.go new file mode 100644 index 0000000..4d434a5 --- /dev/null +++ b/pkg/testing/wrappers/container_wrappers.go @@ -0,0 +1,82 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// ContainerWrapper wraps a Container. +type ContainerWrapper struct{ corev1.Container } + +// MakeContainer creates a wrapper for a Container +func MakeContainer(name, image string) *ContainerWrapper { + return &ContainerWrapper{ + Container: corev1.Container{Name: name, Image: image}, + } +} + +// Obj returns the inner Container. +func (c *ContainerWrapper) Obj() *corev1.Container { + return &c.Container +} + +// Command set command. +func (c *ContainerWrapper) Command(command ...string) *ContainerWrapper { + c.Container.Command = command + return c +} + +// WithRequest add Request to the ResourceList. +func (c *ContainerWrapper) WithRequest(resourceName corev1.ResourceName, quantity resource.Quantity) *ContainerWrapper { + if c.Resources.Requests == nil { + c.Resources.Requests = corev1.ResourceList{} + } + c.Resources.Requests[resourceName] = quantity + return c +} + +// WithResources set Resources. +func (c *ContainerWrapper) WithResources(resourceRequirements corev1.ResourceRequirements) *ContainerWrapper { + c.Container.Resources = resourceRequirements + return c +} + +// WithEnvVar add EnvVar to Env. +func (c *ContainerWrapper) WithEnvVar(envVar corev1.EnvVar) *ContainerWrapper { + c.Container.Env = append(c.Container.Env, envVar) + return c +} + +// WithVolumeMount add VolumeMount to VolumeMounts. +func (c *ContainerWrapper) WithVolumeMount(volumeMount corev1.VolumeMount) *ContainerWrapper { + c.Container.VolumeMounts = append(c.Container.VolumeMounts, volumeMount) + return c +} + +// TTY set tty=true. +func (c *ContainerWrapper) TTY() *ContainerWrapper { + c.Container.TTY = true + return c +} + +// Stdin set stdin=true. +func (c *ContainerWrapper) Stdin() *ContainerWrapper { + c.Container.Stdin = true + return c +} diff --git a/pkg/testing/wrappers/job_template_wrappers.go b/pkg/testing/wrappers/job_template_wrappers.go new file mode 100644 index 0000000..cc72f98 --- /dev/null +++ b/pkg/testing/wrappers/job_template_wrappers.go @@ -0,0 +1,167 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// JobTemplateWrapper wraps a JobTemplate. +type JobTemplateWrapper struct{ v1alpha1.JobTemplate } + +// MakeJobTemplate creates a wrapper for a JobTemplate +func MakeJobTemplate(name, ns string) *JobTemplateWrapper { + return &JobTemplateWrapper{ + JobTemplate: v1alpha1.JobTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + }, + } +} + +// Obj returns the inner JobTemplate. +func (j *JobTemplateWrapper) Obj() *v1alpha1.JobTemplate { + return &j.JobTemplate +} + +// Label sets the label key and value. +func (j *JobTemplateWrapper) Label(key, value string) *JobTemplateWrapper { + if j.Template.ObjectMeta.Labels == nil { + j.Template.ObjectMeta.Labels = make(map[string]string) + } + j.Template.ObjectMeta.Labels[key] = value + return j +} + +// Annotation sets the label key and value. +func (j *JobTemplateWrapper) Annotation(key, value string) *JobTemplateWrapper { + if j.Template.ObjectMeta.Annotations == nil { + j.Template.ObjectMeta.Annotations = make(map[string]string) + } + j.Template.ObjectMeta.Annotations[key] = value + return j +} + +// Completions updates the completions on the job template. +func (j *JobTemplateWrapper) Completions(completion int32) *JobTemplateWrapper { + j.Template.Spec.Completions = ptr.To(completion) + return j +} + +// Parallelism updates the parallelism on the job template. +func (j *JobTemplateWrapper) Parallelism(parallelism int32) *JobTemplateWrapper { + j.Template.Spec.Parallelism = ptr.To(parallelism) + return j +} + +// RestartPolicy updates the restartPolicy on the pod template. +func (j *JobTemplateWrapper) RestartPolicy(restartPolicy corev1.RestartPolicy) *JobTemplateWrapper { + j.Template.Spec.Template.Spec.RestartPolicy = restartPolicy + return j +} + +// BackoffLimitPerIndex updates the backoffLimitPerIndex on the job template. +func (j *JobTemplateWrapper) BackoffLimitPerIndex(backoffLimitPerIndex int32) *JobTemplateWrapper { + j.Template.Spec.BackoffLimitPerIndex = ptr.To[int32](backoffLimitPerIndex) + return j +} + +// WithInitContainer add init container to the pod template. +func (j *JobTemplateWrapper) WithInitContainer(container corev1.Container) *JobTemplateWrapper { + j.Template.Spec.Template.Spec.InitContainers = append(j.Template.Spec.Template.Spec.InitContainers, container) + return j +} + +// WithContainer add container to the pod template. +func (j *JobTemplateWrapper) WithContainer(container corev1.Container) *JobTemplateWrapper { + j.Template.Spec.Template.Spec.Containers = append(j.Template.Spec.Template.Spec.Containers, container) + return j +} + +// Clone clone JobTemplateWrapper. +func (j *JobTemplateWrapper) Clone() *JobTemplateWrapper { + return &JobTemplateWrapper{ + JobTemplate: *j.JobTemplate.DeepCopy(), + } +} + +// WithVolume add volume to the job template. +func (j *JobTemplateWrapper) WithVolume(name, localObjectReferenceName string) *JobTemplateWrapper { + j.Template.Spec.Template.Spec.Volumes = append(j.Template.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: localObjectReferenceName, + }, + }, + }, + }) + return j +} + +// WithEnvVar add volume to the job template. +func (j *JobTemplateWrapper) WithEnvVar(envVar corev1.EnvVar) *JobTemplateWrapper { + for index := range j.Template.Spec.Template.Spec.InitContainers { + j.Template.Spec.Template.Spec.InitContainers[index].Env = + append(j.Template.Spec.Template.Spec.InitContainers[index].Env, envVar) + } + for index := range j.Template.Spec.Template.Spec.Containers { + j.Template.Spec.Template.Spec.Containers[index].Env = + append(j.Template.Spec.Template.Spec.Containers[index].Env, envVar) + } + return j +} + +// WithVolumeMount add volume mount to pod templates. +func (j *JobTemplateWrapper) WithVolumeMount(volumeMount corev1.VolumeMount) *JobTemplateWrapper { + for index := range j.Template.Spec.Template.Spec.InitContainers { + j.Template.Spec.Template.Spec.InitContainers[index].VolumeMounts = + append(j.Template.Spec.Template.Spec.InitContainers[index].VolumeMounts, volumeMount) + } + for index := range j.Template.Spec.Template.Spec.Containers { + j.Template.Spec.Template.Spec.Containers[index].VolumeMounts = + append(j.Template.Spec.Template.Spec.Containers[index].VolumeMounts, volumeMount) + } + return j +} + +// Command set command to primary pod templates. +func (j *JobTemplateWrapper) Command(command []string) *JobTemplateWrapper { + if len(j.Template.Spec.Template.Spec.Containers) > 0 { + j.Template.Spec.Template.Spec.Containers[0].Command = command + } + return j +} + +// WithRequest set command to primary pod templates. +func (j *JobTemplateWrapper) WithRequest(key corev1.ResourceName, value resource.Quantity) *JobTemplateWrapper { + if len(j.Template.Spec.Template.Spec.Containers) > 0 { + if j.Template.Spec.Template.Spec.Containers[0].Resources.Requests == nil { + j.Template.Spec.Template.Spec.Containers[0].Resources.Requests = make(corev1.ResourceList) + } + j.Template.Spec.Template.Spec.Containers[0].Resources.Requests[key] = value + } + return j +} diff --git a/pkg/testing/wrappers/job_wrappers.go b/pkg/testing/wrappers/job_wrappers.go new file mode 100644 index 0000000..15b6fca --- /dev/null +++ b/pkg/testing/wrappers/job_wrappers.go @@ -0,0 +1,197 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + "strconv" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +// JobWrapper wraps a Job. +type JobWrapper struct{ batchv1.Job } + +// MakeJob creates a wrapper for a Job +func MakeJob(name, ns string) *JobWrapper { + return &JobWrapper{ + batchv1.Job{ + TypeMeta: metav1.TypeMeta{Kind: "Job", APIVersion: "batch/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + }, + } +} + +// Obj returns the inner Job. +func (j *JobWrapper) Obj() *batchv1.Job { + return &j.Job +} + +// GenerateName updates generateName. +func (j *JobWrapper) GenerateName(v string) *JobWrapper { + j.ObjectMeta.GenerateName = v + return j +} + +// Completions updates job completions. +func (j *JobWrapper) Completions(v int32) *JobWrapper { + j.Job.Spec.Completions = ptr.To(v) + return j +} + +// CompletionMode updates job completions. +func (j *JobWrapper) CompletionMode(completionMode batchv1.CompletionMode) *JobWrapper { + j.Job.Spec.CompletionMode = &completionMode + return j +} + +// Subdomain updates pod template subdomain. +func (j *JobWrapper) Subdomain(subdomain string) *JobWrapper { + j.Job.Spec.Template.Spec.Subdomain = subdomain + return j +} + +// Parallelism updates job parallelism. +func (j *JobWrapper) Parallelism(v int32) *JobWrapper { + j.Job.Spec.Parallelism = ptr.To(v) + return j +} + +// Profile sets the profile label. +func (j *JobWrapper) Profile(v string) *JobWrapper { + return j.Label(constants.ProfileLabel, v) +} + +// Priority sets the workload priority class label. +func (j *JobWrapper) Priority(v string) *JobWrapper { + return j.Label(kueueconstants.WorkloadPriorityClassLabel, v) +} + +// MaxExecTimeSecondsLabel sets the max exec time seconds label. +func (j *JobWrapper) MaxExecTimeSecondsLabel(v string) *JobWrapper { + return j.Label(kueueconstants.MaxExecTimeSecondsLabel, v) +} + +// Mode sets the profile label. +func (j *JobWrapper) Mode(v v1alpha1.ApplicationProfileMode) *JobWrapper { + return j.Label(constants.ModeLabel, string(v)) +} + +// LocalQueue sets the localqueue label. +func (j *JobWrapper) LocalQueue(v string) *JobWrapper { + return j.Label(kueueconstants.QueueLabel, v) +} + +// Label sets the label key and value. +func (j *JobWrapper) Label(key, value string) *JobWrapper { + if j.Labels == nil { + j.Labels = make(map[string]string) + } + j.ObjectMeta.Labels[key] = value + return j +} + +// Annotation sets the label key and value. +func (j *JobWrapper) Annotation(key, value string) *JobWrapper { + if j.Annotations == nil { + j.Annotations = make(map[string]string) + } + j.ObjectMeta.Annotations[key] = value + return j +} + +// WithContainer add container on the pod template. +func (j *JobWrapper) WithContainer(container corev1.Container) *JobWrapper { + j.Job.Spec.Template.Spec.Containers = append(j.Job.Spec.Template.Spec.Containers, container) + return j +} + +// WithInitContainer add init container on the pod template. +func (j *JobWrapper) WithInitContainer(initContainer corev1.Container) *JobWrapper { + j.Job.Spec.Template.Spec.InitContainers = append(j.Job.Spec.Template.Spec.InitContainers, initContainer) + return j +} + +// WithVolume add volume. +func (j *JobWrapper) WithVolume(volume corev1.Volume) *JobWrapper { + j.Job.Spec.Template.Spec.Volumes = append(j.Job.Spec.Template.Spec.Volumes, volume) + return j +} + +// WithEnvVar add env var to the container template. +func (j *JobWrapper) WithEnvVar(envVar corev1.EnvVar) *JobWrapper { + for index := range j.Job.Spec.Template.Spec.Containers { + j.Job.Spec.Template.Spec.Containers[index].Env = + append(j.Job.Spec.Template.Spec.Containers[index].Env, envVar) + } + return j +} + +// WithEnvVarIndexValue add env var to the container template with index value. +func (j *JobWrapper) WithEnvVarIndexValue(name string) *JobWrapper { + for index := range j.Job.Spec.Template.Spec.Containers { + j.Job.Spec.Template.Spec.Containers[index].Env = append(j.Job.Spec.Template.Spec.Containers[index].Env, + corev1.EnvVar{Name: name, Value: strconv.Itoa(index)}) + } + return j +} + +// RestartPolicy updates the restartPolicy on the pod template. +func (j *JobWrapper) RestartPolicy(restartPolicy corev1.RestartPolicy) *JobWrapper { + j.Job.Spec.Template.Spec.RestartPolicy = restartPolicy + return j +} + +// CreationTimestamp sets the .metadata.creationTimestamp +func (j *JobWrapper) CreationTimestamp(t time.Time) *JobWrapper { + j.ObjectMeta.CreationTimestamp = metav1.NewTime(t) + return j +} + +// StartTime sets the .status.startTime +func (j *JobWrapper) StartTime(t time.Time) *JobWrapper { + j.Status.StartTime = &metav1.Time{Time: t} + return j +} + +// CompletionTime sets the .status.completionTime +func (j *JobWrapper) CompletionTime(t time.Time) *JobWrapper { + j.Status.CompletionTime = &metav1.Time{Time: t} + return j +} + +// Succeeded sets the .status.succeeded +func (j *JobWrapper) Succeeded(value int32) *JobWrapper { + j.Status.Succeeded = value + return j +} + +// Spec set job spec. +func (j *JobWrapper) Spec(spec batchv1.JobSpec) *JobWrapper { + j.Job.Spec = spec + return j +} diff --git a/pkg/testing/wrappers/pod_template_wrappers.go b/pkg/testing/wrappers/pod_template_wrappers.go new file mode 100644 index 0000000..8be6683 --- /dev/null +++ b/pkg/testing/wrappers/pod_template_wrappers.go @@ -0,0 +1,153 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// PodTemplateWrapper wraps a PodTemplate. +type PodTemplateWrapper struct{ corev1.PodTemplate } + +// MakePodTemplate creates a wrapper for a PodTemplate. +func MakePodTemplate(name, ns string) *PodTemplateWrapper { + return &PodTemplateWrapper{ + PodTemplate: corev1.PodTemplate{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + }, + } +} + +// Obj returns the inner PodTemplate. +func (p *PodTemplateWrapper) Obj() *corev1.PodTemplate { + return &p.PodTemplate +} + +// Clone clone PodTemplateWrapper. +func (p *PodTemplateWrapper) Clone() *PodTemplateWrapper { + return &PodTemplateWrapper{ + PodTemplate: *p.PodTemplate.DeepCopy(), + } +} + +// Label sets the label key and value. +func (p *PodTemplateWrapper) Label(key, value string) *PodTemplateWrapper { + if p.Template.ObjectMeta.Labels == nil { + p.Template.ObjectMeta.Labels = make(map[string]string) + } + p.Template.ObjectMeta.Labels[key] = value + return p +} + +// Annotation sets the label key and value. +func (p *PodTemplateWrapper) Annotation(key, value string) *PodTemplateWrapper { + if p.Template.ObjectMeta.Annotations == nil { + p.Template.ObjectMeta.Annotations = make(map[string]string) + } + p.Template.ObjectMeta.Annotations[key] = value + return p +} + +// WithInitContainer add container to the pod template. +func (p *PodTemplateWrapper) WithInitContainer(container corev1.Container) *PodTemplateWrapper { + p.PodTemplate.Template.Spec.InitContainers = append(p.PodTemplate.Template.Spec.InitContainers, container) + return p +} + +// WithContainer add container to the pod template. +func (p *PodTemplateWrapper) WithContainer(container corev1.Container) *PodTemplateWrapper { + p.PodTemplate.Template.Spec.Containers = append(p.PodTemplate.Template.Spec.Containers, container) + return p +} + +// WithVolume add volume to the pod template. +func (p *PodTemplateWrapper) WithVolume(name, localObjectReferenceName string) *PodTemplateWrapper { + p.Template.Spec.Volumes = append(p.Template.Spec.Volumes, corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: localObjectReferenceName, + }, + }, + }, + }) + return p +} + +// WithEnvVar add volume to the pod template. +func (p *PodTemplateWrapper) WithEnvVar(envVar corev1.EnvVar) *PodTemplateWrapper { + for index := range p.Template.Spec.InitContainers { + p.Template.Spec.InitContainers[index].Env = + append(p.Template.Spec.InitContainers[index].Env, envVar) + } + for index := range p.Template.Spec.Containers { + p.Template.Spec.Containers[index].Env = + append(p.Template.Spec.Containers[index].Env, envVar) + } + return p +} + +// WithVolumeMount add volume mount to pod templates. +func (p *PodTemplateWrapper) WithVolumeMount(volumeMount corev1.VolumeMount) *PodTemplateWrapper { + for index := range p.Template.Spec.InitContainers { + p.Template.Spec.InitContainers[index].VolumeMounts = + append(p.Template.Spec.InitContainers[index].VolumeMounts, volumeMount) + } + for index := range p.Template.Spec.Containers { + p.Template.Spec.Containers[index].VolumeMounts = + append(p.Template.Spec.Containers[index].VolumeMounts, volumeMount) + } + return p +} + +// Command set command to primary pod templates. +func (p *PodTemplateWrapper) Command(command []string) *PodTemplateWrapper { + if len(p.Template.Spec.Containers) > 0 { + p.Template.Spec.Containers[0].Command = command + } + return p +} + +// WithRequest set command to primary pod templates. +func (p *PodTemplateWrapper) WithRequest(key corev1.ResourceName, value resource.Quantity) *PodTemplateWrapper { + if len(p.Template.Spec.Containers) > 0 { + if p.Template.Spec.Containers[0].Resources.Requests == nil { + p.Template.Spec.Containers[0].Resources.Requests = make(corev1.ResourceList) + } + p.Template.Spec.Containers[0].Resources.Requests[key] = value + } + return p +} + +// TTY set tty=true on primary pod templates. +func (p *PodTemplateWrapper) TTY() *PodTemplateWrapper { + if len(p.Template.Spec.Containers) > 0 { + p.Template.Spec.Containers[0].TTY = true + } + return p +} + +// Stdin set stdin=true on primary pod templates. +func (p *PodTemplateWrapper) Stdin() *PodTemplateWrapper { + if len(p.Template.Spec.Containers) > 0 { + p.Template.Spec.Containers[0].Stdin = true + } + return p +} diff --git a/pkg/testing/wrappers/pod_wrappers.go b/pkg/testing/wrappers/pod_wrappers.go new file mode 100644 index 0000000..f05c636 --- /dev/null +++ b/pkg/testing/wrappers/pod_wrappers.go @@ -0,0 +1,121 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +// PodWrapper wraps a Pod. +type PodWrapper struct{ corev1.Pod } + +// MakePod creates a wrapper for a Pod +func MakePod(name, ns string) *PodWrapper { + return &PodWrapper{ + corev1.Pod{ + TypeMeta: metav1.TypeMeta{Kind: "Pod", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + }, + } +} + +// Obj returns the inner Pod. +func (j *PodWrapper) Obj() *corev1.Pod { + return &j.Pod +} + +// GenerateName updates generateName. +func (j *PodWrapper) GenerateName(v string) *PodWrapper { + j.ObjectMeta.GenerateName = v + return j +} + +// Profile sets the profile label. +func (j *PodWrapper) Profile(v string) *PodWrapper { + return j.Label(constants.ProfileLabel, v) +} + +// Mode sets the profile label. +func (j *PodWrapper) Mode(v v1alpha1.ApplicationProfileMode) *PodWrapper { + return j.Label(constants.ModeLabel, string(v)) +} + +// LocalQueue sets the localqueue label. +func (j *PodWrapper) LocalQueue(v string) *PodWrapper { + return j.Label(kueueconstants.QueueLabel, v) +} + +// Label sets the label key and value. +func (j *PodWrapper) Label(key, value string) *PodWrapper { + if j.Labels == nil { + j.Labels = make(map[string]string) + } + j.ObjectMeta.Labels[key] = value + return j +} + +// Annotation sets the label key and value. +func (j *PodWrapper) Annotation(key, value string) *PodWrapper { + if j.Annotations == nil { + j.Annotations = make(map[string]string) + } + j.ObjectMeta.Annotations[key] = value + return j +} + +// WithContainer add container on the pod template. +func (j *PodWrapper) WithContainer(container corev1.Container) *PodWrapper { + j.Pod.Spec.Containers = append(j.Pod.Spec.Containers, container) + return j +} + +// RestartPolicy updates the restartPolicy on the pod template. +func (j *PodWrapper) RestartPolicy(restartPolicy corev1.RestartPolicy) *PodWrapper { + j.Pod.Spec.RestartPolicy = restartPolicy + return j +} + +// CreationTimestamp sets the .metadata.creationTimestamp. +func (j *PodWrapper) CreationTimestamp(t time.Time) *PodWrapper { + j.ObjectMeta.CreationTimestamp = metav1.NewTime(t) + return j +} + +// StartTime sets the .status.startTime. +func (j *PodWrapper) StartTime(t time.Time) *PodWrapper { + j.Status.StartTime = &metav1.Time{Time: t} + return j +} + +// Spec set pod spec. +func (j *PodWrapper) Spec(spec corev1.PodSpec) *PodWrapper { + j.Pod.Spec = spec + return j +} + +// Phase set pod status phase. +func (j *PodWrapper) Phase(phase corev1.PodPhase) *PodWrapper { + j.Status.Phase = phase + return j +} diff --git a/pkg/testing/wrappers/ray_cluster_spec_wrapper.go b/pkg/testing/wrappers/ray_cluster_spec_wrapper.go new file mode 100644 index 0000000..1d24779 --- /dev/null +++ b/pkg/testing/wrappers/ray_cluster_spec_wrapper.go @@ -0,0 +1,192 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" +) + +// RayClusterSpecWrapper wraps a RayClusterSpec. +type RayClusterSpecWrapper struct{ rayv1.RayClusterSpec } + +// MakeRayClusterSpec creates a wrapper for a RayClusterSpec +func MakeRayClusterSpec() *RayClusterSpecWrapper { + return &RayClusterSpecWrapper{} +} + +// FromRayClusterSpec creates a wrapper for a RayClusterSpec. +func FromRayClusterSpec(spec rayv1.RayClusterSpec) *RayClusterSpecWrapper { + return &RayClusterSpecWrapper{ + RayClusterSpec: spec, + } +} + +// Obj returns the inner RayClusterSpec. +func (w *RayClusterSpecWrapper) Obj() *rayv1.RayClusterSpec { + return &w.RayClusterSpec +} + +// Clone RayClusterSpecWrapper. +func (w *RayClusterSpecWrapper) Clone() *RayClusterSpecWrapper { + return &RayClusterSpecWrapper{ + RayClusterSpec: *w.RayClusterSpec.DeepCopy(), + } +} + +// HeadGroupSpec add worker group to the ray cluster spec. +func (w *RayClusterSpecWrapper) HeadGroupSpec(spec rayv1.HeadGroupSpec) *RayClusterSpecWrapper { + w.RayClusterSpec.HeadGroupSpec = spec + return w +} + +// WithWorkerGroupSpec add worker group to the ray cluster spec. +func (w *RayClusterSpecWrapper) WithWorkerGroupSpec(spec rayv1.WorkerGroupSpec) *RayClusterSpecWrapper { + w.RayClusterSpec.WorkerGroupSpecs = append(w.RayClusterSpec.WorkerGroupSpecs, spec) + + return w +} + +// WithVolume add volume to the ray cluster spec. +func (w *RayClusterSpecWrapper) WithVolume(name, localObjectReferenceName string) *RayClusterSpecWrapper { + headGroupSpec := &w.RayClusterSpec.HeadGroupSpec + headGroupSpec.Template.Spec.Volumes = + append(headGroupSpec.Template.Spec.Volumes, corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: localObjectReferenceName, + }, + }, + }, + }) + + for index := range w.RayClusterSpec.WorkerGroupSpecs { + workerGroupSpec := &w.RayClusterSpec.WorkerGroupSpecs[index] + workerGroupSpec.Template.Spec.Volumes = + append(workerGroupSpec.Template.Spec.Volumes, corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: localObjectReferenceName, + }, + }, + }, + }) + } + + return w +} + +// WithEnvVar add volume to the ray cluster spec. +func (w *RayClusterSpecWrapper) WithEnvVar(envVar corev1.EnvVar) *RayClusterSpecWrapper { + headGroupSpec := &w.RayClusterSpec.HeadGroupSpec + for j := range headGroupSpec.Template.Spec.InitContainers { + container := &headGroupSpec.Template.Spec.InitContainers[j] + container.Env = append(container.Env, envVar) + } + for j := range headGroupSpec.Template.Spec.Containers { + container := &headGroupSpec.Template.Spec.Containers[j] + container.Env = append(container.Env, envVar) + } + + for i := range w.RayClusterSpec.WorkerGroupSpecs { + workerGroupSpec := &w.RayClusterSpec.WorkerGroupSpecs[i] + for j := range workerGroupSpec.Template.Spec.InitContainers { + container := &workerGroupSpec.Template.Spec.InitContainers[j] + container.Env = append(container.Env, envVar) + } + for j := range workerGroupSpec.Template.Spec.Containers { + container := &workerGroupSpec.Template.Spec.Containers[j] + container.Env = append(container.Env, envVar) + } + } + + return w +} + +// WithVolumeMount add volume mount to pod templates. +func (w *RayClusterSpecWrapper) WithVolumeMount(volumeMount corev1.VolumeMount) *RayClusterSpecWrapper { + headGroupSpec := &w.RayClusterSpec.HeadGroupSpec + for j := range headGroupSpec.Template.Spec.InitContainers { + container := &headGroupSpec.Template.Spec.InitContainers[j] + container.VolumeMounts = append(container.VolumeMounts, volumeMount) + } + for j := range headGroupSpec.Template.Spec.Containers { + container := &headGroupSpec.Template.Spec.Containers[j] + container.VolumeMounts = append(container.VolumeMounts, volumeMount) + } + + for i := range w.RayClusterSpec.WorkerGroupSpecs { + workerGroupSpec := &w.RayClusterSpec.WorkerGroupSpecs[i] + for j := range workerGroupSpec.Template.Spec.InitContainers { + container := &workerGroupSpec.Template.Spec.InitContainers[j] + container.VolumeMounts = append(container.VolumeMounts, volumeMount) + } + for j := range workerGroupSpec.Template.Spec.Containers { + container := &workerGroupSpec.Template.Spec.Containers[j] + container.VolumeMounts = append(container.VolumeMounts, volumeMount) + } + } + + return w +} + +// Replicas set Replicas on WorkerGroupSpec. +func (w *RayClusterSpecWrapper) Replicas(groupName string, replicas int32) *RayClusterSpecWrapper { + for i := range w.RayClusterSpec.WorkerGroupSpecs { + if w.RayClusterSpec.WorkerGroupSpecs[i].GroupName == groupName { + w.RayClusterSpec.WorkerGroupSpecs[i].Replicas = &replicas + break + } + } + + return w +} + +// MinReplicas set MinReplicas on WorkerGroupSpec. +func (w *RayClusterSpecWrapper) MinReplicas(groupName string, minReplicas int32) *RayClusterSpecWrapper { + for i := range w.RayClusterSpec.WorkerGroupSpecs { + if w.RayClusterSpec.WorkerGroupSpecs[i].GroupName == groupName { + w.RayClusterSpec.WorkerGroupSpecs[i].MinReplicas = &minReplicas + break + } + } + + return w +} + +// MaxReplicas set MaxReplicas on WorkerGroupSpec. +func (w *RayClusterSpecWrapper) MaxReplicas(groupName string, maxReplicas int32) *RayClusterSpecWrapper { + for i := range w.RayClusterSpec.WorkerGroupSpecs { + if w.RayClusterSpec.WorkerGroupSpecs[i].GroupName == groupName { + w.RayClusterSpec.WorkerGroupSpecs[i].MaxReplicas = &maxReplicas + break + } + } + + return w +} + +// Suspend set suspend. +func (w *RayClusterSpecWrapper) Suspend(suspend bool) *RayClusterSpecWrapper { + w.RayClusterSpec.Suspend = ptr.To(suspend) + return w +} diff --git a/pkg/testing/wrappers/ray_cluster_template_wrappers.go b/pkg/testing/wrappers/ray_cluster_template_wrappers.go new file mode 100644 index 0000000..bc3459d --- /dev/null +++ b/pkg/testing/wrappers/ray_cluster_template_wrappers.go @@ -0,0 +1,75 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// RayClusterTemplateWrapper wraps a RayClusterTemplate. +type RayClusterTemplateWrapper struct{ v1alpha1.RayClusterTemplate } + +// MakeRayClusterTemplate creates a wrapper for a RayClusterTemplate +func MakeRayClusterTemplate(name, ns string) *RayClusterTemplateWrapper { + return &RayClusterTemplateWrapper{ + RayClusterTemplate: v1alpha1.RayClusterTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + }, + } +} + +// Obj returns the inner RayClusterTemplate. +func (w *RayClusterTemplateWrapper) Obj() *v1alpha1.RayClusterTemplate { + return &w.RayClusterTemplate +} + +// Clone RayClusterTemplateWrapper. +func (w *RayClusterTemplateWrapper) Clone() *RayClusterTemplateWrapper { + return &RayClusterTemplateWrapper{ + RayClusterTemplate: *w.RayClusterTemplate.DeepCopy(), + } +} + +// Label sets the label key and value. +func (w *RayClusterTemplateWrapper) Label(key, value string) *RayClusterTemplateWrapper { + if w.Template.ObjectMeta.Labels == nil { + w.Template.ObjectMeta.Labels = make(map[string]string) + } + w.Template.ObjectMeta.Labels[key] = value + return w +} + +// Annotation sets the label key and value. +func (w *RayClusterTemplateWrapper) Annotation(key, value string) *RayClusterTemplateWrapper { + if w.Template.ObjectMeta.Annotations == nil { + w.Template.ObjectMeta.Annotations = make(map[string]string) + } + w.Template.ObjectMeta.Annotations[key] = value + return w +} + +// Spec set entrypoint. +func (w *RayClusterTemplateWrapper) Spec(spec rayv1.RayClusterSpec) *RayClusterTemplateWrapper { + w.Template.Spec = spec + return w +} diff --git a/pkg/testing/wrappers/ray_cluster_wrappers.go b/pkg/testing/wrappers/ray_cluster_wrappers.go new file mode 100644 index 0000000..ad0f45e --- /dev/null +++ b/pkg/testing/wrappers/ray_cluster_wrappers.go @@ -0,0 +1,178 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + "time" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +// RayClusterWrapper wraps a RayCluster. +type RayClusterWrapper struct{ rayv1.RayCluster } + +// MakeRayCluster creates a wrapper for a RayCluster +func MakeRayCluster(name, ns string) *RayClusterWrapper { + return &RayClusterWrapper{ + rayv1.RayCluster{ + TypeMeta: metav1.TypeMeta{Kind: "RayCluster", APIVersion: "ray.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + }, + } +} + +// Obj returns the inner RayCluster. +func (j *RayClusterWrapper) Obj() *rayv1.RayCluster { + return &j.RayCluster +} + +// GenerateName updates generateName. +func (j *RayClusterWrapper) GenerateName(v string) *RayClusterWrapper { + j.ObjectMeta.GenerateName = v + return j +} + +// CreationTimestamp sets the .metadata.creationTimestamp +func (j *RayClusterWrapper) CreationTimestamp(t time.Time) *RayClusterWrapper { + j.RayCluster.ObjectMeta.CreationTimestamp = metav1.NewTime(t) + return j +} + +// Profile sets the profile label. +func (j *RayClusterWrapper) Profile(v string) *RayClusterWrapper { + return j.Label(constants.ProfileLabel, v) +} + +// Mode sets the profile label. +func (j *RayClusterWrapper) Mode(v v1alpha1.ApplicationProfileMode) *RayClusterWrapper { + return j.Label(constants.ModeLabel, string(v)) +} + +// LocalQueue sets the localqueue label. +func (j *RayClusterWrapper) LocalQueue(v string) *RayClusterWrapper { + return j.Label(kueueconstants.QueueLabel, v) +} + +// Label sets the label key and value. +func (j *RayClusterWrapper) Label(key, value string) *RayClusterWrapper { + if j.Labels == nil { + j.Labels = make(map[string]string) + } + j.ObjectMeta.Labels[key] = value + return j +} + +// Annotation sets the label key and value. +func (j *RayClusterWrapper) Annotation(key, value string) *RayClusterWrapper { + if j.Annotations == nil { + j.Annotations = make(map[string]string) + } + j.ObjectMeta.Annotations[key] = value + return j +} + +// WithWorkerGroupSpec add worker group to the ray cluster template. +func (j *RayClusterWrapper) WithWorkerGroupSpec(spec rayv1.WorkerGroupSpec) *RayClusterWrapper { + j.RayCluster.Spec.WorkerGroupSpecs = append(j.RayCluster.Spec.WorkerGroupSpecs, spec) + return j +} + +// Spec set job spec. +func (j *RayClusterWrapper) Spec(spec rayv1.RayClusterSpec) *RayClusterWrapper { + j.RayCluster.Spec = spec + return j +} + +// DesiredWorkerReplicas set DesiredWorkerReplicas. +func (j *RayClusterWrapper) DesiredWorkerReplicas(desiredWorkerReplicas int32) *RayClusterWrapper { + j.RayCluster.Status.DesiredWorkerReplicas = desiredWorkerReplicas + return j +} + +// ReadyWorkerReplicas set ReadyWorkerReplicas. +func (j *RayClusterWrapper) ReadyWorkerReplicas(readyWorkerReplicas int32) *RayClusterWrapper { + j.RayCluster.Status.ReadyWorkerReplicas = readyWorkerReplicas + return j +} + +// AvailableWorkerReplicas set AvailableWorkerReplicas. +func (j *RayClusterWrapper) AvailableWorkerReplicas(availableWorkerReplicas int32) *RayClusterWrapper { + j.RayCluster.Status.AvailableWorkerReplicas = availableWorkerReplicas + return j +} + +// DesiredCPU set DesiredCPU. +func (j *RayClusterWrapper) DesiredCPU(desiredCPU resource.Quantity) *RayClusterWrapper { + j.RayCluster.Status.DesiredCPU = desiredCPU + return j +} + +// DesiredMemory set DesiredMemory. +func (j *RayClusterWrapper) DesiredMemory(desiredMemory resource.Quantity) *RayClusterWrapper { + j.RayCluster.Status.DesiredMemory = desiredMemory + return j +} + +// DesiredGPU set DesiredGPU. +func (j *RayClusterWrapper) DesiredGPU(desiredGPU resource.Quantity) *RayClusterWrapper { + j.RayCluster.Status.DesiredGPU = desiredGPU + return j +} + +// DesiredTPU set DesiredTPU. +func (j *RayClusterWrapper) DesiredTPU(desiredTPU resource.Quantity) *RayClusterWrapper { + j.RayCluster.Status.DesiredTPU = desiredTPU + return j +} + +// MinWorkerReplicas set MinWorkerReplicas. +func (j *RayClusterWrapper) MinWorkerReplicas(minWorkerReplicas int32) *RayClusterWrapper { + j.RayCluster.Status.MinWorkerReplicas = minWorkerReplicas + return j +} + +// MaxWorkerReplicas set MaxWorkerReplicas. +func (j *RayClusterWrapper) MaxWorkerReplicas(maxWorkerReplicas int32) *RayClusterWrapper { + j.RayCluster.Status.MaxWorkerReplicas = maxWorkerReplicas + return j +} + +// State set State. +func (j *RayClusterWrapper) State(state rayv1.ClusterState) *RayClusterWrapper { + j.RayCluster.Status.State = state + return j +} + +// Reason set Reason. +func (j *RayClusterWrapper) Reason(reason string) *RayClusterWrapper { + j.RayCluster.Status.Reason = reason + return j +} + +// Priority sets the workload priority class label. +func (j *RayClusterWrapper) Priority(v string) *RayClusterWrapper { + return j.Label(kueueconstants.WorkloadPriorityClassLabel, v) +} diff --git a/pkg/testing/wrappers/ray_job_template_wrappers.go b/pkg/testing/wrappers/ray_job_template_wrappers.go new file mode 100644 index 0000000..cacd4e0 --- /dev/null +++ b/pkg/testing/wrappers/ray_job_template_wrappers.go @@ -0,0 +1,81 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// RayJobTemplateWrapper wraps a RayJobTemplate. +type RayJobTemplateWrapper struct{ v1alpha1.RayJobTemplate } + +// MakeRayJobTemplate creates a wrapper for a RayJobTemplate +func MakeRayJobTemplate(name, ns string) *RayJobTemplateWrapper { + return &RayJobTemplateWrapper{ + RayJobTemplate: v1alpha1.RayJobTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + }, + } +} + +// Obj returns the inner RayJobTemplate. +func (w *RayJobTemplateWrapper) Obj() *v1alpha1.RayJobTemplate { + return &w.RayJobTemplate +} + +// Clone RayJobTemplateWrapper. +func (w *RayJobTemplateWrapper) Clone() *RayJobTemplateWrapper { + return &RayJobTemplateWrapper{ + RayJobTemplate: *w.RayJobTemplate.DeepCopy(), + } +} + +// Label sets the label key and value. +func (w *RayJobTemplateWrapper) Label(key, value string) *RayJobTemplateWrapper { + if w.Template.ObjectMeta.Labels == nil { + w.Template.ObjectMeta.Labels = make(map[string]string) + } + w.Template.ObjectMeta.Labels[key] = value + return w +} + +// Annotation sets the label key and value. +func (w *RayJobTemplateWrapper) Annotation(key, value string) *RayJobTemplateWrapper { + if w.Template.ObjectMeta.Annotations == nil { + w.Template.ObjectMeta.Annotations = make(map[string]string) + } + w.Template.ObjectMeta.Annotations[key] = value + return w +} + +// Entrypoint set entrypoint. +func (w *RayJobTemplateWrapper) Entrypoint(entrypoint string) *RayJobTemplateWrapper { + w.Template.Spec.Entrypoint = entrypoint + return w +} + +// WithRayClusterSpec set entrypoint. +func (w *RayJobTemplateWrapper) WithRayClusterSpec(spec *rayv1.RayClusterSpec) *RayJobTemplateWrapper { + w.Template.Spec.RayClusterSpec = spec + return w +} diff --git a/pkg/testing/wrappers/ray_job_wrappers.go b/pkg/testing/wrappers/ray_job_wrappers.go new file mode 100644 index 0000000..b8557d6 --- /dev/null +++ b/pkg/testing/wrappers/ray_job_wrappers.go @@ -0,0 +1,181 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + "time" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + rayutil "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +// RayJobWrapper wraps a RayJob. +type RayJobWrapper struct{ rayv1.RayJob } + +// MakeRayJob creates a wrapper for a RayJob +func MakeRayJob(name, ns string) *RayJobWrapper { + return &RayJobWrapper{ + rayv1.RayJob{ + TypeMeta: metav1.TypeMeta{Kind: "RayJob", APIVersion: "ray.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + }, + } +} + +// Obj returns the inner RayJob. +func (j *RayJobWrapper) Obj() *rayv1.RayJob { + return &j.RayJob +} + +// GenerateName updates generateName. +func (j *RayJobWrapper) GenerateName(v string) *RayJobWrapper { + j.ObjectMeta.GenerateName = v + return j +} + +// CreationTimestamp sets the .metadata.creationTimestamp +func (j *RayJobWrapper) CreationTimestamp(t time.Time) *RayJobWrapper { + j.RayJob.ObjectMeta.CreationTimestamp = metav1.NewTime(t) + return j +} + +// Profile sets the profile label. +func (j *RayJobWrapper) Profile(v string) *RayJobWrapper { + return j.Label(constants.ProfileLabel, v) +} + +// Mode sets the profile label. +func (j *RayJobWrapper) Mode(v v1alpha1.ApplicationProfileMode) *RayJobWrapper { + return j.Label(constants.ModeLabel, string(v)) +} + +// LocalQueue sets the localqueue label. +func (j *RayJobWrapper) LocalQueue(v string) *RayJobWrapper { + return j.Label(kueueconstants.QueueLabel, v) +} + +// Label sets the label key and value. +func (j *RayJobWrapper) Label(key, value string) *RayJobWrapper { + if j.Labels == nil { + j.Labels = make(map[string]string) + } + j.ObjectMeta.Labels[key] = value + return j +} + +// Annotation sets the label key and value. +func (j *RayJobWrapper) Annotation(key, value string) *RayJobWrapper { + if j.Annotations == nil { + j.Annotations = make(map[string]string) + } + j.ObjectMeta.Annotations[key] = value + return j +} + +// WithWorkerGroupSpec add worker group to the ray cluster template. +func (j *RayJobWrapper) WithWorkerGroupSpec(spec rayv1.WorkerGroupSpec) *RayJobWrapper { + if j.RayJob.Spec.RayClusterSpec == nil { + j.RayJob.Spec.RayClusterSpec = &rayv1.RayClusterSpec{} + } + + j.RayJob.Spec.RayClusterSpec.WorkerGroupSpecs = append(j.RayJob.Spec.RayClusterSpec.WorkerGroupSpecs, spec) + + return j +} + +// Spec set job spec. +func (j *RayJobWrapper) Spec(spec rayv1.RayJobSpec) *RayJobWrapper { + j.RayJob.Spec = spec + return j +} + +// Entrypoint set entrypoint. +func (j *RayJobWrapper) Entrypoint(entrypoint string) *RayJobWrapper { + j.RayJob.Spec.Entrypoint = entrypoint + return j +} + +// JobStatus set jobStatus. +func (j *RayJobWrapper) JobStatus(jobStatus rayv1.JobStatus) *RayJobWrapper { + j.RayJob.Status.JobStatus = jobStatus + return j +} + +// JobDeploymentStatus set jobDeploymentStatus. +func (j *RayJobWrapper) JobDeploymentStatus(jobDeploymentStatus rayv1.JobDeploymentStatus) *RayJobWrapper { + j.RayJob.Status.JobDeploymentStatus = jobDeploymentStatus + return j +} + +// Reason set reason. +func (j *RayJobWrapper) Reason(reason rayv1.JobFailedReason) *RayJobWrapper { + j.RayJob.Status.Reason = reason + return j +} + +// Message set message. +func (j *RayJobWrapper) Message(message string) *RayJobWrapper { + j.RayJob.Status.Message = message + return j +} + +// StartTime set startTime. +func (j *RayJobWrapper) StartTime(startTime time.Time) *RayJobWrapper { + j.RayJob.Status.StartTime = ptr.To(metav1.NewTime(startTime)) + return j +} + +// EndTime set endTime. +func (j *RayJobWrapper) EndTime(endTime time.Time) *RayJobWrapper { + j.RayJob.Status.EndTime = ptr.To(metav1.NewTime(endTime)) + return j +} + +// Suspend set suspend. +func (j *RayJobWrapper) Suspend(suspend bool) *RayJobWrapper { + j.RayJob.Spec.Suspend = suspend + return j +} + +// WithRayClusterLabelSelector sets the ClusterSelector. +func (j *RayJobWrapper) WithRayClusterLabelSelector(v string) *RayJobWrapper { + if j.RayJob.Spec.ClusterSelector == nil { + j.RayJob.Spec.ClusterSelector = map[string]string{} + } + j.RayJob.Spec.ClusterSelector[rayutil.RayClusterLabelKey] = v + return j +} + +// RayClusterName set rayClusterName. +func (j *RayJobWrapper) RayClusterName(rayClusterName string) *RayJobWrapper { + j.RayJob.Status.RayClusterName = rayClusterName + return j +} + +// Priority sets the workload priority class label. +func (j *RayJobWrapper) Priority(v string) *RayJobWrapper { + return j.Label(kueueconstants.WorkloadPriorityClassLabel, v) +} diff --git a/pkg/testing/wrappers/service_wrappers.go b/pkg/testing/wrappers/service_wrappers.go new file mode 100644 index 0000000..8a2abba --- /dev/null +++ b/pkg/testing/wrappers/service_wrappers.go @@ -0,0 +1,92 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +// ServiceWrapper wraps a Service. +type ServiceWrapper struct{ corev1.Service } + +// MakeService creates a wrapper for a Service +func MakeService(name, ns string) *ServiceWrapper { + return &ServiceWrapper{ + corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + }, + } +} + +// Obj returns the inner Service. +func (w *ServiceWrapper) Obj() *corev1.Service { + return &w.Service +} + +// Profile sets the profile label. +func (w *ServiceWrapper) Profile(v string) *ServiceWrapper { + return w.Label(constants.ProfileLabel, v) +} + +// Mode sets the mode label. +func (w *ServiceWrapper) Mode(v v1alpha1.ApplicationProfileMode) *ServiceWrapper { + return w.Label(constants.ModeLabel, string(v)) +} + +// LocalQueue sets the localqueue label. +func (w *ServiceWrapper) LocalQueue(v string) *ServiceWrapper { + return w.Label(kueueconstants.QueueLabel, v) +} + +// Label sets the label key and value. +func (w *ServiceWrapper) Label(key, value string) *ServiceWrapper { + if w.Labels == nil { + w.Labels = make(map[string]string) + } + w.ObjectMeta.Labels[key] = value + return w +} + +// ClusterIP sets clusterIP. +func (w *ServiceWrapper) ClusterIP(clusterIP string) *ServiceWrapper { + w.Service.Spec.ClusterIP = clusterIP + return w +} + +// Selector sets the selector key and value. +func (w *ServiceWrapper) Selector(key, value string) *ServiceWrapper { + if w.Service.Spec.Selector == nil { + w.Service.Spec.Selector = make(map[string]string) + } + w.Service.Spec.Selector[key] = value + return w +} + +// WithOwnerReference adds the owner reference. +func (w *ServiceWrapper) WithOwnerReference(ref metav1.OwnerReference) *ServiceWrapper { + w.OwnerReferences = append(w.OwnerReferences, ref) + return w +} diff --git a/pkg/testing/wrappers/supported_mode_wrappers.go b/pkg/testing/wrappers/supported_mode_wrappers.go new file mode 100644 index 0000000..2788ef6 --- /dev/null +++ b/pkg/testing/wrappers/supported_mode_wrappers.go @@ -0,0 +1,39 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// SupportedModeWrapper wraps a SupportedMode. +type SupportedModeWrapper struct{ v1alpha1.SupportedMode } + +// MakeSupportedMode creates a wrapper for a SupportedMode +func MakeSupportedMode(name v1alpha1.ApplicationProfileMode, template v1alpha1.TemplateReference) *SupportedModeWrapper { + return &SupportedModeWrapper{ + SupportedMode: v1alpha1.SupportedMode{ + Name: name, + Template: template, + }, + } +} + +// Obj returns the inner SupportedMode. +func (m *SupportedModeWrapper) Obj() *v1alpha1.SupportedMode { + return &m.SupportedMode +} diff --git a/pkg/testing/wrappers/volume_bundle_wrappers.go b/pkg/testing/wrappers/volume_bundle_wrappers.go new file mode 100644 index 0000000..d220c93 --- /dev/null +++ b/pkg/testing/wrappers/volume_bundle_wrappers.go @@ -0,0 +1,71 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// VolumeBundleWrapper wraps a VolumeBundle. +type VolumeBundleWrapper struct{ v1alpha1.VolumeBundle } + +// MakeVolumeBundle creates a wrapper for a VolumeBundle +func MakeVolumeBundle(name, namespace string) *VolumeBundleWrapper { + return &VolumeBundleWrapper{ + VolumeBundle: v1alpha1.VolumeBundle{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + }, + } +} + +// Obj returns the inner VolumeBundle. +func (vb *VolumeBundleWrapper) Obj() *v1alpha1.VolumeBundle { + return &vb.VolumeBundle +} + +// WithVolume add volume on the volumes. +func (vb *VolumeBundleWrapper) WithVolume(name, localObjectReferenceName string) *VolumeBundleWrapper { + vb.VolumeBundle.Spec.Volumes = append(vb.VolumeBundle.Spec.Volumes, corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: localObjectReferenceName, + }, + }, + }, + }) + return vb +} + +// WithVolumeMount add volume mount on the volumes. +func (vb *VolumeBundleWrapper) WithVolumeMount(volumeMount corev1.VolumeMount) *VolumeBundleWrapper { + vb.VolumeBundle.Spec.ContainerVolumeMounts = append(vb.VolumeBundle.Spec.ContainerVolumeMounts, volumeMount) + return vb +} + +// WithEnvVar add EnvVar to EnvVars. +func (vb *VolumeBundleWrapper) WithEnvVar(envVar corev1.EnvVar) *VolumeBundleWrapper { + vb.Spec.EnvVars = append(vb.Spec.EnvVars, envVar) + return vb +} diff --git a/pkg/testing/wrappers/worker_group_spec_wrapper.go b/pkg/testing/wrappers/worker_group_spec_wrapper.go new file mode 100644 index 0000000..23f4c3d --- /dev/null +++ b/pkg/testing/wrappers/worker_group_spec_wrapper.go @@ -0,0 +1,94 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + corev1 "k8s.io/api/core/v1" +) + +// WorkerGroupSpecWrapper wraps a WorkerGroupSpec. +type WorkerGroupSpecWrapper struct{ rayv1.WorkerGroupSpec } + +// MakeWorkerGroupSpec creates a wrapper for a WorkerGroupSpec +func MakeWorkerGroupSpec(groupName string) *WorkerGroupSpecWrapper { + return &WorkerGroupSpecWrapper{WorkerGroupSpec: rayv1.WorkerGroupSpec{GroupName: groupName}} +} + +// Obj returns the inner WorkerGroupSpec. +func (w *WorkerGroupSpecWrapper) Obj() *rayv1.WorkerGroupSpec { + return &w.WorkerGroupSpec +} + +// Clone WorkerGroupSpecWrapper. +func (w *WorkerGroupSpecWrapper) Clone() *WorkerGroupSpecWrapper { + return &WorkerGroupSpecWrapper{ + WorkerGroupSpec: *w.WorkerGroupSpec.DeepCopy(), + } +} + +// WithInitContainer add init container to the pod template. +func (w *WorkerGroupSpecWrapper) WithInitContainer(container corev1.Container) *WorkerGroupSpecWrapper { + w.Template.Spec.InitContainers = append(w.Template.Spec.InitContainers, container) + return w +} + +// WithContainer add container to the pod template. +func (w *WorkerGroupSpecWrapper) WithContainer(container corev1.Container) *WorkerGroupSpecWrapper { + w.Template.Spec.Containers = append(w.Template.Spec.Containers, container) + return w +} + +// WithVolume add volume to the WorkerGroupSpec. +func (w *WorkerGroupSpecWrapper) WithVolume(name, localObjectReferenceName string) *WorkerGroupSpecWrapper { + w.Template.Spec.Volumes = append(w.Template.Spec.Volumes, corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: localObjectReferenceName, + }, + }, + }, + }) + + return w +} + +// Replicas set replicas on WorkerGroupSpec. +func (w *WorkerGroupSpecWrapper) Replicas(replicas int32) *WorkerGroupSpecWrapper { + w.WorkerGroupSpec.Replicas = &replicas + return w +} + +// MinReplicas set minReplicas on WorkerGroupSpec. +func (w *WorkerGroupSpecWrapper) MinReplicas(minReplicas int32) *WorkerGroupSpecWrapper { + w.WorkerGroupSpec.MinReplicas = &minReplicas + return w +} + +// MaxReplicas set maxReplicas on WorkerGroupSpec. +func (w *WorkerGroupSpecWrapper) MaxReplicas(maxReplicas int32) *WorkerGroupSpecWrapper { + w.WorkerGroupSpec.MaxReplicas = &maxReplicas + return w +} + +// RayStartParams set rayStartParams on WorkerGroupSpec. +func (w *WorkerGroupSpecWrapper) RayStartParams(rayStartParams map[string]string) *WorkerGroupSpecWrapper { + w.WorkerGroupSpec.RayStartParams = rayStartParams + return w +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go new file mode 100644 index 0000000..a11fc53 --- /dev/null +++ b/test/e2e/e2e_test.go @@ -0,0 +1,53 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "os/exec" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/test/util" +) + +var _ = ginkgo.Describe("kjobctl", ginkgo.Ordered, func() { + var ns *corev1.Namespace + + ginkgo.BeforeEach(func() { + ns = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "e2e-", + }, + } + gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed()) + }) + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + }) + + ginkgo.It("Should print kjobctl information", func() { + cmd := exec.Command(kjobctlPath, "--help") + out, err := util.Run(cmd) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + info := util.GetNonEmptyLines(string(out)) + gomega.Expect(info[0]).To(gomega.Equal("ML/AI/Batch Jobs Made Easy")) + }) +}) diff --git a/test/e2e/slurm_test.go b/test/e2e/slurm_test.go new file mode 100644 index 0000000..eaec679 --- /dev/null +++ b/test/e2e/slurm_test.go @@ -0,0 +1,369 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "regexp" + "strconv" + + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/kueue/pkg/util/maps" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + kjobctlconstants "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/test/util" +) + +const ( + BatchJobNameLabel = "batch.kubernetes.io/job-name" + BatchJobCompletionIndexLabel = "batch.kubernetes.io/job-completion-index" +) + +var _ = ginkgo.Describe("Slurm", ginkgo.Ordered, func() { + var ( + ns *corev1.Namespace + profile *v1alpha1.ApplicationProfile + jobTemplate *v1alpha1.JobTemplate + ) + + ginkgo.BeforeEach(func() { + ns = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "e2e-slurm-", + }, + } + gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed()) + + jobTemplate = wrappers.MakeJobTemplate("job-template", ns.Name). + RestartPolicy(corev1.RestartPolicyNever). + BackoffLimitPerIndex(0). + WithContainer(*wrappers.MakeContainer("c1", util.E2eTestBashImage).Obj()). + Obj() + gomega.Expect(k8sClient.Create(ctx, jobTemplate)).To(gomega.Succeed()) + + profile = wrappers.MakeApplicationProfile("profile", ns.Name). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "job-template").Obj()). + Obj() + gomega.Expect(k8sClient.Create(ctx, profile)).To(gomega.Succeed()) + }) + + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + }) + + ginkgo.DescribeTable("should be created", func( + args, slurmArgs []string, + expectCompletions, expectParallelism int32, + expectCommonVars map[string]string, expectPods []map[string]map[string]string, + withFirstNodeIP bool, + ) { + ginkgo.By("Create temporary file") + script, err := os.CreateTemp("", "e2e-slurm-") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer script.Close() + defer os.Remove(script.Name()) + + ginkgo.By("Prepare script", func() { + _, err := script.WriteString("#!/bin/bash\nwhile true; do printenv | grep SLURM_ > /env.out; sleep 0.25; done") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + var jobName, configMapName, serviceName string + + ginkgo.By("Create slurm", func() { + cmdArgs := []string{"create", "slurm", "-n", ns.Name, "--profile", profile.Name} + cmdArgs = append(cmdArgs, args...) + cmdArgs = append(cmdArgs, "--", script.Name()) + cmdArgs = append(cmdArgs, slurmArgs...) + + cmd := exec.Command(kjobctlPath, cmdArgs...) + out, err := util.Run(cmd) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%s: %s", err, out) + gomega.Expect(out).NotTo(gomega.BeEmpty()) + + jobName, configMapName, serviceName, err = parseSlurmCreateOutput(out, profile.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(jobName).NotTo(gomega.BeEmpty()) + gomega.Expect(configMapName).NotTo(gomega.BeEmpty()) + gomega.Expect(serviceName).NotTo(gomega.BeEmpty()) + }) + + job := &batchv1.Job{} + configMap := &corev1.Service{} + service := &corev1.Service{} + + ginkgo.By("Check slurm is created", func() { + gomega.Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: ns.Name, Name: jobName}, job)).To(gomega.Succeed()) + gomega.Expect(ptr.Deref(job.Spec.Completions, 1)).To(gomega.Equal(expectCompletions)) + gomega.Expect(ptr.Deref(job.Spec.Parallelism, 1)).To(gomega.Equal(expectParallelism)) + gomega.Expect(job.Annotations).To(gomega.HaveKeyWithValue(kjobctlconstants.ScriptAnnotation, script.Name())) + gomega.Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: ns.Name, Name: configMapName}, configMap)).To(gomega.Succeed()) + gomega.Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: ns.Name, Name: serviceName}, service)).To(gomega.Succeed()) + }) + + var firstPod *corev1.Pod + + for completionIndex, expectPod := range expectPods { + podList := &corev1.PodList{} + + ginkgo.By(fmt.Sprintf("Wait for pod with completion index %d is running", completionIndex), func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.List(ctx, podList, client.InNamespace(ns.Name), client.MatchingLabels(map[string]string{ + BatchJobNameLabel: job.Name, + BatchJobCompletionIndexLabel: strconv.Itoa(completionIndex), + }))).Should(gomega.Succeed()) + g.Expect(podList.Items).Should(gomega.HaveLen(1)) + g.Expect(podList.Items[0].Status.Phase).To(gomega.Equal(corev1.PodRunning)) + if completionIndex == 0 { + firstPod = &podList.Items[0] + } + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + pod := podList.Items[0] + + for containerName, expectContainerVars := range expectPod { + ginkgo.By(fmt.Sprintf("Check env variables in index %d and container name %s", completionIndex, containerName), func() { + wantOut := maps.MergeKeepFirst(expectContainerVars, expectCommonVars) + if withFirstNodeIP { + wantOut["SLURM_JOB_FIRST_NODE_IP"] = firstPod.Status.PodIP + } + + gomega.Eventually(func(g gomega.Gomega) { + out, outErr, err := util.KExecute(ctx, cfg, restClient, ns.Name, pod.Name, containerName) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(string(outErr)).To(gomega.BeEmpty()) + g.Expect(parseSlurmEnvOutput(out)).To(gomega.BeComparableTo(wantOut, + cmpopts.AcyclicTransformer("RemoveGeneratedNameSuffixInMap", func(m map[string]string) map[string]string { + for key, val := range m { + m[key] = regexp.MustCompile("(profile-slurm)(-.{5})").ReplaceAllString(val, "$1") + } + return m + }), + )) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + } + + if completionIndex != 0 || !withFirstNodeIP { + gomega.Expect(k8sClient.Delete(ctx, &pod)).To(gomega.Succeed()) + } + } + }, + ginkgo.Entry( + "without arguments", + []string{}, []string{}, + int32(1), int32(1), + map[string]string{ + "SLURM_NTASKS_PER_NODE": "1", + "SLURM_ARRAY_JOB_ID": "1", + "SLURM_MEM_PER_CPU": "", + "SLURM_GPUS": "", + "SLURM_NNODES": "1", + "SLURM_MEM_PER_GPU": "", + "SLURM_NTASKS": "1", + "SLURM_ARRAY_TASK_COUNT": "1", + "SLURM_TASKS_PER_NODE": "1", + "SLURM_CPUS_PER_TASK": "", + "SLURM_ARRAY_TASK_MAX": "0", + "SLURM_CPUS_PER_GPU": "", + "SLURM_SUBMIT_DIR": "/slurm/scripts", + "SLURM_NPROCS": "1", + "SLURM_CPUS_ON_NODE": "", + "SLURM_ARRAY_TASK_MIN": "0", + "SLURM_JOB_NODELIST": "profile-slurm-xxxxx-0.profile-slurm-xxxxx", + "SLURM_JOB_CPUS_PER_NODE": "", + "SLURM_JOB_FIRST_NODE": "profile-slurm-xxxxx-1.profile-slurm-xxxxx", + "SLURM_MEM_PER_NODE": "", + "SLURM_JOB_FIRST_NODE_IP": "", + }, + []map[string]map[string]string{ + { + "c1": { + "SLURM_ARRAY_TASK_ID": "0", + "SLURM_JOB_ID": "1", + "SLURM_JOBID": "1", + "SLURM_SUBMIT_HOST": "profile-slurm-xxxxx-0", + }, + }, + }, + false, + ), + ginkgo.Entry( + "with --first-node-ip", + []string{"--first-node-ip"}, []string{"--array", "1-5%2", "--nodes", "2", "--ntasks", "2"}, + int32(3), int32(2), + map[string]string{ + "SLURM_NTASKS_PER_NODE": "2", + "SLURM_ARRAY_JOB_ID": "1", + "SLURM_MEM_PER_CPU": "", + "SLURM_GPUS": "", + "SLURM_NNODES": "2", + "SLURM_MEM_PER_GPU": "", + "SLURM_NTASKS": "2", + "SLURM_ARRAY_TASK_COUNT": "5", + "SLURM_TASKS_PER_NODE": "2", + "SLURM_CPUS_PER_TASK": "", + "SLURM_ARRAY_TASK_MAX": "5", + "SLURM_CPUS_PER_GPU": "", + "SLURM_SUBMIT_DIR": "/slurm/scripts", + "SLURM_NPROCS": "2", + "SLURM_CPUS_ON_NODE": "", + "SLURM_ARRAY_TASK_MIN": "1", + "SLURM_JOB_NODELIST": "profile-slurm-xxxxx-0.profile-slurm-xxxxx,profile-slurm-xxxxx-1.profile-slurm-xxxxx", + "SLURM_JOB_CPUS_PER_NODE": "", + "SLURM_JOB_FIRST_NODE": "profile-slurm-xxxxx-1.profile-slurm-xxxxx", + "SLURM_MEM_PER_NODE": "", + "SLURM_JOB_FIRST_NODE_IP": "", + }, + []map[string]map[string]string{ + { + "c1-0": { + "SLURM_ARRAY_TASK_ID": "1", + "SLURM_JOB_ID": "1", + "SLURM_JOBID": "1", + "SLURM_SUBMIT_HOST": "profile-slurm-xxxxx-0", + }, + "c1-1": { + "SLURM_ARRAY_TASK_ID": "2", + "SLURM_JOB_ID": "2", + "SLURM_JOBID": "2", + "SLURM_SUBMIT_HOST": "profile-slurm-xxxxx-0", + }, + }, + { + "c1-0": { + "SLURM_ARRAY_TASK_ID": "3", + "SLURM_JOB_ID": "3", + "SLURM_JOBID": "3", + "SLURM_SUBMIT_HOST": "profile-slurm-xxxxx-1", + }, + "c1-1": { + "SLURM_ARRAY_TASK_ID": "4", + "SLURM_JOB_ID": "4", + "SLURM_JOBID": "4", + "SLURM_SUBMIT_HOST": "profile-slurm-xxxxx-1", + }, + }, + { + "c1-0": { + "SLURM_ARRAY_TASK_ID": "5", + "SLURM_JOB_ID": "5", + "SLURM_JOBID": "5", + "SLURM_SUBMIT_HOST": "profile-slurm-xxxxx-2", + }, + }, + }, + true, + ), + ) + + ginkgo.When("delete", func() { + ginkgo.It("should delete job and child objects", func() { + ginkgo.By("Create temporary file") + script, err := os.CreateTemp("", "e2e-slurm-") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer script.Close() + defer os.Remove(script.Name()) + + ginkgo.By("Prepare script", func() { + _, err := script.WriteString("#!/bin/bash\nsleep 600") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + var jobName, configMapName, serviceName string + + ginkgo.By("Create slurm", func() { + cmdArgs := []string{"create", "slurm", "-n", ns.Name, "--profile", profile.Name} + cmdArgs = append(cmdArgs, "--", script.Name()) + + cmd := exec.Command(kjobctlPath, cmdArgs...) + out, err := util.Run(cmd) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%s: %s", err, out) + gomega.Expect(out).NotTo(gomega.BeEmpty()) + + jobName, configMapName, serviceName, err = parseSlurmCreateOutput(out, profile.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(jobName).NotTo(gomega.BeEmpty()) + gomega.Expect(configMapName).NotTo(gomega.BeEmpty()) + gomega.Expect(serviceName).NotTo(gomega.BeEmpty()) + }) + + job := &batchv1.Job{} + configMap := &corev1.Service{} + service := &corev1.Service{} + + ginkgo.By("Check slurm is created", func() { + gomega.Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: ns.Name, Name: jobName}, job)).To(gomega.Succeed()) + gomega.Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: ns.Name, Name: configMapName}, configMap)).To(gomega.Succeed()) + gomega.Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: ns.Name, Name: serviceName}, service)).To(gomega.Succeed()) + }) + + ginkgo.By("Delete slurm", func() { + cmd := exec.Command(kjobctlPath, "delete", "slurm", "-n", ns.Name, jobName) + out, err := util.Run(cmd) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%s: %s", err, out) + gomega.Expect(string(out)).To(gomega.Equal(fmt.Sprintf("job.batch/%s deleted\n", jobName))) + }) + + ginkgo.By("Check job and child objects are deleted", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(errors.IsNotFound(k8sClient.Get(ctx, client.ObjectKeyFromObject(job), job))).To(gomega.BeTrue()) + g.Expect(errors.IsNotFound(k8sClient.Get(ctx, client.ObjectKeyFromObject(configMap), configMap))).To(gomega.BeTrue()) + g.Expect(errors.IsNotFound(k8sClient.Get(ctx, client.ObjectKeyFromObject(service), service))).To(gomega.BeTrue()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + }) + }) +}) + +func parseSlurmCreateOutput(output []byte, profileName string) (string, string, string, error) { + output = bytes.ReplaceAll(output, []byte("\n"), []byte("")) + re := regexp.MustCompile(fmt.Sprintf(`^job\.batch\/(%[1]s-slurm-.{5}) createdconfigmap\/(%[1]s-slurm-.{5}) createdservice\/(%[1]s-slurm-.{5}) created$`, profileName)) + matches := re.FindSubmatch(output) + + if len(matches) < 4 { + return "", "", "", fmt.Errorf("unexpected output format: %s", output) + } + + return string(matches[1]), string(matches[2]), string(matches[3]), nil +} + +func parseSlurmEnvOutput(output []byte) map[string]string { + parts := bytes.Split(output, []byte{'\n'}) + gotOut := make(map[string]string, len(parts)) + for _, part := range parts { + pair := bytes.SplitN(part, []byte{'='}, 2) + if len(pair) == 2 { + gotOut[string(pair[0])] = string(pair[1]) + } + } + return gotOut +} diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go new file mode 100644 index 0000000..3f98444 --- /dev/null +++ b/test/e2e/suite_test.go @@ -0,0 +1,54 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "path/filepath" + "testing" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/test/util" +) + +var ( + kjobctlPath string + cfg *rest.Config + k8sClient client.Client + restClient *rest.RESTClient + ctx context.Context +) + +// Run e2e tests using the Ginkgo runner. +func TestE2E(t *testing.T) { + suiteName := "End To End Suite" + gomega.RegisterFailHandler(ginkgo.Fail) + ginkgo.RunSpecs(t, suiteName) +} + +var _ = ginkgo.BeforeSuite(func() { + dir, _ := util.GetProjectDir() + kjobctlPath = filepath.Join(dir, "bin", "kubectl-kjob") + cfg = util.GetConfigWithContext("") + k8sClient = util.CreateClient(cfg) + restClient = util.CreateRestClient(cfg) + ctx = context.Background() +}) diff --git a/test/framework/framework.go b/test/framework/framework.go new file mode 100644 index 0000000..e28179c --- /dev/null +++ b/test/framework/framework.go @@ -0,0 +1,80 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +type Framework struct { + CRDPath string + DepCRDPaths []string + testEnv *envtest.Environment + cancel context.CancelFunc +} + +func (f *Framework) Init() *rest.Config { + ginkgo.By("bootstrapping test environment") + + f.testEnv = &envtest.Environment{ + CRDDirectoryPaths: append(f.DepCRDPaths, f.CRDPath), + ErrorIfCRDPathMissing: true, + } + + cfg, err := f.testEnv.Start() + + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + gomega.ExpectWithOffset(1, cfg).NotTo(gomega.BeNil()) + + return cfg +} + +func (f *Framework) SetupClient(cfg *rest.Config) (context.Context, client.Client) { + err := v1alpha1.AddToScheme(scheme.Scheme) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + + err = rayv1.AddToScheme(scheme.Scheme) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + + k8sClient, err := client.New(cfg, client.Options{Scheme: scheme.Scheme}) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + gomega.ExpectWithOffset(1, k8sClient).NotTo(gomega.BeNil()) + + ctx, cancel := context.WithCancel(context.Background()) + f.cancel = cancel + + return ctx, k8sClient +} + +func (f *Framework) Teardown() { + ginkgo.By("tearing down the test environment") + if f.cancel != nil { + f.cancel() + } + err := f.testEnv.Stop() + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) +} diff --git a/test/integration/kjobctl/create_test.go b/test/integration/kjobctl/create_test.go new file mode 100644 index 0000000..e6e7543 --- /dev/null +++ b/test/integration/kjobctl/create_test.go @@ -0,0 +1,172 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kjobctl + +import ( + "fmt" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericiooptions" + testingclock "k8s.io/utils/clock/testing" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/test/util" +) + +var _ = ginkgo.Describe("Kjobctl Create", ginkgo.Ordered, ginkgo.ContinueOnFailure, func() { + var ns *corev1.Namespace + + ginkgo.BeforeEach(func() { + ns = &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-"}} + gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed()) + }) + + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + }) + + ginkgo.When("Create a Job", func() { + var ( + jobTemplate *v1alpha1.JobTemplate + profile *v1alpha1.ApplicationProfile + ) + + ginkgo.BeforeEach(func() { + jobTemplate = wrappers.MakeJobTemplate("job-template", ns.Name). + RestartPolicy(corev1.RestartPolicyOnFailure). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "sleep").Obj()). + Obj() + gomega.Expect(k8sClient.Create(ctx, jobTemplate)).To(gomega.Succeed()) + + profile = wrappers.MakeApplicationProfile("profile", ns.Name). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj() + gomega.Expect(k8sClient.Create(ctx, profile)).To(gomega.Succeed()) + }) + + ginkgo.It("Should create job", func() { + testStartTime := time.Now() + + ginkgo.By("Create a Job", func() { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + configFlags := CreateConfigFlagsWithRestConfig(cfg, streams) + + kjobctlCmd := cmd.NewKjobctlCmd(cmd.KjobctlOptions{ + ConfigFlags: configFlags, + IOStreams: streams, + Clock: testingclock.NewFakeClock(testStartTime), + }) + kjobctlCmd.SetOut(out) + kjobctlCmd.SetErr(outErr) + kjobctlCmd.SetArgs([]string{ + "create", "job", + "-n", ns.Name, + "--profile", profile.Name, + "--cmd", "sleep 60s", + "--parallelism", "2", + "--completions", "3", + "--request", "cpu=100m,memory=4Gi", + "--localqueue", "lq1", + "--skip-localqueue-validation", + }) + + err := kjobctlCmd.Execute() + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%s: %s", err, out) + gomega.Expect(outErr.String()).Should(gomega.BeEmpty()) + gomega.Expect(out.String()).Should(gomega.MatchRegexp( + fmt.Sprintf("^job.batch\\/%s-%s-[a-zA-Z0-9]+ created\\n$", profile.Name, "job"))) + }) + + ginkgo.By("Check that Job created", func() { + timestamp := testStartTime.Format(time.RFC3339) + expectedTaskName := fmt.Sprintf("%s_%s", ns.Name, profile.Name) + expectedProfileName := fmt.Sprintf("%s_%s", ns.Name, profile.Name) + expectedTaskID := fmt.Sprintf("%s_%s_%s_%s", userID, timestamp, ns.Name, profile.Name) + + jobList := &batchv1.JobList{} + gomega.Expect(k8sClient.List(ctx, jobList, client.InNamespace(ns.Name))).To(gomega.Succeed()) + gomega.Expect(jobList.Items).To(gomega.HaveLen(1)) + gomega.Expect(jobList.Items[0].Labels[constants.ProfileLabel]).To(gomega.Equal(profile.Name)) + gomega.Expect(jobList.Items[0].Labels[kueueconstants.QueueLabel]).To(gomega.Equal("lq1")) + gomega.Expect(jobList.Items[0].Name).To(gomega.HavePrefix(profile.Name)) + gomega.Expect(jobList.Items[0].Spec.Parallelism).To(gomega.Equal(ptr.To[int32](2))) + gomega.Expect(jobList.Items[0].Spec.Completions).To(gomega.Equal(ptr.To[int32](3))) + gomega.Expect(jobList.Items[0].Spec.Template.Spec.RestartPolicy).To(gomega.Equal(corev1.RestartPolicyOnFailure)) + gomega.Expect(jobList.Items[0].Spec.Template.Spec.Containers).To(gomega.HaveLen(2)) + gomega.Expect(jobList.Items[0].Spec.Template.Spec.Containers[0].Command).To(gomega.Equal([]string{"sleep", "60s"})) + gomega.Expect(jobList.Items[0].Spec.Template.Spec.Containers[0].Resources.Requests).To(gomega.Equal(corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("4Gi"), + })) + gomega.Expect(jobList.Items[0].Spec.Template.Spec.Containers[0].Env).To(gomega.Equal([]corev1.EnvVar{ + {Name: constants.EnvVarNameUserID, Value: userID}, + {Name: constants.EnvVarTaskName, Value: expectedTaskName}, + {Name: constants.EnvVarTaskID, Value: expectedTaskID}, + {Name: constants.EnvVarNameProfile, Value: expectedProfileName}, + {Name: constants.EnvVarNameTimestamp, Value: timestamp}, + })) + gomega.Expect(jobList.Items[0].Spec.Template.Spec.Containers[1].Command).To(gomega.BeNil()) + gomega.Expect(jobList.Items[0].Spec.Template.Spec.Containers[1].Resources.Requests).To(gomega.BeNil()) + gomega.Expect(jobList.Items[0].Spec.Template.Spec.Containers[1].Env).To(gomega.Equal([]corev1.EnvVar{ + {Name: constants.EnvVarNameUserID, Value: userID}, + {Name: constants.EnvVarTaskName, Value: expectedTaskName}, + {Name: constants.EnvVarTaskID, Value: expectedTaskID}, + {Name: constants.EnvVarNameProfile, Value: expectedProfileName}, + {Name: constants.EnvVarNameTimestamp, Value: timestamp}, + })) + }) + }) + + // SimpleDynamicClient didn't allow to check server dry run flag. + ginkgo.It("Shouldn't create job with server dry run", func() { + ginkgo.By("Create a Job", func() { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + configFlags := CreateConfigFlagsWithRestConfig(cfg, streams) + + kjobctlCmd := cmd.NewKjobctlCmd(cmd.KjobctlOptions{ConfigFlags: configFlags, IOStreams: streams}) + kjobctlCmd.SetOut(out) + kjobctlCmd.SetErr(outErr) + kjobctlCmd.SetArgs([]string{"create", "job", "-n", ns.Name, "--profile", profile.Name, "--dry-run", "server"}) + + err := kjobctlCmd.Execute() + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%s: %s", err, out) + gomega.Expect(outErr.String()).Should(gomega.BeEmpty()) + gomega.Expect(out.String()).Should(gomega.MatchRegexp( + fmt.Sprintf("job.batch\\/%s-%s-[a-zA-Z0-9]+ created \\(server dry run\\)", profile.Name, "job"))) + }) + + ginkgo.By("Check that Job not created", func() { + jobList := &batchv1.JobList{} + gomega.Expect(k8sClient.List(ctx, jobList, client.InNamespace(ns.Name))).To(gomega.Succeed()) + gomega.Expect(jobList.Items).To(gomega.BeEmpty()) + }) + }) + }) +}) diff --git a/test/integration/kjobctl/list_test.go b/test/integration/kjobctl/list_test.go new file mode 100644 index 0000000..cae7e9c --- /dev/null +++ b/test/integration/kjobctl/list_test.go @@ -0,0 +1,346 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kjobctl + +import ( + "fmt" + "os" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/duration" + "k8s.io/cli-runtime/pkg/genericiooptions" + testingclock "k8s.io/utils/clock/testing" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/list" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/test/util" +) + +var _ = ginkgo.Describe("Kjobctl List", ginkgo.Ordered, ginkgo.ContinueOnFailure, func() { + var ns *corev1.Namespace + + ginkgo.BeforeEach(func() { + ns = &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-"}} + gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed()) + }) + + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + os.Unsetenv(list.KjobctlListRequestLimitEnvName) + }) + + ginkgo.When("List Jobs", func() { + var ( + j1 *batchv1.Job + j2 *batchv1.Job + j3 *batchv1.Job + ) + + ginkgo.JustBeforeEach(func() { + j1 = wrappers.MakeJob("j1", ns.Name). + Profile("profile1"). + Mode(v1alpha1.JobMode). + Completions(3). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + RestartPolicy(corev1.RestartPolicyOnFailure). + Obj() + gomega.Expect(k8sClient.Create(ctx, j1)).To(gomega.Succeed()) + + j2 = wrappers.MakeJob("j2", ns.Name). + Profile("profile1"). + Mode(v1alpha1.JobMode). + Completions(3). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + RestartPolicy(corev1.RestartPolicyOnFailure). + Obj() + gomega.Expect(k8sClient.Create(ctx, j2)).To(gomega.Succeed()) + + j3 = wrappers.MakeJob("very-long-job-name", ns.Name). + Profile("profile1"). + Mode(v1alpha1.JobMode). + Completions(3). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + RestartPolicy(corev1.RestartPolicyOnFailure). + Obj() + gomega.Expect(k8sClient.Create(ctx, j3)).To(gomega.Succeed()) + }) + + // Simple client set that are using on unit tests not allow paging. + ginkgo.It("Should print jobs list with paging", func() { + streams, _, output, errOutput := genericiooptions.NewTestIOStreams() + configFlags := CreateConfigFlagsWithRestConfig(cfg, streams) + executeTime := time.Now() + kjobctl := cmd.NewKjobctlCmd(cmd.KjobctlOptions{ConfigFlags: configFlags, IOStreams: streams, + Clock: testingclock.NewFakeClock(executeTime)}) + + os.Setenv(list.KjobctlListRequestLimitEnvName, "1") + kjobctl.SetArgs([]string{"list", "job", "--namespace", ns.Name}) + err := kjobctl.Execute() + + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%s: %s", err, output) + gomega.Expect(errOutput.String()).Should(gomega.BeEmpty()) + gomega.Expect(output.String()).Should(gomega.Equal(fmt.Sprintf(`NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 0/3 %s +j2 profile1 0/3 %s +very-long-job-name profile1 0/3 %s +`, + duration.HumanDuration(executeTime.Sub(j1.CreationTimestamp.Time)), + duration.HumanDuration(executeTime.Sub(j2.CreationTimestamp.Time)), + duration.HumanDuration(executeTime.Sub(j3.CreationTimestamp.Time)), + ))) + }) + }) + + ginkgo.When("List RayJobs", func() { + var ( + rj1 *rayv1.RayJob + rj2 *rayv1.RayJob + rj3 *rayv1.RayJob + ) + + ginkgo.JustBeforeEach(func() { + rj1 = wrappers.MakeRayJob("rj1", ns.Name). + Profile("profile1"). + Suspend(true). + Obj() + gomega.Expect(k8sClient.Create(ctx, rj1)).To(gomega.Succeed()) + + rj2 = wrappers.MakeRayJob("rj2", ns.Name). + Profile("profile1"). + Suspend(true). + Obj() + gomega.Expect(k8sClient.Create(ctx, rj2)).To(gomega.Succeed()) + + rj3 = wrappers.MakeRayJob("very-long-job-name", ns.Name). + Profile("profile1"). + Suspend(true). + Obj() + gomega.Expect(k8sClient.Create(ctx, rj3)).To(gomega.Succeed()) + }) + + // Simple client set that are using on unit tests not allow paging. + ginkgo.It("Should print jobs list with paging", func() { + streams, _, output, errOutput := genericiooptions.NewTestIOStreams() + configFlags := CreateConfigFlagsWithRestConfig(cfg, streams) + executeTime := time.Now() + kjobctl := cmd.NewKjobctlCmd(cmd.KjobctlOptions{ConfigFlags: configFlags, IOStreams: streams, + Clock: testingclock.NewFakeClock(executeTime)}) + + os.Setenv(list.KjobctlListRequestLimitEnvName, "1") + kjobctl.SetArgs([]string{"list", "rayjob", "--namespace", ns.Name}) + err := kjobctl.Execute() + + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%s: %s", err, output) + gomega.Expect(errOutput.String()).Should(gomega.BeEmpty()) + gomega.Expect(output.String()).Should(gomega.Equal(fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 %s +rj2 profile1 %s +very-long-job-name profile1 %s +`, + duration.HumanDuration(executeTime.Sub(rj1.CreationTimestamp.Time)), + duration.HumanDuration(executeTime.Sub(rj2.CreationTimestamp.Time)), + duration.HumanDuration(executeTime.Sub(rj3.CreationTimestamp.Time)), + ))) + }) + }) + + ginkgo.When("List RayClusters", func() { + var ( + rc1 *rayv1.RayCluster + rc2 *rayv1.RayCluster + rc3 *rayv1.RayCluster + ) + + ginkgo.JustBeforeEach(func() { + rc1 = wrappers.MakeRayCluster("rc1", ns.Name). + Profile("profile1"). + Spec(*wrappers.MakeRayClusterSpec(). + HeadGroupSpec(rayv1.HeadGroupSpec{ + RayStartParams: make(map[string]string), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "ray-head", Image: "rayproject/ray:2.9.0"}, + }, + }, + }, + }). + WithWorkerGroupSpec(rayv1.WorkerGroupSpec{ + RayStartParams: make(map[string]string), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "ray-worker", Image: "rayproject/ray:2.9.0"}, + }, + }, + }, + }). + Obj()). + Obj() + gomega.Expect(k8sClient.Create(ctx, rc1)).To(gomega.Succeed()) + + rc2 = wrappers.MakeRayCluster("rc2", ns.Name). + Profile("profile1"). + Spec(*wrappers.MakeRayClusterSpec(). + HeadGroupSpec(rayv1.HeadGroupSpec{ + RayStartParams: make(map[string]string), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "ray-head", Image: "rayproject/ray:2.9.0"}, + }, + }, + }, + }). + WithWorkerGroupSpec(rayv1.WorkerGroupSpec{ + RayStartParams: make(map[string]string), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "ray-worker", Image: "rayproject/ray:2.9.0"}, + }, + }, + }, + }). + Obj()). + Obj() + gomega.Expect(k8sClient.Create(ctx, rc2)).To(gomega.Succeed()) + + rc3 = wrappers.MakeRayCluster("very-long-ray-cluster-name", ns.Name). + Profile("profile1"). + Spec(*wrappers.MakeRayClusterSpec(). + HeadGroupSpec(rayv1.HeadGroupSpec{ + RayStartParams: make(map[string]string), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "ray-head", Image: "rayproject/ray:2.9.0"}, + }, + }, + }, + }). + WithWorkerGroupSpec(rayv1.WorkerGroupSpec{ + RayStartParams: make(map[string]string), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "ray-worker", Image: "rayproject/ray:2.9.0"}, + }, + }, + }, + }). + Obj()). + Obj() + gomega.Expect(k8sClient.Create(ctx, rc3)).To(gomega.Succeed()) + }) + + // Simple client set that are using on unit tests not allow paging. + ginkgo.It("Should print ray clusters list with paging", func() { + streams, _, output, errOutput := genericiooptions.NewTestIOStreams() + configFlags := CreateConfigFlagsWithRestConfig(cfg, streams) + executeTime := time.Now() + kjobctl := cmd.NewKjobctlCmd(cmd.KjobctlOptions{ConfigFlags: configFlags, IOStreams: streams, + Clock: testingclock.NewFakeClock(executeTime)}) + + os.Setenv(list.KjobctlListRequestLimitEnvName, "1") + kjobctl.SetArgs([]string{"list", "raycluster", "--namespace", ns.Name}) + err := kjobctl.Execute() + + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%s: %s", err, output) + gomega.Expect(errOutput.String()).Should(gomega.BeEmpty()) + gomega.Expect(output.String()).Should(gomega.Equal(fmt.Sprintf(`NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 0 0 0 0 0 %s +rc2 profile1 0 0 0 0 0 %s +very-long-ray-cluster-name profile1 0 0 0 0 0 %s +`, + duration.HumanDuration(executeTime.Sub(rc1.CreationTimestamp.Time)), + duration.HumanDuration(executeTime.Sub(rc2.CreationTimestamp.Time)), + duration.HumanDuration(executeTime.Sub(rc3.CreationTimestamp.Time)), + ))) + }) + }) + + ginkgo.When("List Slurm Jobs", func() { + var ( + j1 *batchv1.Job + j2 *batchv1.Job + j3 *batchv1.Job + ) + + ginkgo.JustBeforeEach(func() { + j1 = wrappers.MakeJob("j1", ns.Name). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + Completions(3). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + RestartPolicy(corev1.RestartPolicyOnFailure). + Obj() + gomega.Expect(k8sClient.Create(ctx, j1)).To(gomega.Succeed()) + + j2 = wrappers.MakeJob("j2", ns.Name). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + Completions(3). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + RestartPolicy(corev1.RestartPolicyOnFailure). + Obj() + gomega.Expect(k8sClient.Create(ctx, j2)).To(gomega.Succeed()) + + j3 = wrappers.MakeJob("very-long-job-name", ns.Name). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + Completions(3). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + RestartPolicy(corev1.RestartPolicyOnFailure). + Obj() + gomega.Expect(k8sClient.Create(ctx, j3)).To(gomega.Succeed()) + }) + + // Simple client set that are using on unit tests not allow paging. + ginkgo.It("Should print slurm jobs list with paging", func() { + streams, _, output, errOutput := genericiooptions.NewTestIOStreams() + configFlags := CreateConfigFlagsWithRestConfig(cfg, streams) + executeTime := time.Now() + kjobctl := cmd.NewKjobctlCmd(cmd.KjobctlOptions{ConfigFlags: configFlags, IOStreams: streams, + Clock: testingclock.NewFakeClock(executeTime)}) + + os.Setenv(list.KjobctlListRequestLimitEnvName, "1") + kjobctl.SetArgs([]string{"list", "slurm", "--namespace", ns.Name}) + err := kjobctl.Execute() + + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%s: %s", err, output) + gomega.Expect(errOutput.String()).Should(gomega.BeEmpty()) + gomega.Expect(output.String()).Should(gomega.Equal(fmt.Sprintf(`NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 0/3 %s +j2 profile1 0/3 %s +very-long-job-name profile1 0/3 %s +`, + duration.HumanDuration(executeTime.Sub(j1.CreationTimestamp.Time)), + duration.HumanDuration(executeTime.Sub(j2.CreationTimestamp.Time)), + duration.HumanDuration(executeTime.Sub(j3.CreationTimestamp.Time)), + ))) + }) + }) +}) diff --git a/test/integration/kjobctl/suite_test.go b/test/integration/kjobctl/suite_test.go new file mode 100644 index 0000000..c89946a --- /dev/null +++ b/test/integration/kjobctl/suite_test.go @@ -0,0 +1,61 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kjobctl + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/test/framework" +) + +var ( + userID string + cfg *rest.Config + k8sClient client.Client + ctx context.Context + fwk *framework.Framework + crdPath = filepath.Join("..", "..", "..", "config", "crd", "bases") + rayCrdPath = filepath.Join("..", "..", "..", "dep-crds", "ray-operator") +) + +func TestKjobctl(t *testing.T) { + userID = os.Getenv(constants.SystemEnvVarNameUser) + gomega.RegisterFailHandler(ginkgo.Fail) + ginkgo.RunSpecs(t, "Kjobctl Suite") +} + +var _ = ginkgo.BeforeSuite(func() { + fwk = &framework.Framework{ + CRDPath: crdPath, + DepCRDPaths: []string{rayCrdPath}, + } + cfg = fwk.Init() + ctx, k8sClient = fwk.SetupClient(cfg) +}) + +var _ = ginkgo.AfterSuite(func() { + fwk.Teardown() +}) diff --git a/test/integration/kjobctl/util.go b/test/integration/kjobctl/util.go new file mode 100644 index 0000000..02002ad --- /dev/null +++ b/test/integration/kjobctl/util.go @@ -0,0 +1,33 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kjobctl + +import ( + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/client-go/rest" +) + +func CreateConfigFlagsWithRestConfig(cfg *rest.Config, streams genericiooptions.IOStreams) *genericclioptions.ConfigFlags { + return genericclioptions. + NewConfigFlags(true). + WithDiscoveryQPS(50.0). + WithWarningPrinter(streams). + WithWrapConfigFn(func(*rest.Config) *rest.Config { + return cfg + }) +} diff --git a/test/util/constants.go b/test/util/constants.go new file mode 100644 index 0000000..efe9d95 --- /dev/null +++ b/test/util/constants.go @@ -0,0 +1,27 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import "time" + +const ( + Timeout = 5 * time.Second + LongTimeout = 45 * time.Second + Interval = time.Millisecond * 250 + + E2eTestBashImage = "registry.k8s.io/alpine-with-bash:1.0@sha256:0955672451201896cf9e2e5ce30bec0c7f10757af33bf78b7a6afa5672c596f5" +) diff --git a/test/util/util.go b/test/util/util.go new file mode 100644 index 0000000..e0041df --- /dev/null +++ b/test/util/util.go @@ -0,0 +1,171 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path" + "slices" + "strings" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + + kjob "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// DeleteNamespace deletes all objects the tests typically create in the namespace. +func DeleteNamespace(ctx context.Context, c client.Client, ns *corev1.Namespace) error { + if ns == nil { + return nil + } + if err := c.DeleteAllOf(ctx, &batchv1.Job{}, client.InNamespace(ns.Name)); err != nil && !apierrors.IsNotFound(err) { + return err + } + if err := c.DeleteAllOf(ctx, &kjob.ApplicationProfile{}, client.InNamespace(ns.Name)); err != nil && !apierrors.IsNotFound(err) { + return err + } + if err := c.DeleteAllOf(ctx, &kjob.VolumeBundle{}, client.InNamespace(ns.Name)); err != nil && !apierrors.IsNotFound(err) { + return err + } + if err := c.DeleteAllOf(ctx, &kjob.JobTemplate{}, client.InNamespace(ns.Name)); err != nil && !apierrors.IsNotFound(err) { + return err + } + if err := c.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(ns.Name)); err != nil && !apierrors.IsNotFound(err) { + return err + } + if err := c.DeleteAllOf(ctx, &corev1.ConfigMap{}, client.InNamespace(ns.Name)); err != nil && !apierrors.IsNotFound(err) { + return err + } + if err := c.DeleteAllOf(ctx, &corev1.Service{}, client.InNamespace(ns.Name)); err != nil && !apierrors.IsNotFound(err) { + return err + } + if err := c.Delete(ctx, ns); err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} + +// Run executes the provided command +func Run(cmd *exec.Cmd) ([]byte, error) { + dir, _ := GetProjectDir() + cmd.Dir = dir + + command := cmd.String() + fmt.Fprintf(ginkgo.GinkgoWriter, "running: %s\n", command) + output, err := cmd.CombinedOutput() + if err != nil { + return output, fmt.Errorf("%s failed with error: %v", command, err) + } + + return output, nil +} + +// GetNonEmptyLines converts given command output string into individual objects +// according to line breakers, and ignores the empty elements in it. +func GetNonEmptyLines(output string) []string { + return slices.DeleteFunc(strings.Split(output, "\n"), func(s string) bool { + return s == "" + }) +} + +// GetProjectDir will return the directory where the project is +func GetProjectDir() (string, error) { + wd, err := os.Getwd() + if err != nil { + return wd, err + } + return path.Clean(wd + "/../.."), nil +} + +// GetConfigWithContext creates a *rest.Config for talking to a Kubernetes API server +// with a specific context. +func GetConfigWithContext(kContext string) *rest.Config { + cfg, err := config.GetConfigWithContext(kContext) + if err != nil { + fmt.Printf("unable to get kubeconfig for context %q: %s", kContext, err) + os.Exit(1) + } + gomega.ExpectWithOffset(1, cfg).NotTo(gomega.BeNil()) + + cfg.APIPath = "/api" + cfg.ContentConfig.GroupVersion = &schema.GroupVersion{Group: "", Version: "v1"} + cfg.ContentConfig.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + + return cfg +} + +// CreateClient creates a client.Client using the provided config. +func CreateClient(cfg *rest.Config) client.Client { + err := kjob.AddToScheme(scheme.Scheme) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + + client, err := client.New(cfg, client.Options{Scheme: scheme.Scheme}) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + gomega.ExpectWithOffset(1, client).NotTo(gomega.BeNil()) + + return client +} + +// CreateRestClient creates a *rest.RESTClient using the provided config. +func CreateRestClient(cfg *rest.Config) *rest.RESTClient { + restClient, err := rest.RESTClientFor(cfg) + gomega.ExpectWithOffset(1, err).Should(gomega.Succeed()) + gomega.ExpectWithOffset(1, restClient).NotTo(gomega.BeNil()) + + return restClient +} + +func KExecute(ctx context.Context, cfg *rest.Config, client *rest.RESTClient, ns, pod, container string) ([]byte, []byte, error) { + var out, outErr bytes.Buffer + + req := client.Post(). + Resource("pods"). + Namespace(ns). + Name(pod). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: container, + Command: []string{"cat", "/env.out"}, + Stdout: true, + Stderr: true, + }, scheme.ParameterCodec) + + executor, err := remotecommand.NewSPDYExecutor(cfg, "POST", req.URL()) + if err != nil { + return nil, nil, err + } + + if err = executor.StreamWithContext(ctx, remotecommand.StreamOptions{Stdout: &out, Stderr: &outErr}); err != nil { + return nil, nil, err + } + + return out.Bytes(), outErr.Bytes(), nil +}